|
| 1 | +// Licensed to the Apache Software Foundation (ASF) under one |
| 2 | +// or more contributor license agreements. See the NOTICE file |
| 3 | +// distributed with this work for additional information |
| 4 | +// regarding copyright ownership. The ASF licenses this file |
| 5 | +// to you under the Apache License, Version 2.0 (the |
| 6 | +// "License"); you may not use this file except in compliance |
| 7 | +// with the License. You may obtain a copy of the License at |
| 8 | +// |
| 9 | +// http://www.apache.org/licenses/LICENSE-2.0 |
| 10 | +// |
| 11 | +// Unless required by applicable law or agreed to in writing, |
| 12 | +// software distributed under the License is distributed on an |
| 13 | +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY |
| 14 | +// KIND, either express or implied. See the License for the |
| 15 | +// specific language governing permissions and limitations |
| 16 | +// under the License. |
| 17 | + |
| 18 | +//! PostgreSQL-style enumerated types. |
| 19 | +
|
| 20 | +use crate::Result; |
| 21 | +use crate::datatype::DataTypeExt; |
| 22 | +use crate::error::{_internal_err, _plan_datafusion_err, _plan_err}; |
| 23 | +use crate::types::extension::DFExtensionType; |
| 24 | +use arrow::array::{Array, AsArray, Int16Array, StringArray}; |
| 25 | +use arrow::datatypes::{DataType, Field, FieldRef}; |
| 26 | +use arrow::util::display::{ArrayFormatter, DisplayIndex, FormatOptions, FormatResult}; |
| 27 | +use arrow_schema::extension::{EXTENSION_TYPE_METADATA_KEY, EXTENSION_TYPE_NAME_KEY}; |
| 28 | +use std::collections::HashMap; |
| 29 | +use std::fmt::Write; |
| 30 | +use std::sync::Arc; |
| 31 | + |
| 32 | +/// An enumerated type: a static, ordered set of text labels, like |
| 33 | +/// `CREATE TYPE mood AS ENUM ('sad', 'ok', 'happy')` in PostgreSQL. |
| 34 | +/// |
| 35 | +/// Values are stored as [`DataType::Int16`] codes, where a code is the position of the label |
| 36 | +/// in the declaration. Comparing and sorting codes therefore follows the declaration order of |
| 37 | +/// the labels, as in PostgreSQL. |
| 38 | +/// |
| 39 | +/// The type is identified by the `datafusion.enum` extension type name. Its metadata is a JSON |
| 40 | +/// object with the type name and the labels, for example |
| 41 | +/// `{"name":"mood","labels":["sad","ok","happy"]}`. As the metadata is part of the Arrow schema, |
| 42 | +/// file formats that store the Arrow schema (such as Parquet and Arrow IPC) keep the type |
| 43 | +/// definition next to the data. |
| 44 | +#[derive(Debug, Clone, PartialEq, Eq, Hash)] |
| 45 | +pub struct DFEnum { |
| 46 | + name: String, |
| 47 | + labels: Vec<String>, |
| 48 | +} |
| 49 | + |
| 50 | +impl DFEnum { |
| 51 | + /// The extension type name of enumerated types. |
| 52 | + pub const NAME: &'static str = "datafusion.enum"; |
| 53 | + |
| 54 | + /// The storage type of the codes. |
| 55 | + pub const STORAGE_TYPE: DataType = DataType::Int16; |
| 56 | + |
| 57 | + /// Creates a new enumerated type named `name` with the `labels` in declaration order. |
| 58 | + pub fn try_new(name: impl Into<String>, labels: Vec<String>) -> Result<Self> { |
| 59 | + let name = name.into(); |
| 60 | + if labels.len() > i16::MAX as usize + 1 { |
| 61 | + return _plan_err!( |
| 62 | + "Enum type {name} has {} labels, at most {} are supported", |
| 63 | + labels.len(), |
| 64 | + i16::MAX as usize + 1 |
| 65 | + ); |
| 66 | + } |
| 67 | + let mut seen = std::collections::HashSet::with_capacity(labels.len()); |
| 68 | + for label in &labels { |
| 69 | + if !seen.insert(label.as_str()) { |
| 70 | + return _plan_err!("Enum label \"{label}\" used more than once"); |
| 71 | + } |
| 72 | + } |
| 73 | + Ok(Self { name, labels }) |
| 74 | + } |
| 75 | + |
| 76 | + /// Creates an enumerated type from the storage type and the serialized metadata of a field. |
| 77 | + pub fn try_from_metadata( |
| 78 | + data_type: &DataType, |
| 79 | + metadata: Option<&str>, |
| 80 | + ) -> Result<Self> { |
| 81 | + if data_type != &Self::STORAGE_TYPE { |
| 82 | + return _plan_err!( |
| 83 | + "Enum type requires storage type {}, got {data_type}", |
| 84 | + Self::STORAGE_TYPE |
| 85 | + ); |
| 86 | + } |
| 87 | + let Some(metadata) = metadata else { |
| 88 | + return _plan_err!("Enum type requires metadata"); |
| 89 | + }; |
| 90 | + let invalid = || _plan_datafusion_err!("Invalid enum type metadata: {metadata}"); |
| 91 | + let value: serde_json::Value = |
| 92 | + serde_json::from_str(metadata).map_err(|_| invalid())?; |
| 93 | + let name = value |
| 94 | + .get("name") |
| 95 | + .and_then(|name| name.as_str()) |
| 96 | + .ok_or_else(invalid)?; |
| 97 | + let labels = value |
| 98 | + .get("labels") |
| 99 | + .and_then(|labels| labels.as_array()) |
| 100 | + .ok_or_else(invalid)? |
| 101 | + .iter() |
| 102 | + .map(|label| label.as_str().map(str::to_owned).ok_or_else(invalid)) |
| 103 | + .collect::<Result<Vec<_>>>()?; |
| 104 | + Self::try_new(name, labels) |
| 105 | + } |
| 106 | + |
| 107 | + /// Returns the enumerated type of `field`, or `None` if the field is not an enum. |
| 108 | + pub fn try_from_field(field: &Field) -> Result<Option<Self>> { |
| 109 | + if field.extension_type_name() != Some(Self::NAME) { |
| 110 | + return Ok(None); |
| 111 | + } |
| 112 | + Self::try_from_metadata(field.data_type(), field.extension_type_metadata()) |
| 113 | + .map(Some) |
| 114 | + } |
| 115 | + |
| 116 | + /// The name of the type. |
| 117 | + pub fn name(&self) -> &str { |
| 118 | + &self.name |
| 119 | + } |
| 120 | + |
| 121 | + /// The labels in declaration order. |
| 122 | + pub fn labels(&self) -> &[String] { |
| 123 | + &self.labels |
| 124 | + } |
| 125 | + |
| 126 | + /// Returns the metadata to attach to a field of this type. |
| 127 | + pub fn field_metadata(&self) -> HashMap<String, String> { |
| 128 | + HashMap::from([ |
| 129 | + (EXTENSION_TYPE_NAME_KEY.to_owned(), Self::NAME.to_owned()), |
| 130 | + ( |
| 131 | + EXTENSION_TYPE_METADATA_KEY.to_owned(), |
| 132 | + self.serialize_metadata_json(), |
| 133 | + ), |
| 134 | + ]) |
| 135 | + } |
| 136 | + |
| 137 | + /// Returns a nullable, unnamed field of this type. |
| 138 | + pub fn to_field_ref(&self) -> FieldRef { |
| 139 | + Arc::new( |
| 140 | + Self::STORAGE_TYPE |
| 141 | + .into_nullable_field() |
| 142 | + .with_metadata(self.field_metadata()), |
| 143 | + ) |
| 144 | + } |
| 145 | + |
| 146 | + /// Returns the code of `label`. |
| 147 | + pub fn code(&self, label: &str) -> Result<i16> { |
| 148 | + self.labels |
| 149 | + .iter() |
| 150 | + .position(|l| l == label) |
| 151 | + .map(|code| code as i16) |
| 152 | + .ok_or_else(|| { |
| 153 | + _plan_datafusion_err!( |
| 154 | + "invalid input value for enum {}: \"{label}\"", |
| 155 | + self.name |
| 156 | + ) |
| 157 | + }) |
| 158 | + } |
| 159 | + |
| 160 | + /// Returns the label of `code`. |
| 161 | + pub fn label(&self, code: i16) -> Result<&str> { |
| 162 | + usize::try_from(code) |
| 163 | + .ok() |
| 164 | + .and_then(|code| self.labels.get(code)) |
| 165 | + .map(String::as_str) |
| 166 | + .ok_or_else(|| { |
| 167 | + _plan_datafusion_err!("invalid code for enum {}: {code}", self.name) |
| 168 | + }) |
| 169 | + } |
| 170 | + |
| 171 | + /// Converts an array of labels (`Utf8`, `LargeUtf8` or `Utf8View`) to an array of codes. |
| 172 | + pub fn encode(&self, array: &dyn Array) -> Result<Int16Array> { |
| 173 | + let codes: HashMap<&str, i16> = self |
| 174 | + .labels |
| 175 | + .iter() |
| 176 | + .enumerate() |
| 177 | + .map(|(code, label)| (label.as_str(), code as i16)) |
| 178 | + .collect(); |
| 179 | + let encode = |label: Option<&str>| { |
| 180 | + label |
| 181 | + .map(|label| { |
| 182 | + codes.get(label).copied().ok_or_else(|| { |
| 183 | + _plan_datafusion_err!( |
| 184 | + "invalid input value for enum {}: \"{label}\"", |
| 185 | + self.name |
| 186 | + ) |
| 187 | + }) |
| 188 | + }) |
| 189 | + .transpose() |
| 190 | + }; |
| 191 | + match array.data_type() { |
| 192 | + DataType::Utf8 => array.as_string::<i32>().iter().map(encode).collect(), |
| 193 | + DataType::LargeUtf8 => array.as_string::<i64>().iter().map(encode).collect(), |
| 194 | + DataType::Utf8View => array.as_string_view().iter().map(encode).collect(), |
| 195 | + other => _internal_err!("Cannot convert {other} to enum {}", self.name), |
| 196 | + } |
| 197 | + } |
| 198 | + |
| 199 | + /// Converts an array of codes to an array of labels. |
| 200 | + pub fn decode(&self, array: &Int16Array) -> Result<StringArray> { |
| 201 | + array |
| 202 | + .iter() |
| 203 | + .map(|code| code.map(|code| self.label(code)).transpose()) |
| 204 | + .collect() |
| 205 | + } |
| 206 | + |
| 207 | + fn serialize_metadata_json(&self) -> String { |
| 208 | + serde_json::json!({ "name": self.name, "labels": self.labels }).to_string() |
| 209 | + } |
| 210 | +} |
| 211 | + |
| 212 | +impl DFExtensionType for DFEnum { |
| 213 | + fn storage_type(&self) -> DataType { |
| 214 | + Self::STORAGE_TYPE |
| 215 | + } |
| 216 | + |
| 217 | + fn serialize_metadata(&self) -> Option<String> { |
| 218 | + Some(self.serialize_metadata_json()) |
| 219 | + } |
| 220 | + |
| 221 | + fn create_array_formatter<'fmt>( |
| 222 | + &self, |
| 223 | + array: &'fmt dyn Array, |
| 224 | + options: &FormatOptions<'fmt>, |
| 225 | + ) -> Result<Option<ArrayFormatter<'fmt>>> { |
| 226 | + let Some(codes) = array.as_primitive_opt() else { |
| 227 | + return _internal_err!("Wrong array type for enum {}", self.name); |
| 228 | + }; |
| 229 | + let display_index = EnumValueDisplayIndex { |
| 230 | + codes, |
| 231 | + labels: self.labels.clone(), |
| 232 | + null_str: options.null(), |
| 233 | + }; |
| 234 | + Ok(Some(ArrayFormatter::new( |
| 235 | + Box::new(display_index), |
| 236 | + options.safe(), |
| 237 | + ))) |
| 238 | + } |
| 239 | +} |
| 240 | + |
| 241 | +/// Pretty printer for enum codes. |
| 242 | +struct EnumValueDisplayIndex<'a> { |
| 243 | + codes: &'a Int16Array, |
| 244 | + labels: Vec<String>, |
| 245 | + null_str: &'a str, |
| 246 | +} |
| 247 | + |
| 248 | +impl DisplayIndex for EnumValueDisplayIndex<'_> { |
| 249 | + fn write(&self, idx: usize, f: &mut dyn Write) -> FormatResult { |
| 250 | + if self.codes.is_null(idx) { |
| 251 | + write!(f, "{}", self.null_str)?; |
| 252 | + return Ok(()); |
| 253 | + } |
| 254 | + let code = self.codes.value(idx); |
| 255 | + match usize::try_from(code).ok().and_then(|c| self.labels.get(c)) { |
| 256 | + Some(label) => write!(f, "{label}")?, |
| 257 | + None => write!(f, "<invalid enum code {code}>")?, |
| 258 | + } |
| 259 | + Ok(()) |
| 260 | + } |
| 261 | +} |
| 262 | + |
| 263 | +#[cfg(test)] |
| 264 | +mod tests { |
| 265 | + use super::*; |
| 266 | + |
| 267 | + fn mood() -> DFEnum { |
| 268 | + DFEnum::try_new("mood", vec!["sad".into(), "ok".into(), "happy".into()]).unwrap() |
| 269 | + } |
| 270 | + |
| 271 | + #[test] |
| 272 | + fn metadata_roundtrip() -> Result<()> { |
| 273 | + let mood = mood(); |
| 274 | + let field = mood.to_field_ref(); |
| 275 | + assert_eq!(DFEnum::try_from_field(&field)?, Some(mood)); |
| 276 | + assert_eq!( |
| 277 | + DFEnum::try_from_field(&Field::new("", DataType::Int16, true))?, |
| 278 | + None |
| 279 | + ); |
| 280 | + Ok(()) |
| 281 | + } |
| 282 | + |
| 283 | + #[test] |
| 284 | + fn rejects_duplicate_labels() { |
| 285 | + let err = DFEnum::try_new("e", vec!["a".into(), "a".into()]).unwrap_err(); |
| 286 | + assert!(err.to_string().contains("used more than once"), "{err}"); |
| 287 | + } |
| 288 | + |
| 289 | + #[test] |
| 290 | + fn encode_decode() -> Result<()> { |
| 291 | + let mood = mood(); |
| 292 | + let labels = StringArray::from(vec![Some("happy"), None, Some("sad")]); |
| 293 | + let codes = mood.encode(&labels)?; |
| 294 | + assert_eq!(codes, Int16Array::from(vec![Some(2), None, Some(0)])); |
| 295 | + assert_eq!(mood.decode(&codes)?, labels); |
| 296 | + |
| 297 | + let err = mood.encode(&StringArray::from(vec!["angry"])).unwrap_err(); |
| 298 | + assert!( |
| 299 | + err.to_string() |
| 300 | + .contains("invalid input value for enum mood: \"angry\""), |
| 301 | + "{err}" |
| 302 | + ); |
| 303 | + Ok(()) |
| 304 | + } |
| 305 | + |
| 306 | + #[test] |
| 307 | + fn pretty_print() -> Result<()> { |
| 308 | + let codes = Int16Array::from(vec![Some(1), None]); |
| 309 | + let formatter = mood() |
| 310 | + .create_array_formatter(&codes, &FormatOptions::default().with_null("NULL"))? |
| 311 | + .unwrap(); |
| 312 | + assert_eq!(formatter.value(0).to_string(), "ok"); |
| 313 | + assert_eq!(formatter.value(1).to_string(), "NULL"); |
| 314 | + Ok(()) |
| 315 | + } |
| 316 | +} |
0 commit comments