Skip to content

Commit 7711800

Browse files
alambcblpclaude
authored
[branch-56] feat: show extension type names in DESCRIBE and information_schema.columns (backport #26893) (#27057)
## Which issue does this PR close? - part of #24461 - Backport of #26893 to `branch-56` - Related to #26866. Co-authored-by: Yury Syrovetsky <[email protected]> Co-authored-by: Claude Opus 5.5 <[email protected]>
1 parent cc631dc commit 7711800

9 files changed

Lines changed: 222 additions & 9 deletions

File tree

‎datafusion/catalog/src/information_schema.rs‎

Lines changed: 22 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@ use datafusion_expr::logical_plan::information_schema::{
3636
pub use datafusion_expr::logical_plan::information_schema::{
3737
InformationSchemataBuilder, schemata_schema,
3838
};
39+
use datafusion_expr::registry::ExtensionTypeRegistryRef;
3940
use datafusion_expr::type_coercion::functions::fields_with_udf;
4041
use datafusion_expr::{
4142
AggregateUDF, Documentation, ReturnFieldArgs, ScalarUDF, Signature, TypeSignature,
@@ -93,6 +94,7 @@ impl InformationSchemaProvider {
9394
catalog_list,
9495
catalog_filter: None,
9596
table_functions: HashMap::new(),
97+
extension_types: None,
9698
},
9799
}
98100
}
@@ -114,6 +116,18 @@ impl InformationSchemaProvider {
114116
self
115117
}
116118

119+
/// Attach the session's extension type registry so that `DESCRIBE`,
120+
/// `SHOW COLUMNS` and `information_schema.columns` show the
121+
/// [`display_name`](datafusion_common::types::DFExtensionType::display_name)
122+
/// of registered extension types.
123+
pub fn with_extension_type_registry(
124+
mut self,
125+
extension_types: ExtensionTypeRegistryRef,
126+
) -> Self {
127+
self.config.extension_types = Some(extension_types);
128+
self
129+
}
130+
117131
/// Restricts the reported objects to the given catalog. When `None`, all
118132
/// catalogs are reported.
119133
pub fn with_catalog_filter(mut self, catalog_name: Option<String>) -> Self {
@@ -136,7 +150,8 @@ impl InformationSchemaProvider {
136150
/// Creates an execution plan for a `DESCRIBE <query>` statement, where
137151
/// `schema` is the output schema of the query.
138152
pub fn describe_schema(&self, schema: &Schema) -> Result<Arc<dyn ExecutionPlan>> {
139-
let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Describe);
153+
let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Describe)
154+
.with_extension_type_registry(self.config.extension_types.clone());
140155
for (position, field) in schema.fields().iter().enumerate() {
141156
builder.add_column("", "", "", position, field);
142157
}
@@ -152,7 +167,8 @@ impl InformationSchemaProvider {
152167
) -> Result<Arc<dyn ExecutionPlan>> {
153168
let table = self.resolve_table(table_ref).await?;
154169

155-
let mut builder = InformationSchemaColumnsBuilder::new(detail);
170+
let mut builder = InformationSchemaColumnsBuilder::new(detail)
171+
.with_extension_type_registry(self.config.extension_types.clone());
156172
for (position, field) in table.schema().fields().iter().enumerate() {
157173
builder.add_column(
158174
&table_ref.catalog,
@@ -407,6 +423,7 @@ struct InformationSchemaConfig {
407423
catalog_list: Arc<dyn CatalogProviderList>,
408424
catalog_filter: Option<String>,
409425
table_functions: HashMap<String, Arc<TableFunction>>,
426+
extension_types: Option<ExtensionTypeRegistryRef>,
410427
}
411428

412429
impl InformationSchemaConfig {
@@ -1041,7 +1058,8 @@ impl PartitionStream for InformationSchemaColumns {
10411058
}
10421059

10431060
fn execute(&self, _ctx: Arc<TaskContext>) -> SendableRecordBatchStream {
1044-
let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Full);
1061+
let mut builder = InformationSchemaColumnsBuilder::new(ColumnsDetail::Full)
1062+
.with_extension_type_registry(self.config.extension_types.clone());
10451063
let config = self.config.clone();
10461064
Box::pin(RecordBatchStreamAdapter::new(
10471065
Arc::clone(&self.schema),
@@ -1223,6 +1241,7 @@ mod tests {
12231241
catalog_list: Arc::new(Fixture),
12241242
catalog_filter: None,
12251243
table_functions: HashMap::new(),
1244+
extension_types: None,
12261245
};
12271246
let mut builder = InformationSchemaTablesBuilder::new();
12281247

‎datafusion/common/src/types/canonical_extensions/json.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ use crate::Result;
66
use crate::types::extension::DFExtensionType;
77
use arrow::datatypes::DataType;
88
use arrow_schema::extension::{ExtensionType, Json};
9+
use std::borrow::Cow;
910

1011
/// Defines the extension type logic for the canonical `arrow.json` extension type. This extension
1112
/// type defines that a particular string field stores JSON values.
@@ -42,4 +43,8 @@ impl DFExtensionType for DFJson {
4243
fn serialize_metadata(&self) -> Option<String> {
4344
self.inner.serialize_metadata()
4445
}
46+
47+
fn display_name(&self) -> Option<Cow<'_, str>> {
48+
Some(Cow::Borrowed("JSON"))
49+
}
4550
}

‎datafusion/common/src/types/canonical_extensions/uuid.rs‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -9,6 +9,7 @@ use arrow::array::{Array, FixedSizeBinaryArray};
99
use arrow::datatypes::DataType;
1010
use arrow::util::display::{ArrayFormatter, DisplayIndex, FormatOptions, FormatResult};
1111
use arrow_schema::extension::{ExtensionType, Uuid};
12+
use std::borrow::Cow;
1213
use std::fmt::Write;
1314
use uuid::Bytes;
1415

@@ -43,6 +44,10 @@ impl DFExtensionType for DFUuid {
4344
self.0.serialize_metadata()
4445
}
4546

47+
fn display_name(&self) -> Option<Cow<'_, str>> {
48+
Some(Cow::Borrowed("UUID"))
49+
}
50+
4651
fn create_array_formatter<'fmt>(
4752
&self,
4853
array: &'fmt dyn Array,

‎datafusion/common/src/types/extension.rs‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@ use crate::error::Result;
66
use arrow::array::Array;
77
use arrow::util::display::{ArrayFormatter, FormatOptions};
88
use arrow_schema::DataType;
9+
use std::borrow::Cow;
910
use std::fmt::Debug;
1011
use std::sync::Arc;
1112

@@ -30,6 +31,7 @@ pub type DFExtensionTypeRef = Arc<dyn DFExtensionType>;
3031
/// Note that DataFusion's extension type support is still young and therefore might not cover all
3132
/// relevant use cases. Currently, the following operations can be customized:
3233
/// - Pretty-printing values in record batches
34+
/// - The type name shown by schema introspection (`DESCRIBE`, `information_schema.columns`)
3335
///
3436
/// # Relation to Arrow's [`ExtensionType`](arrow_schema::extension::ExtensionType)
3537
///
@@ -63,6 +65,16 @@ pub trait DFExtensionType: Debug + Send + Sync {
6365
/// Returns the serialized metadata.
6466
fn serialize_metadata(&self) -> Option<String>;
6567

68+
/// Returns the type's presentation label for schema introspection, such as
69+
/// `DESCRIBE`, `SHOW COLUMNS` and `information_schema.columns`.
70+
///
71+
/// Names may borrow from this instance or be computed from its metadata.
72+
/// The label is not a unique type identifier or SQL syntax guaranteed to round-trip.
73+
/// If `None` is returned, the storage type is shown.
74+
fn display_name(&self) -> Option<Cow<'_, str>> {
75+
None
76+
}
77+
6678
/// Returns an [`ArrayFormatter`] that can format values of this type.
6779
///
6880
/// If `Ok(None)` is returned, the default implementation will be used.

‎datafusion/core/src/execution/session_state.rs‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -419,7 +419,10 @@ impl SessionState {
419419
let information_schema_provider =
420420
InformationSchemaProvider::new(Arc::clone(self.catalog_list()))
421421
.with_information_schema_enabled(true)
422-
.with_table_functions(self.inner.table_functions.clone());
422+
.with_table_functions(self.inner.table_functions.clone())
423+
.with_extension_type_registry(Arc::clone(
424+
&self.inner.extension_types,
425+
));
423426

424427
// If partitioned information schema is enabled, restrict the information schema to
425428
// the containing catalog

‎datafusion/core/src/physical_planner.rs‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2354,7 +2354,9 @@ fn create_information_schema_provider(
23542354
// Do not set a catalog filter. `SHOW` statements describe all catalogs,
23552355
// regardless of `SessionConfig::partition_information_schema`
23562356
let provider = InformationSchemaProvider::new(session.catalog_list());
2357-
provider.with_information_schema_enabled(session.config().information_schema())
2357+
provider
2358+
.with_information_schema_enabled(session.config().information_schema())
2359+
.with_extension_type_registry(Arc::clone(session.extension_type_registry()))
23582360
}
23592361

23602362
/// Expand and align a GROUPING SET expression.

‎datafusion/expr/src/logical_plan/information_schema.rs‎

Lines changed: 104 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -6,12 +6,14 @@
66
//! tables and `SHOW` statements.
77
88
use crate::TableType;
9+
use crate::registry::{ExtensionTypeRegistry, ExtensionTypeRegistryRef};
910
use arrow::array::ArrayRef;
1011
use arrow::array::builder::{BooleanBuilder, StringBuilder, UInt8Builder, UInt64Builder};
1112
use arrow::datatypes::{DataType, Field, Schema, SchemaRef};
1213
use arrow::record_batch::RecordBatch;
1314
use datafusion_common::Result;
1415
use datafusion_common::config::ConfigEntry;
16+
use std::fmt::Write;
1517
use std::sync::Arc;
1618

1719
/// Returns the Arrow schema of `information_schema.tables` rows.
@@ -230,6 +232,7 @@ pub struct InformationSchemaColumnsBuilder {
230232
numeric_scales: Option<UInt64Builder>,
231233
datetime_precisions: Option<UInt64Builder>,
232234
interval_types: Option<StringBuilder>,
235+
extension_types: Option<ExtensionTypeRegistryRef>,
233236
}
234237

235238
impl InformationSchemaColumnsBuilder {
@@ -269,9 +272,21 @@ impl InformationSchemaColumnsBuilder {
269272
datetime_precisions: full
270273
.then(|| UInt64Builder::with_capacity(default_capacity)),
271274
interval_types: full.then(StringBuilder::new),
275+
extension_types: None,
272276
}
273277
}
274278

279+
/// Use the extension type registry to show the
280+
/// [`display_name`](datafusion_common::types::DFExtensionType::display_name)
281+
/// of registered extension types in `data_type`.
282+
pub fn with_extension_type_registry(
283+
mut self,
284+
extension_types: Option<ExtensionTypeRegistryRef>,
285+
) -> Self {
286+
self.extension_types = extension_types;
287+
self
288+
}
289+
275290
/// Append one row describing `field`. Values not needed at this builder's
276291
/// detail level are discarded.
277292
pub fn add_column(
@@ -310,8 +325,13 @@ impl InformationSchemaColumnsBuilder {
310325
let nullable_str = if field.is_nullable() { "YES" } else { "NO" };
311326
self.is_nullables.append_value(nullable_str);
312327

313-
// "System supplied type" --> Use display format of the datatype
314-
self.data_types.append_value(field.data_type().to_string());
328+
// "System supplied type" --> Use the display name of the extension type or the
329+
// display format of the datatype
330+
append_field_type_display_name(
331+
self.extension_types.as_deref(),
332+
field,
333+
&mut self.data_types,
334+
);
315335

316336
// "If data_type identifies a character or bit string type, the
317337
// declared maximum length; null for all other data types or
@@ -435,6 +455,35 @@ impl InformationSchemaColumnsBuilder {
435455
}
436456
}
437457

458+
/// Appends the name of the type of `field` shown in `data_type` to `builder`.
459+
///
460+
/// If the field has an extension type registered in `registry`, appends
461+
/// [`display_name`] if possible. appended. Otherwise, the storage type (raw
462+
/// Arrow physical type name) is appended.
463+
///
464+
/// [`display_name`]: datafusion_common::types::DFExtensionType::display_name
465+
fn append_field_type_display_name(
466+
registry: Option<&dyn ExtensionTypeRegistry>,
467+
field: &Field,
468+
builder: &mut StringBuilder,
469+
) {
470+
let extension_type = registry
471+
.and_then(|registry| registry.create_extension_type_for_field(field).ok())
472+
.flatten();
473+
match extension_type
474+
.as_ref()
475+
.and_then(|extension_type| extension_type.display_name())
476+
{
477+
Some(name) => builder.append_value(name),
478+
None => {
479+
// Write the storage type straight into the builder, then finish the row.
480+
// Writing to a `StringBuilder` does not fail.
481+
let _ = write!(builder, "{}", field.data_type());
482+
builder.append_value("");
483+
}
484+
}
485+
}
486+
438487
/// Returns the Arrow schema of `information_schema.schemata` rows.
439488
///
440489
/// Columns and nullability match
@@ -827,3 +876,56 @@ pub fn show_functions_schema() -> Schema {
827876
Field::new("syntax_example", DataType::Utf8, true),
828877
])
829878
}
879+
880+
#[cfg(test)]
881+
mod tests {
882+
use super::*;
883+
use crate::registry::MemoryExtensionTypeRegistry;
884+
use arrow::array::Array;
885+
use arrow_schema::extension::{EXTENSION_TYPE_NAME_KEY, Uuid};
886+
use std::collections::HashMap;
887+
888+
fn display_names(
889+
registry: Option<&dyn ExtensionTypeRegistry>,
890+
fields: &[Field],
891+
) -> Vec<String> {
892+
let mut builder = StringBuilder::new();
893+
for field in fields {
894+
append_field_type_display_name(registry, field, &mut builder);
895+
}
896+
let array = builder.finish();
897+
(0..array.len())
898+
.map(|i| array.value(i).to_owned())
899+
.collect()
900+
}
901+
902+
/// Fields whose extension type cannot be resolved show their storage type. The
903+
/// resolvable cases are covered by `sql_extension_types.slt`.
904+
#[test]
905+
fn test_append_field_type_display_name_fallbacks() {
906+
let registry = MemoryExtensionTypeRegistry::new_with_canonical_extension_types();
907+
908+
let unregistered = Field::new("x", DataType::Int32, true).with_metadata(
909+
HashMap::from([(EXTENSION_TYPE_NAME_KEY.to_owned(), "example.x".to_owned())]),
910+
);
911+
// `arrow.uuid` with a storage type that the extension type does not support
912+
let invalid =
913+
Field::new("i", DataType::Int32, true).with_metadata(HashMap::from([(
914+
EXTENSION_TYPE_NAME_KEY.to_owned(),
915+
"arrow.uuid".to_owned(),
916+
)]));
917+
assert_eq!(
918+
display_names(Some(&registry), &[unregistered, invalid]),
919+
["Int32", "Int32"]
920+
);
921+
922+
let uuid = Field::new("u", DataType::FixedSizeBinary(16), true)
923+
.with_extension_type(Uuid);
924+
let empty_registry = MemoryExtensionTypeRegistry::new_empty();
925+
assert_eq!(
926+
display_names(Some(&empty_registry), std::slice::from_ref(&uuid)),
927+
["FixedSizeBinary(16)"]
928+
);
929+
assert_eq!(display_names(None, &[uuid]), ["FixedSizeBinary(16)"]);
930+
}
931+
}

‎datafusion/sqllogictest/src/test_context.rs‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@ use datafusion::common::stats::Precision;
2727
use datafusion::common::{DataFusionError, Result, not_impl_err};
2828
use datafusion::functions::math::abs;
2929
use datafusion::logical_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};
30+
use datafusion::logical_expr::registry::MemoryExtensionTypeRegistry;
3031
use datafusion::logical_expr::{
3132
ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
3233
Volatility, create_udf,
@@ -116,6 +117,12 @@ impl TestContext {
116117
}
117118
}
118119

120+
if file_name == "sql_extension_types.slt" {
121+
state_builder = state_builder.with_extension_type_registry(Arc::new(
122+
MemoryExtensionTypeRegistry::new_with_canonical_extension_types(),
123+
));
124+
}
125+
119126
if matches!(
120127
relative_path.file_name().and_then(|name| name.to_str()),
121128
Some("statistics_registry.slt")

0 commit comments

Comments
 (0)