Skip to content

Commit 49c67c2

Browse files
Feat/map sql extension types (#21714)
## Which issue does this PR close? - Closes #21626 ## Rationale for this change Ensures that DataFusion natively maps SQL UUID and JSON types to their respective Arrow Canonical Extension Types. This standardizes how these types are stored and metadata-tagged, allowing for seamless interoperability with other Arrow-native tools. <!-- Why are you proposing this change? If this is already explained clearly in the issue then this section is not needed. Explaining clearly why changes are proposed helps reviewers understand your changes and offer better suggestions for fixes. --> ## What changes are included in this PR? - Updated planner.rs to map UUID to FixedSizeBinary(16) and JSON to Utf8 with official arrow.uuid and arrow.json metadata keys. - Arrow Standards: Integrated arrow-schema extension constants (EXTENSION_TYPE_NAME_KEY, Uuid::NAME, etc.) and included the required empty string metadata for JSON types. - Metadata Preservation in Statements: Refactored statement.rs to use alias_with_metadata. This ensures that in CREATE TABLE ... AS SELECT (CTAS) operations, the extension metadata survives the projection and is applied to the new table schema. <!-- There is no need to duplicate the description in the issue here but it is sometimes worth providing a summary of the individual changes in this PR. --> ## Are these changes tested? Yes, - Added sql_extension_types.slt which performs end-to-end testing of CAST and CREATE TABLE for both UUID and JSON. - Existing SQL logic tests pass, confirming no regressions in standard type handling. <!-- We typically require tests for all PRs in order to: 1. Prevent the code from being accidentally broken by subsequent changes 2. Serve as another way to document the expected behavior of the code If tests are not included in your PR, please explain why (for example, are they covered by existing tests)? --> ## Are there any user-facing changes? Yes. UUID and JSON are now officially recognized SQL keywords in DataFusion DDL and Casts, automatically resulting in Arrow-compliant canonical extension types. <!-- If there are user-facing changes then we may require documentation to be updated before approving the PR. --> <!-- If there are any breaking changes to public APIs, please add the `api change` label. --> --------- Co-authored-by: Adrian Garcia Badaracco <[email protected]>
1 parent be44fab commit 49c67c2

8 files changed

Lines changed: 159 additions & 43 deletions

File tree

‎Cargo.lock‎

Lines changed: 1 addition & 0 deletions
Some generated files are not rendered by default. Learn more about customizing how changed files appear on GitHub.

‎datafusion/expr/src/logical_plan/builder.rs‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -429,7 +429,6 @@ impl LogicalPlanBuilder {
429429
let mut fields = ValuesFields::new();
430430
for j in 0..n_cols {
431431
let field_type = schema.field(j).data_type();
432-
let field_nullable = schema.field(j).is_nullable();
433432
for row in values.iter() {
434433
let value = &row[j];
435434
let data_type = value.get_type(schema)?;
@@ -444,7 +443,13 @@ impl LogicalPlanBuilder {
444443
);
445444
}
446445
}
447-
fields.push(field_type.to_owned(), field_nullable);
446+
let field = schema.field(j);
447+
448+
fields.push_with_metadata(
449+
field_type.clone(),
450+
field.is_nullable(),
451+
Some(FieldMetadata::new_from_field(field)),
452+
);
448453
}
449454

450455
Self::infer_inner(values, fields, schema)
@@ -1835,10 +1840,6 @@ impl ValuesFields {
18351840
Self::default()
18361841
}
18371842

1838-
pub fn push(&mut self, data_type: DataType, nullable: bool) {
1839-
self.push_with_metadata(data_type, nullable, None);
1840-
}
1841-
18421843
pub fn push_with_metadata(
18431844
&mut self,
18441845
data_type: DataType,

‎datafusion/sql/Cargo.toml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,7 @@ recursive_protection = ["dep:recursive", "dep:stacker"]
5252
# They are used for testing purposes only, so they are in the dev-dependencies section.
5353
[dependencies]
5454
arrow = { workspace = true }
55+
arrow-schema = { workspace = true }
5556
bigdecimal = { workspace = true }
5657
chrono = { workspace = true }
5758
datafusion-common = { workspace = true, features = ["sql"] }

‎datafusion/sql/src/planner.rs‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,7 @@ use std::vec;
1010

1111
use crate::utils::make_decimal_type;
1212
use arrow::datatypes::*;
13+
use arrow_schema::extension::{Json, Uuid};
1314
use datafusion_common::TableReference;
1415
use datafusion_common::config::SqlParserOptions;
1516
use datafusion_common::datatype::{DataTypeExt, FieldExt};
@@ -664,6 +665,14 @@ impl<'a, S: ContextProvider> SqlToRel<'a, S> {
664665

665666
// If no type_planner can handle this type, use the default conversion
666667
match sql_type {
668+
// Canonical Arrow extension types
669+
SQLDataType::Uuid => Ok(Arc::new(
670+
Field::new("", DataType::FixedSizeBinary(16), true)
671+
.with_extension_type(Uuid),
672+
)),
673+
SQLDataType::JSON => Ok(Arc::new(
674+
Field::new("", DataType::Utf8, true).with_extension_type(Json::default()),
675+
)),
667676
SQLDataType::Array(ArrayElemTypeDef::AngleBracket(inner_sql_type)) => {
668677
// Arrays may be multi-dimensional.
669678
Ok(self.convert_data_type_to_field(inner_sql_type)?.into_list())

‎datafusion/sql/tests/cases/params.rs‎

Lines changed: 4 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -822,11 +822,11 @@ fn test_insert_infer_with_metadata() {
822822
@r#"
823823
** Initial Plan:
824824
Dml: op=[Insert Into] table=[person_with_uuid_extension]
825-
Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
825+
Projection: column1 AS id, column2 AS first_name, column3 AS last_name
826826
Values: ($1, $2, $3)
827827
** Final Plan:
828828
Dml: op=[Insert Into] table=[person_with_uuid_extension]
829-
Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
829+
Projection: column1 AS id, column2 AS first_name, column3 AS last_name
830830
Values: (FixedSizeBinary(16, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16") FieldMetadata { inner: {"ARROW:extension:name": "arrow.uuid"} } AS $1, Utf8("Alan") AS $2, Utf8("Turing") AS $3)
831831
"#
832832
);
@@ -844,11 +844,11 @@ fn test_insert_infer_with_metadata() {
844844
** Initial Plan:
845845
Prepare: "my_plan" [FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>, Utf8, Utf8]
846846
Dml: op=[Insert Into] table=[person_with_uuid_extension]
847-
Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
847+
Projection: column1 AS id, column2 AS first_name, column3 AS last_name
848848
Values: ($1, $2, $3)
849849
** Final Plan:
850850
Dml: op=[Insert Into] table=[person_with_uuid_extension]
851-
Projection: CAST(column1 AS FixedSizeBinary(16)<{"ARROW:extension:name": "arrow.uuid"}>) AS id, column2 AS first_name, column3 AS last_name
851+
Projection: column1 AS id, column2 AS first_name, column3 AS last_name
852852
Values: (FixedSizeBinary(16, "1,2,3,4,5,6,7,8,9,10,11,12,13,14,15,16") FieldMetadata { inner: {"ARROW:extension:name": "arrow.uuid"} } AS $1, Utf8("Alan") AS $2, Utf8("Turing") AS $3)
853853
"#
854854
);

‎datafusion/sql/tests/common/mod.rs‎

Lines changed: 0 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -387,11 +387,6 @@ impl TypePlanner for CustomTypePlanner {
387387
sql_type: &sqlparser::ast::DataType,
388388
) -> Result<Option<FieldRef>> {
389389
match sql_type {
390-
sqlparser::ast::DataType::Uuid => Ok(Some(Arc::new(
391-
Field::new("", DataType::FixedSizeBinary(16), true).with_metadata(
392-
Metadata::new().with("ARROW:extension:name", "arrow.uuid"),
393-
),
394-
))),
395390
sqlparser::ast::DataType::Datetime(precision) => {
396391
let precision = match precision {
397392
Some(0) => TimeUnit::Second,

‎datafusion/sqllogictest/src/test_context.rs‎

Lines changed: 1 addition & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -16,8 +16,7 @@ use arrow::array::{
1616
};
1717
use arrow::buffer::ScalarBuffer;
1818
use arrow::datatypes::{
19-
DataType, Field, FieldRef, Fields, Metadata, Schema, SchemaRef, TimeUnit, UInt32Type,
20-
UnionFields,
19+
DataType, Field, Fields, Schema, SchemaRef, TimeUnit, UInt32Type, UnionFields,
2120
};
2221
use arrow::record_batch::RecordBatch;
2322
use datafusion::catalog::{
@@ -28,7 +27,6 @@ use datafusion::common::stats::Precision;
2827
use datafusion::common::{DataFusionError, Result, not_impl_err};
2928
use datafusion::functions::math::abs;
3029
use datafusion::logical_expr::async_udf::{AsyncScalarUDF, AsyncScalarUDFImpl};
31-
use datafusion::logical_expr::planner::TypePlanner;
3230
use datafusion::logical_expr::{
3331
ColumnarValue, Expr, ScalarFunctionArgs, ScalarUDF, ScalarUDFImpl, Signature,
3432
Volatility, create_udf,
@@ -58,7 +56,6 @@ use datafusion::physical_plan::operator_statistics::{
5856
};
5957
use datafusion::physical_plan::statistics::StatisticsArgs;
6058
use log::info;
61-
use sqlparser::ast;
6259
use tempfile::TempDir;
6360

6461
mod custom_sort_pushdown;
@@ -72,22 +69,6 @@ pub struct TestContext {
7269
test_dir: Option<TempDir>,
7370
}
7471

75-
#[derive(Debug)]
76-
struct SqlLogicTestTypePlanner;
77-
78-
impl TypePlanner for SqlLogicTestTypePlanner {
79-
fn plan_type_field(&self, sql_type: &ast::DataType) -> Result<Option<FieldRef>> {
80-
match sql_type {
81-
ast::DataType::Uuid => Ok(Some(Arc::new(
82-
Field::new("", DataType::FixedSizeBinary(16), true).with_metadata(
83-
Metadata::new().with("ARROW:extension:name", "arrow.uuid"),
84-
),
85-
))),
86-
_ => Ok(None),
87-
}
88-
}
89-
}
90-
9172
impl TestContext {
9273
pub fn new(ctx: SessionContext) -> Self {
9374
Self {
@@ -133,14 +114,6 @@ impl TestContext {
133114
}
134115
}
135116

136-
if matches!(
137-
relative_path.file_name().and_then(|name| name.to_str()),
138-
Some("cast_extension_type_metadata.slt")
139-
) {
140-
state_builder =
141-
state_builder.with_type_planner(Arc::new(SqlLogicTestTypePlanner));
142-
}
143-
144117
if matches!(
145118
relative_path.file_name().and_then(|name| name.to_str()),
146119
Some("statistics_registry.slt")
Lines changed: 136 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,136 @@
1+
# SPDX-License-Identifier: Apache-2.0
2+
# SPDX-FileCopyrightText: See the NOTICE file distributed with this work for additional information regarding copyright ownership
3+
# SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
4+
5+
# Tests for built-in mapping of SQL types to Arrow canonical extension types.
6+
# See: https://arrow.apache.org/docs/format/CanonicalExtensions.html
7+
8+
#######
9+
# UUID Extension Type
10+
#######
11+
12+
# CAST to UUID preserves extension metadata
13+
query ?T
14+
SELECT CAST(X'00010203040506070809000102030506' AS UUID),
15+
arrow_metadata(CAST(X'00010203040506070809000102030506' AS UUID), 'ARROW:extension:name');
16+
----
17+
00010203040506070809000102030506 arrow.uuid
18+
19+
# Test table creation and various insertion methods (Null, Naked Binary, and Cast)
20+
statement ok
21+
CREATE TABLE test_uuid (value UUID);
22+
23+
query TTT
24+
DESCRIBE test_uuid;
25+
----
26+
value FixedSizeBinary(16) YES
27+
28+
# 1. NULL insertion
29+
statement ok
30+
INSERT INTO test_uuid VALUES (NULL);
31+
32+
# 2. Exercises Implicit Promotion to UUID from binary
33+
statement ok
34+
INSERT INTO test_uuid VALUES (X'00010203040506070809000102030506');
35+
36+
# 3. Explicit CAST
37+
statement ok
38+
INSERT INTO test_uuid VALUES (CAST(X'00010203040506070809000102030507' AS UUID));
39+
40+
query T
41+
SELECT arrow_metadata(value, 'ARROW:extension:name') FROM test_uuid;
42+
----
43+
arrow.uuid
44+
arrow.uuid
45+
arrow.uuid
46+
47+
# Batch of multiple values (some NULL)
48+
statement ok
49+
INSERT INTO test_uuid VALUES
50+
(CAST(X'00010203040506070809000102030506' AS UUID)),
51+
(NULL),
52+
(CAST(X'00010203040506070809000102030507' AS UUID));
53+
54+
query I
55+
SELECT COUNT(*) FROM test_uuid;
56+
----
57+
6
58+
59+
statement ok
60+
DROP TABLE test_uuid;
61+
62+
#######
63+
# JSON Extension Type
64+
#######
65+
66+
# CAST to JSON preserves extension metadata
67+
query TT
68+
SELECT CAST('{"a": 1}' AS JSON),
69+
arrow_metadata(CAST('{"a": 1}' AS JSON), 'ARROW:extension:name');
70+
----
71+
{"a": 1} arrow.json
72+
73+
statement ok
74+
CREATE TABLE test_json (data JSON);
75+
76+
query TTT
77+
DESCRIBE test_json;
78+
----
79+
data Utf8 YES
80+
81+
# 1. NULL insertion
82+
statement ok
83+
INSERT INTO test_json VALUES (NULL);
84+
85+
# 2. Bare string literal (Implicit Promotion)
86+
statement ok
87+
INSERT INTO test_json VALUES ('{"bare": "literal"}');
88+
89+
# 3. Explicit CAST
90+
statement ok
91+
INSERT INTO test_json VALUES (CAST('{"hello": "world"}' AS JSON));
92+
93+
query T
94+
SELECT arrow_metadata(data, 'ARROW:extension:name') FROM test_json;
95+
----
96+
arrow.json
97+
arrow.json
98+
arrow.json
99+
100+
# Verify Type Decay: Concatenation results in a bare Utf8 string
101+
query T
102+
SELECT arrow_metadata(data || '{"extra": 1}', 'ARROW:extension:name') FROM test_json;
103+
----
104+
NULL
105+
NULL
106+
NULL
107+
108+
statement ok
109+
DROP TABLE test_json;
110+
111+
#######
112+
# Error Handling
113+
#######
114+
115+
# Error: Cannot cast an integer to UUID
116+
statement error Unsupported CAST from Int64 to FixedSizeBinary\(16\)
117+
SELECT CAST(12345 AS UUID);
118+
119+
# Error: Cannot cast a short string to UUID
120+
statement error Unsupported CAST from Utf8 to FixedSizeBinary\(16\)
121+
SELECT CAST('too-short' AS UUID);
122+
123+
# This should trigger cast plan_err: "Types don't match and no valid cast exists"
124+
# Because 'not-a-uuid' is Utf8, and it's being compared to a UUID target in a VALUES list.
125+
statement ok
126+
CREATE TABLE t1 (id UUID);
127+
128+
# Then, try to insert the invalid type
129+
statement error Types don't match and no valid cast exists, received data of type Utf8 for field of type FixedSizeBinary\(16\)
130+
INSERT INTO t1 VALUES ('not-a-uuid');
131+
132+
statement ok
133+
INSERT INTO t1 VALUES (NULL);
134+
135+
statement ok
136+
DROP TABLE t1;

0 commit comments

Comments
 (0)