Skip to content

Commit d939afc

Browse files
committed
fix: arrow_cast must not elide a metadata-changing cast
`ArrowCastFunc::simplify` short-circuited whenever the argument's data type already equalled the requested one and returned the argument untouched. That was sound while a same-type cast could not change anything, but a cast target's metadata is authoritative, and `arrow_cast` names a storage type and nothing else - its declared return field (`return_field_from_args`) never carries metadata. So casting an extension-typed value back to its own storage type is not a no-op, it strips the extension metadata, and the short circuit swallowed it: SELECT arrow_metadata(arrow_cast(uuid_val, 'FixedSizeBinary(16)'), 'ARROW:extension:name') -- 'arrow.uuid' before, NULL after which is an `arrow.uuid` value that escaped a cast back to plain `FixedSizeBinary(16)` (apache#22079). Elide the cast only when the argument carries no metadata for it to strip. Checking metadata emptiness rather than just `ARROW:extension:name` follows from the rule itself: the target field carries no metadata at all, so any metadata on the argument is metadata the cast removes.
1 parent 0bb24a2 commit d939afc

2 files changed

Lines changed: 60 additions & 4 deletions

File tree

‎datafusion/functions/src/core/arrow_cast.rs‎

Lines changed: 21 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -27,8 +27,8 @@ use datafusion_common::{
2727

2828
use datafusion_expr::simplify::{ExprSimplifyResult, SimplifyContext};
2929
use datafusion_expr::{
30-
Coercion, ColumnarValue, Documentation, Expr, ReturnFieldArgs, ScalarFunctionArgs,
31-
ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
30+
Coercion, ColumnarValue, Documentation, Expr, ExprSchemable, ReturnFieldArgs,
31+
ScalarFunctionArgs, ScalarUDFImpl, Signature, TypeSignatureClass, Volatility,
3232
};
3333
use datafusion_macros::user_doc;
3434

@@ -161,8 +161,25 @@ impl ScalarUDFImpl for ArrowCastFunc {
161161
let [source_arg, type_arg] = take_function_args(self.name(), args)?;
162162
let target_type = data_type_from_type_arg(self.name(), &type_arg)?;
163163
let source_type = info.get_data_type(&source_arg)?;
164-
let new_expr = if source_type == target_type {
165-
// the argument's data type is already the correct type
164+
// `arrow_cast` names a `DataType` and nothing else: its declared return
165+
// field (see `return_field_from_args`) never carries metadata. Dropping
166+
// the cast is therefore only sound when the argument has nothing for the
167+
// cast to drop - the cast target's metadata is authoritative, so a cast
168+
// whose argument carries metadata is a metadata-changing cast even when
169+
// the two data types are identical.
170+
//
171+
// Eliding it there would leave, for example,
172+
// `arrow_cast(uuid_val, 'FixedSizeBinary(16)')` still labelled
173+
// `ARROW:extension:name = arrow.uuid`, i.e. an extension value that
174+
// escaped a cast back to its storage type.
175+
// See https://github.com/apache/datafusion/issues/22079
176+
let is_noop = source_type == target_type && {
177+
let (_, source_field) = source_arg.to_field(info.schema().as_ref())?;
178+
source_field.metadata().is_empty()
179+
};
180+
let new_expr = if is_noop {
181+
// the argument's data type is already the correct type and it
182+
// carries no metadata that the cast would strip
166183
source_arg
167184
} else {
168185
// Use an actual cast to get the correct type

‎datafusion/sqllogictest/test_files/cast_extension_type_metadata.slt‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,3 +61,42 @@ FROM (
6161
);
6262
----
6363
00010203040506070809000102030506 NULL
64+
65+
# `arrow_cast` names a storage type and nothing else, so casting an
66+
# extension-typed value back to its own storage type is not a no-op: it strips
67+
# the extension metadata. The same-type short circuit in `ArrowCastFunc::simplify`
68+
# must not swallow it.
69+
# See https://github.com/apache/datafusion/issues/22079
70+
query ?T
71+
SELECT
72+
arrow_cast(uuid_val, 'FixedSizeBinary(16)'),
73+
arrow_metadata(arrow_cast(uuid_val, 'FixedSizeBinary(16)'), 'ARROW:extension:name')
74+
FROM (
75+
SELECT CAST(arrow_cast(X'00010203040506070809000102030506', 'FixedSizeBinary(16)') AS UUID) AS uuid_val
76+
);
77+
----
78+
00010203040506070809000102030506 NULL
79+
80+
# ... and casting it to a different storage type, through `arrow_cast` rather
81+
# than a SQL CAST, strips it too
82+
query ?T
83+
SELECT
84+
arrow_cast(uuid_val, 'Binary'),
85+
arrow_metadata(arrow_cast(uuid_val, 'Binary'), 'ARROW:extension:name')
86+
FROM (
87+
SELECT CAST(arrow_cast(X'00010203040506070809000102030506', 'FixedSizeBinary(16)') AS UUID) AS uuid_val
88+
);
89+
----
90+
00010203040506070809000102030506 NULL
91+
92+
# An `arrow_cast` whose argument carries no metadata has nothing to strip and is
93+
# still simplified away entirely
94+
query TT
95+
EXPLAIN SELECT arrow_cast(a, 'Int64') FROM (SELECT arrow_cast(1, 'Int64') AS a);
96+
----
97+
logical_plan
98+
01)Projection: Int64(1) AS arrow_cast(a,Utf8("Int64"))
99+
02)--EmptyRelation: rows=1
100+
physical_plan
101+
01)ProjectionExec: expr=[1 as arrow_cast(a,Utf8("Int64"))]
102+
02)--PlaceholderRowExec

0 commit comments

Comments
 (0)