Skip to content

Commit c7774fd

Browse files
committed
fix: forbid lossy cast(for now)
Signed-off-by: discord9 <[email protected]>
1 parent 7bf54db commit c7774fd

3 files changed

Lines changed: 823 additions & 73 deletions

File tree

‎datafusion/expr-common/src/casts.rs‎

Lines changed: 225 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -1,19 +1,16 @@
1-
// Licensed to the Apache Software Foundation (ASF) under one
2-
// or more contributor license agreements. See the NOTICE file
3-
// distributed with this work for additional information
4-
// regarding copyright ownership. The ASF licenses this file
5-
// to you under the Apache License, Version 2.0 (the
6-
// "License"); you may not use this file except in compliance
7-
// with the License. You may obtain a copy of the License at
1+
// Copyright 2023 Greptime Team
82
//
9-
// http://www.apache.org/licenses/LICENSE-2.0
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
106
//
11-
// Unless required by applicable law or agreed to in writing,
12-
// software distributed under the License is distributed on an
13-
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14-
// KIND, either express or implied. See the License for the
15-
// specific language governing permissions and limitations
16-
// under the License.
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
1714

1815
//! Utilities for casting scalar literals to different data types
1916
//!
@@ -103,7 +100,125 @@ fn is_lossy_temporal_cast(from_type: &DataType, to_type: &DataType) -> bool {
103100
|| (is_date_type(to_type) && from_type.is_temporal())
104101
}
105102

106-
/// Returns true if unwrap_cast_in_comparison supports this numeric type
103+
/// Returns true if casting from `from_type` to `to_type` is unsafe to unwrap in comparisons.
104+
///
105+
/// Such casts can map multiple input values to the same output value (for example
106+
/// `Timestamp(Nanosecond)` to `Timestamp(Millisecond)`, `Float64` to `Float32`,
107+
/// `Decimal(10, 4)` to `Decimal(10, 2)`, or `Decimal(18, 2)` to
108+
/// `Decimal(10, 2)`). They are not safe to unwrap by
109+
/// simply casting the literal back to the input type.
110+
pub fn is_lossy_cast_for_comparison_unwrap(
111+
from_type: &DataType,
112+
to_type: &DataType,
113+
) -> bool {
114+
is_lossy_temporal_cast(from_type, to_type)
115+
|| is_temporal_precision_downcast(from_type, to_type)
116+
|| is_decimal_comparison_unwrap_lossy(from_type, to_type)
117+
|| is_decimal_to_integer_lossy_cast(from_type, to_type)
118+
|| is_float_to_integer_cast(from_type, to_type)
119+
|| is_float_precision_downcast(from_type, to_type)
120+
}
121+
122+
fn is_temporal_precision_downcast(from_type: &DataType, to_type: &DataType) -> bool {
123+
match (from_type, to_type) {
124+
(DataType::Timestamp(from_unit, _), DataType::Timestamp(to_unit, _))
125+
| (DataType::Duration(from_unit), DataType::Duration(to_unit))
126+
| (DataType::Time32(from_unit), DataType::Time32(to_unit))
127+
| (DataType::Time32(from_unit), DataType::Time64(to_unit))
128+
| (DataType::Time64(from_unit), DataType::Time32(to_unit))
129+
| (DataType::Time64(from_unit), DataType::Time64(to_unit)) => {
130+
timestamp_unit_scale(from_unit) > timestamp_unit_scale(to_unit)
131+
}
132+
_ => false,
133+
}
134+
}
135+
136+
fn timestamp_unit_scale(unit: &TimeUnit) -> i128 {
137+
match unit {
138+
TimeUnit::Second => 1,
139+
TimeUnit::Millisecond => MILLISECONDS as i128,
140+
TimeUnit::Microsecond => MICROSECONDS as i128,
141+
TimeUnit::Nanosecond => NANOSECONDS as i128,
142+
}
143+
}
144+
145+
fn decimal_scale(data_type: &DataType) -> Option<i8> {
146+
decimal_precision_and_scale(data_type).map(|(_, scale)| scale)
147+
}
148+
149+
fn decimal_precision_and_scale(data_type: &DataType) -> Option<(u8, i8)> {
150+
match data_type {
151+
DataType::Decimal32(precision, scale)
152+
| DataType::Decimal64(precision, scale)
153+
| DataType::Decimal128(precision, scale) => Some((*precision, *scale)),
154+
_ => None,
155+
}
156+
}
157+
158+
fn is_decimal_comparison_unwrap_lossy(from_type: &DataType, to_type: &DataType) -> bool {
159+
match (
160+
decimal_precision_and_scale(from_type),
161+
decimal_precision_and_scale(to_type),
162+
) {
163+
(Some((from_precision, from_scale)), Some((to_precision, to_scale))) => {
164+
let from_integer_digits = from_precision as i16 - from_scale as i16;
165+
let to_integer_digits = to_precision as i16 - to_scale as i16;
166+
167+
from_scale > to_scale || from_integer_digits > to_integer_digits
168+
}
169+
_ => false,
170+
}
171+
}
172+
173+
fn is_decimal_to_integer_lossy_cast(from_type: &DataType, to_type: &DataType) -> bool {
174+
decimal_scale(from_type).is_some_and(|scale| scale > 0) && is_integer_type(to_type)
175+
}
176+
177+
fn is_float_to_integer_cast(from_type: &DataType, to_type: &DataType) -> bool {
178+
is_floating_point_type(from_type) && is_integer_type(to_type)
179+
}
180+
181+
fn is_float_precision_downcast(from_type: &DataType, to_type: &DataType) -> bool {
182+
match (
183+
float_precision_rank(from_type),
184+
float_precision_rank(to_type),
185+
) {
186+
(Some(from_rank), Some(to_rank)) => from_rank > to_rank,
187+
_ => false,
188+
}
189+
}
190+
191+
fn is_integer_type(data_type: &DataType) -> bool {
192+
matches!(
193+
data_type,
194+
DataType::UInt8
195+
| DataType::UInt16
196+
| DataType::UInt32
197+
| DataType::UInt64
198+
| DataType::Int8
199+
| DataType::Int16
200+
| DataType::Int32
201+
| DataType::Int64
202+
)
203+
}
204+
205+
fn is_floating_point_type(data_type: &DataType) -> bool {
206+
matches!(
207+
data_type,
208+
DataType::Float16 | DataType::Float32 | DataType::Float64
209+
)
210+
}
211+
212+
fn float_precision_rank(data_type: &DataType) -> Option<u8> {
213+
match data_type {
214+
DataType::Float16 => Some(1),
215+
DataType::Float32 => Some(2),
216+
DataType::Float64 => Some(3),
217+
_ => None,
218+
}
219+
}
220+
221+
/// Returns true if unwrap_cast_in_comparison support this numeric type
107222
fn is_supported_numeric_type(data_type: &DataType) -> bool {
108223
matches!(
109224
data_type,
@@ -513,6 +628,101 @@ mod tests {
513628
}
514629
}
515630

631+
#[test]
632+
fn test_lossy_cast_for_comparison_unwrap_detection() {
633+
let ts_s = DataType::Timestamp(TimeUnit::Second, None);
634+
let ts_ms = DataType::Timestamp(TimeUnit::Millisecond, None);
635+
let ts_us = DataType::Timestamp(TimeUnit::Microsecond, None);
636+
let ts_ns = DataType::Timestamp(TimeUnit::Nanosecond, None);
637+
638+
assert!(is_lossy_cast_for_comparison_unwrap(&ts_ns, &ts_ms));
639+
assert!(is_lossy_cast_for_comparison_unwrap(&ts_ns, &ts_us));
640+
assert!(is_lossy_cast_for_comparison_unwrap(&ts_us, &ts_ms));
641+
assert!(is_lossy_cast_for_comparison_unwrap(&ts_ms, &ts_s));
642+
643+
assert!(!is_lossy_cast_for_comparison_unwrap(&ts_s, &ts_ms));
644+
assert!(!is_lossy_cast_for_comparison_unwrap(&ts_ms, &ts_ns));
645+
assert!(!is_lossy_cast_for_comparison_unwrap(&ts_ns, &ts_ns));
646+
assert!(!is_lossy_cast_for_comparison_unwrap(
647+
&DataType::Int32,
648+
&DataType::Int16
649+
));
650+
651+
assert!(is_lossy_cast_for_comparison_unwrap(
652+
&DataType::Timestamp(TimeUnit::Nanosecond, None),
653+
&DataType::Date32
654+
));
655+
assert!(is_lossy_cast_for_comparison_unwrap(
656+
&DataType::Timestamp(TimeUnit::Millisecond, None),
657+
&DataType::Date64
658+
));
659+
assert!(is_lossy_cast_for_comparison_unwrap(
660+
&DataType::Date32,
661+
&DataType::Timestamp(TimeUnit::Nanosecond, None)
662+
));
663+
assert!(is_lossy_cast_for_comparison_unwrap(
664+
&DataType::Date64,
665+
&DataType::Timestamp(TimeUnit::Millisecond, None)
666+
));
667+
668+
let utc = Some("+0:00".into());
669+
assert!(is_lossy_cast_for_comparison_unwrap(
670+
&DataType::Timestamp(TimeUnit::Nanosecond, utc.clone()),
671+
&DataType::Timestamp(TimeUnit::Millisecond, utc)
672+
));
673+
674+
assert!(is_lossy_cast_for_comparison_unwrap(
675+
&DataType::Duration(TimeUnit::Nanosecond),
676+
&DataType::Duration(TimeUnit::Microsecond)
677+
));
678+
assert!(is_lossy_cast_for_comparison_unwrap(
679+
&DataType::Time64(TimeUnit::Nanosecond),
680+
&DataType::Time32(TimeUnit::Millisecond)
681+
));
682+
683+
assert!(is_lossy_cast_for_comparison_unwrap(
684+
&DataType::Decimal128(18, 4),
685+
&DataType::Decimal128(18, 2)
686+
));
687+
assert!(is_lossy_cast_for_comparison_unwrap(
688+
&DataType::Decimal128(18, 2),
689+
&DataType::Decimal128(10, 2)
690+
));
691+
assert!(is_lossy_cast_for_comparison_unwrap(
692+
&DataType::Decimal128(18, 2),
693+
&DataType::Decimal128(12, 4)
694+
));
695+
assert!(is_lossy_cast_for_comparison_unwrap(
696+
&DataType::Decimal128(18, 2),
697+
&DataType::Int64
698+
));
699+
assert!(!is_lossy_cast_for_comparison_unwrap(
700+
&DataType::Decimal128(18, 0),
701+
&DataType::Int64
702+
));
703+
assert!(!is_lossy_cast_for_comparison_unwrap(
704+
&DataType::Decimal64(10, 2),
705+
&DataType::Decimal128(20, 4)
706+
));
707+
assert!(!is_lossy_cast_for_comparison_unwrap(
708+
&DataType::Decimal128(10, 2),
709+
&DataType::Decimal128(12, 4)
710+
));
711+
712+
assert!(is_lossy_cast_for_comparison_unwrap(
713+
&DataType::Float32,
714+
&DataType::Int32
715+
));
716+
assert!(is_lossy_cast_for_comparison_unwrap(
717+
&DataType::Float64,
718+
&DataType::Float32
719+
));
720+
assert!(!is_lossy_cast_for_comparison_unwrap(
721+
&DataType::Float32,
722+
&DataType::Float64
723+
));
724+
}
725+
516726
#[test]
517727
fn test_try_cast_to_type_nulls() {
518728
// test that nulls can be cast to/from all integer types

0 commit comments

Comments
 (0)