Skip to content

Commit 4e04fea

Browse files
feat: support casting strings to durations (#11572)
Fixes #4555. The cast kernels supported strings to intervals, but not to durations. This adds casts from `Utf8`, `LargeUtf8`, and `Utf8View` to `Duration` in all four units. Integer strings are interpreted in the target unit, while unit-bearing strings support fixed units, days, and weeks. Month-based units are rejected. - `can_cast_types` accepts the string → duration combination - dispatch through `cast_string_to_duration` (`Utf8`/`LargeUtf8`) and `cast_view_to_duration` (`Utf8View`) - tests: safe casts with nulls and invalid values, unsafe errors, unit-bearing strings, and large/view variants `cargo test -p arrow-cast --lib`: 368 passed.
1 parent 64e0425 commit 4e04fea

3 files changed

Lines changed: 317 additions & 4 deletions

File tree

‎arrow-cast/src/cast/mod.rs‎

Lines changed: 172 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -397,6 +397,7 @@ pub fn can_cast_types(from_type: &DataType, to_type: &DataType) -> bool {
397397
| Time64(Microsecond | Nanosecond)
398398
| Timestamp(Second | Millisecond | Microsecond | Nanosecond, _)
399399
| Interval(_)
400+
| Duration(_)
400401
| BinaryView,
401402
) => true,
402403
(Utf8 | LargeUtf8, Utf8View) => true,
@@ -1173,6 +1174,18 @@ pub fn cast_with_options(
11731174
Interval(IntervalUnit::MonthDayNano) => {
11741175
cast_string_to_month_day_nano_interval::<i32>(array, cast_options)
11751176
}
1177+
Duration(TimeUnit::Second) => {
1178+
cast_string_to_duration::<DurationSecondType, i32>(array, cast_options)
1179+
}
1180+
Duration(TimeUnit::Millisecond) => {
1181+
cast_string_to_duration::<DurationMillisecondType, i32>(array, cast_options)
1182+
}
1183+
Duration(TimeUnit::Microsecond) => {
1184+
cast_string_to_duration::<DurationMicrosecondType, i32>(array, cast_options)
1185+
}
1186+
Duration(TimeUnit::Nanosecond) => {
1187+
cast_string_to_duration::<DurationNanosecondType, i32>(array, cast_options)
1188+
}
11761189
_ => Err(ArrowError::InvalidArgumentError(format!(
11771190
"Casting from {from_type} to {to_type} not supported",
11781191
))),
@@ -1231,6 +1244,18 @@ pub fn cast_with_options(
12311244
Interval(IntervalUnit::MonthDayNano) => {
12321245
cast_view_to_month_day_nano_interval(array, cast_options)
12331246
}
1247+
Duration(TimeUnit::Second) => {
1248+
cast_view_to_duration::<DurationSecondType>(array, cast_options)
1249+
}
1250+
Duration(TimeUnit::Millisecond) => {
1251+
cast_view_to_duration::<DurationMillisecondType>(array, cast_options)
1252+
}
1253+
Duration(TimeUnit::Microsecond) => {
1254+
cast_view_to_duration::<DurationMicrosecondType>(array, cast_options)
1255+
}
1256+
Duration(TimeUnit::Nanosecond) => {
1257+
cast_view_to_duration::<DurationNanosecondType>(array, cast_options)
1258+
}
12341259
_ => Err(ArrowError::InvalidArgumentError(format!(
12351260
"Casting from {from_type} to {to_type} not supported",
12361261
))),
@@ -1306,6 +1331,18 @@ pub fn cast_with_options(
13061331
Interval(IntervalUnit::MonthDayNano) => {
13071332
cast_string_to_month_day_nano_interval::<i64>(array, cast_options)
13081333
}
1334+
Duration(TimeUnit::Second) => {
1335+
cast_string_to_duration::<DurationSecondType, i64>(array, cast_options)
1336+
}
1337+
Duration(TimeUnit::Millisecond) => {
1338+
cast_string_to_duration::<DurationMillisecondType, i64>(array, cast_options)
1339+
}
1340+
Duration(TimeUnit::Microsecond) => {
1341+
cast_string_to_duration::<DurationMicrosecondType, i64>(array, cast_options)
1342+
}
1343+
Duration(TimeUnit::Nanosecond) => {
1344+
cast_string_to_duration::<DurationNanosecondType, i64>(array, cast_options)
1345+
}
13091346
_ => Err(ArrowError::InvalidArgumentError(format!(
13101347
"Casting from {from_type} to {to_type} not supported",
13111348
))),
@@ -2688,6 +2725,141 @@ mod tests {
26882725
}
26892726
}
26902727

2728+
#[test]
2729+
fn test_cast_string_to_duration() {
2730+
let array = StringArray::from(vec![Some("1"), Some("-2"), None, Some("invalid")]);
2731+
let to_type = DataType::Duration(TimeUnit::Second);
2732+
assert!(can_cast_types(array.data_type(), &to_type));
2733+
2734+
let casted = cast(&array, &to_type).unwrap();
2735+
assert_eq!(
2736+
casted.as_primitive::<DurationSecondType>(),
2737+
&DurationSecondArray::from(vec![Some(1), Some(-2), None, None])
2738+
);
2739+
2740+
let options = CastOptions {
2741+
safe: false,
2742+
format_options: FormatOptions::default(),
2743+
};
2744+
let err = cast_with_options(&array, &to_type, &options).unwrap_err();
2745+
assert_eq!(
2746+
err.to_string(),
2747+
"Cast error: Cannot cast string 'invalid' to value of Duration(s) type"
2748+
);
2749+
}
2750+
2751+
#[test]
2752+
fn test_cast_string_with_units_to_duration() {
2753+
let array = StringArray::from(vec![
2754+
Some("1 hour 10 minutes"),
2755+
Some("90 seconds"),
2756+
Some("0.5 hours"),
2757+
Some("-1 hour -30 minutes"),
2758+
Some("2 days"),
2759+
Some("1 week"),
2760+
Some("45 days 14 hours 2 mins 34 secs"),
2761+
None,
2762+
Some("2 months"),
2763+
]);
2764+
let to_type = DataType::Duration(TimeUnit::Second);
2765+
2766+
let casted = cast(&array, &to_type).unwrap();
2767+
assert_eq!(
2768+
casted.as_primitive::<DurationSecondType>(),
2769+
&DurationSecondArray::from(vec![
2770+
Some(4200),
2771+
Some(90),
2772+
Some(1800),
2773+
Some(-5400),
2774+
Some(172800),
2775+
Some(604800),
2776+
Some(3938554),
2777+
None,
2778+
None
2779+
])
2780+
);
2781+
2782+
let options = CastOptions {
2783+
safe: false,
2784+
format_options: FormatOptions::default(),
2785+
};
2786+
let err = cast_with_options(&array, &to_type, &options).unwrap_err();
2787+
assert_eq!(
2788+
err.to_string(),
2789+
"Compute error: Cannot convert interval containing non-zero months to duration"
2790+
);
2791+
}
2792+
2793+
#[test]
2794+
fn test_cast_fractional_calendar_units_to_duration() {
2795+
let array = StringArray::from(vec![
2796+
Some("0.5 months"),
2797+
Some("-0.5 months"),
2798+
Some("1.5 months"),
2799+
Some("0.5 years"),
2800+
Some("0.5 decades"),
2801+
Some("15 days"),
2802+
Some("0.5 days"),
2803+
Some("0.5 weeks"),
2804+
]);
2805+
let to_type = DataType::Duration(TimeUnit::Second);
2806+
2807+
let casted = cast(&array, &to_type).unwrap();
2808+
assert_eq!(
2809+
casted.as_primitive::<DurationSecondType>(),
2810+
&DurationSecondArray::from(vec![
2811+
None,
2812+
None,
2813+
None,
2814+
None,
2815+
None,
2816+
Some(1_296_000),
2817+
Some(43_200),
2818+
Some(302_400)
2819+
])
2820+
);
2821+
2822+
let options = CastOptions {
2823+
safe: false,
2824+
format_options: FormatOptions::default(),
2825+
};
2826+
let err = cast_with_options(&array, &to_type, &options).unwrap_err();
2827+
assert_eq!(
2828+
err.to_string(),
2829+
"Cast error: Cannot cast string '0.5 months' to value of Duration(s) type"
2830+
);
2831+
}
2832+
2833+
#[test]
2834+
fn test_cast_large_string_and_view_to_duration() {
2835+
let to_type = DataType::Duration(TimeUnit::Nanosecond);
2836+
for array in [
2837+
Arc::new(LargeStringArray::from(vec![
2838+
Some("10"),
2839+
Some("1 minute"),
2840+
Some("1 day"),
2841+
None,
2842+
])) as ArrayRef,
2843+
Arc::new(StringViewArray::from(vec![
2844+
Some("10"),
2845+
Some("1 minute"),
2846+
Some("1 day"),
2847+
None,
2848+
])) as ArrayRef,
2849+
] {
2850+
let casted = cast(&array, &to_type).unwrap();
2851+
assert_eq!(
2852+
casted.as_primitive::<DurationNanosecondType>(),
2853+
&DurationNanosecondArray::from(vec![
2854+
Some(10),
2855+
Some(60_000_000_000),
2856+
Some(86_400_000_000_000),
2857+
None
2858+
])
2859+
);
2860+
}
2861+
}
2862+
26912863
#[test]
26922864
fn test_cast_date32_to_int32() {
26932865
let array = Date32Array::from(vec![10000, 17890]);

‎arrow-cast/src/cast/string.rs‎

Lines changed: 111 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,6 +3,10 @@
33
// SPDX-FileContributor: Licensed to the Apache Software Foundation (ASF) under one or more contributor license agreements
44

55
use crate::cast::*;
6+
use crate::parse::{
7+
IntervalParseConfig, IntervalUnit as IntervalParseUnit, NANOS_PER_DAY,
8+
parse_interval_month_day_nano_config,
9+
};
610
use arrow_buffer::NullBuffer;
711

812
pub(crate) fn cast_bool_to_string<A>(array: &dyn Array) -> Result<ArrayRef, ArrowError>
@@ -119,6 +123,113 @@ fn parse_string_iter<
119123
Ok(Arc::new(array) as ArrayRef)
120124
}
121125

126+
/// Parse a string to a duration, accepting integer strings (interpreted
127+
/// in the target unit) and interval strings with units, such as
128+
/// `"1 hour 10 minutes"`.
129+
fn parse_duration<D>(string: &str) -> Result<i64, ArrowError>
130+
where
131+
D: ArrowTemporalType<Native = i64> + Parser,
132+
{
133+
if let Some(duration) = D::parse(string) {
134+
return Ok(duration);
135+
}
136+
137+
let (unit, scale) = match D::DATA_TYPE {
138+
DataType::Duration(TimeUnit::Second) => (IntervalParseUnit::Second, 1_000_000_000),
139+
DataType::Duration(TimeUnit::Millisecond) => (IntervalParseUnit::Millisecond, 1_000_000),
140+
DataType::Duration(TimeUnit::Microsecond) => (IntervalParseUnit::Microsecond, 1_000),
141+
DataType::Duration(TimeUnit::Nanosecond) => (IntervalParseUnit::Nanosecond, 1),
142+
_ => unreachable!(),
143+
};
144+
145+
let interval = parse_interval_month_day_nano_config(
146+
string,
147+
IntervalParseConfig::new(unit).reject_fractional_calendar_units(),
148+
)
149+
.map_err(|_| {
150+
ArrowError::CastError(format!(
151+
"Cannot cast string '{string}' to value of {} type",
152+
D::DATA_TYPE
153+
))
154+
})?;
155+
156+
if interval.months != 0 {
157+
return Err(ArrowError::ComputeError(
158+
"Cannot convert interval containing non-zero months to duration".to_string(),
159+
));
160+
}
161+
162+
let nanos = NANOS_PER_DAY as i128 * interval.days as i128 + interval.nanoseconds as i128;
163+
i64::try_from(nanos / scale as i128).map_err(|_| {
164+
ArrowError::CastError(format!(
165+
"Cannot cast string '{string}' to value of {} type",
166+
D::DATA_TYPE
167+
))
168+
})
169+
}
170+
171+
/// Casts generic string arrays to a Duration type
172+
pub(crate) fn cast_string_to_duration<D, O>(
173+
array: &dyn Array,
174+
cast_options: &CastOptions,
175+
) -> Result<ArrayRef, ArrowError>
176+
where
177+
D: ArrowTemporalType<Native = i64> + Parser,
178+
O: OffsetSizeTrait,
179+
{
180+
let string_array = array.as_string::<O>();
181+
parse_duration_iter::<D, _>(
182+
string_array.iter(),
183+
cast_options,
184+
string_array.nulls().cloned(),
185+
)
186+
}
187+
188+
/// Casts string view arrays to a Duration type
189+
pub(crate) fn cast_view_to_duration<D>(
190+
array: &dyn Array,
191+
cast_options: &CastOptions,
192+
) -> Result<ArrayRef, ArrowError>
193+
where
194+
D: ArrowTemporalType<Native = i64> + Parser,
195+
{
196+
let string_view_array = array.as_string_view();
197+
parse_duration_iter::<D, _>(
198+
string_view_array.iter(),
199+
cast_options,
200+
string_view_array.nulls().cloned(),
201+
)
202+
}
203+
204+
fn parse_duration_iter<'a, D, I>(
205+
iter: I,
206+
cast_options: &CastOptions,
207+
nulls: Option<NullBuffer>,
208+
) -> Result<ArrayRef, ArrowError>
209+
where
210+
D: ArrowTemporalType<Native = i64> + Parser,
211+
I: Iterator<Item = Option<&'a str>>,
212+
{
213+
let array = if cast_options.safe {
214+
let iter = iter.map(|x| {
215+
let v = x?;
216+
parse_duration::<D>(v).ok()
217+
});
218+
219+
unsafe { PrimitiveArray::<D>::from_trusted_len_iter(iter) }
220+
} else {
221+
let v = iter
222+
.map(|x| match x {
223+
Some(v) => parse_duration::<D>(v),
224+
None => Ok(D::Native::default()),
225+
})
226+
.collect::<Result<Vec<_>, ArrowError>>()?;
227+
PrimitiveArray::try_new(v.into(), nulls)?
228+
};
229+
230+
Ok(Arc::new(array) as ArrayRef)
231+
}
232+
122233
/// Casts generic string arrays to an ArrowTimestampType (TimeStampNanosecondArray, etc.)
123234
pub(crate) fn cast_string_to_timestamp<O: OffsetSizeTrait, T: ArrowTimestampType>(
124235
array: &dyn Array,

0 commit comments

Comments
 (0)