Skip to content

Commit fab751c

Browse files
authored
fix: relabel CASE and COALESCE timestamp branches instead of panicking (#6347)
create_case_expr reconciled branch types with casts built without a timezone, so a timestamp branch whose Arrow timezone label differed from the others hit assert!(!timezone.is_empty()) in array_with_timezone and panicked. Give the coercion casts the UTC label that every TimestampType value carries, and return an error from array_with_timezone instead of asserting. Closes #6327.
1 parent 8e4bded commit fab751c

2 files changed

Lines changed: 70 additions & 9 deletions

File tree

‎native/core/src/execution/planner.rs‎

Lines changed: 43 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -4799,7 +4799,10 @@ fn create_case_expr(
47994799

48004800
if let Some(coerce_type) = get_coerce_type_for_case_expression(&then_types, else_type.as_ref())
48014801
{
4802-
let cast_options = SparkCastOptions::new_without_timezone(EvalMode::Legacy, false);
4802+
// The branches share a Spark type, so any difference is in the Arrow representation. For
4803+
// a timestamp that is the timezone label, and the cast only relabels it, but Comet's cast
4804+
// still needs a timezone. Every `TimestampType` value in a native plan is labelled UTC.
4805+
let cast_options = SparkCastOptions::new(EvalMode::Legacy, "UTC", false);
48034806

48044807
let when_then_pairs = when_then_pairs
48054808
.iter()
@@ -7863,4 +7866,43 @@ mod tests {
78637866
"unexpected error: {err}"
78647867
);
78657868
}
7869+
7870+
/// CASE branches that share a Spark timestamp type but carry different Arrow timezone labels
7871+
/// are reconciled by relabelling them, which used to panic because the cast had no timezone.
7872+
#[test]
7873+
fn case_reconciles_timestamp_timezone_labels() {
7874+
use arrow::array::{AsArray, BooleanArray, TimestampMicrosecondArray};
7875+
use arrow::datatypes::{TimeUnit, TimestampMicrosecondType};
7876+
7877+
let utc = DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into()));
7878+
for then_label in [Some("Etc/UTC"), None] {
7879+
let then_type = DataType::Timestamp(TimeUnit::Microsecond, then_label.map(Into::into));
7880+
let schema = Schema::new(vec![
7881+
Field::new("b", DataType::Boolean, true),
7882+
Field::new("t", then_type, true),
7883+
Field::new("e", utc.clone(), true),
7884+
]);
7885+
let batch = RecordBatch::try_new(
7886+
Arc::new(schema.clone()),
7887+
vec![
7888+
Arc::new(BooleanArray::from(vec![true, false])),
7889+
Arc::new(
7890+
TimestampMicrosecondArray::from(vec![1, 2]).with_timezone_opt(then_label),
7891+
),
7892+
Arc::new(TimestampMicrosecondArray::from(vec![10, 20]).with_timezone("UTC")),
7893+
],
7894+
)
7895+
.unwrap();
7896+
let case = super::create_case_expr(
7897+
vec![(Arc::new(Column::new("b", 0)), Arc::new(Column::new("t", 1)))],
7898+
Some(Arc::new(Column::new("e", 2))),
7899+
&schema,
7900+
)
7901+
.unwrap();
7902+
let result = case.evaluate(&batch).unwrap().into_array(2).unwrap();
7903+
assert_eq!(result.data_type(), &utc, "{then_label:?}");
7904+
let values = result.as_primitive::<TimestampMicrosecondType>();
7905+
assert_eq!(values.values().to_vec(), vec![1, 20], "{then_label:?}");
7906+
}
7907+
}
78667908
}

‎native/spark-expr/src/utils.rs‎

Lines changed: 27 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -71,7 +71,7 @@ pub fn array_with_timezone(
7171
) -> Result<ArrayRef, ArrowError> {
7272
match array.data_type() {
7373
DataType::Timestamp(TimeUnit::Millisecond, None) => {
74-
assert!(!timezone.is_empty());
74+
require_timezone(&timezone)?;
7575
match to_type {
7676
Some(DataType::Utf8) | Some(DataType::Date32) => Ok(array),
7777
Some(DataType::Timestamp(_, Some(target_tz))) => {
@@ -90,7 +90,7 @@ pub fn array_with_timezone(
9090
}
9191
}
9292
DataType::Timestamp(TimeUnit::Microsecond, None) => {
93-
assert!(!timezone.is_empty());
93+
require_timezone(&timezone)?;
9494
match to_type {
9595
Some(DataType::Utf8) | Some(DataType::Date32) => Ok(array),
9696
Some(DataType::Timestamp(_, Some(target_tz))) => {
@@ -107,7 +107,7 @@ pub fn array_with_timezone(
107107
}
108108
}
109109
DataType::Timestamp(_, None) => {
110-
assert!(!timezone.is_empty());
110+
require_timezone(&timezone)?;
111111
match to_type {
112112
Some(DataType::Utf8) | Some(DataType::Date32) => Ok(array),
113113
Some(DataType::Timestamp(_, Some(target_tz))) => {
@@ -124,7 +124,7 @@ pub fn array_with_timezone(
124124
}
125125
}
126126
DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => {
127-
assert!(!timezone.is_empty());
127+
require_timezone(&timezone)?;
128128
let array = as_primitive_array::<TimestampMicrosecondType>(&array);
129129
let array_with_timezone = array.clone().with_timezone(timezone.clone());
130130
let array = Arc::new(array_with_timezone) as ArrayRef;
@@ -136,7 +136,7 @@ pub fn array_with_timezone(
136136
}
137137
}
138138
DataType::Timestamp(TimeUnit::Millisecond, Some(_)) => {
139-
assert!(!timezone.is_empty());
139+
require_timezone(&timezone)?;
140140
let array = as_primitive_array::<TimestampMillisecondType>(&array);
141141
let array_with_timezone = array.clone().with_timezone(timezone.clone());
142142
let array = Arc::new(array_with_timezone) as ArrayRef;
@@ -161,6 +161,17 @@ pub fn array_with_timezone(
161161
}
162162
}
163163

164+
/// Converting a timestamp needs the session timezone. An empty one means the caller built the
165+
/// conversion without it, which is reported as an error rather than a panic.
166+
fn require_timezone(timezone: &str) -> Result<(), ArrowError> {
167+
if timezone.is_empty() {
168+
return Err(ArrowError::InvalidArgumentError(
169+
"Converting a timestamp requires a timezone, but none was given".to_string(),
170+
));
171+
}
172+
Ok(())
173+
}
174+
164175
fn datetime_cast_err(value: i64) -> ArrowError {
165176
ArrowError::CastError(format!(
166177
"Cannot convert TimestampMicrosecondType {value} to datetime. Comet only supports dates between Jan 1, 262145 BCE and Dec 31, 262143 CE",
@@ -208,7 +219,6 @@ pub(crate) fn timestamp_ntz_to_timestamp(
208219
tz: &str,
209220
to_timezone: Option<&str>,
210221
) -> Result<ArrayRef, ArrowError> {
211-
assert!(!tz.is_empty());
212222
match array.data_type() {
213223
DataType::Timestamp(TimeUnit::Microsecond, None) => {
214224
let array = as_primitive_array::<TimestampMicrosecondType>(&array);
@@ -261,7 +271,6 @@ pub(crate) fn cast_timestamp_to_ntz(
261271
array: ArrayRef,
262272
timezone: &str,
263273
) -> Result<ArrayRef, ArrowError> {
264-
assert!(!timezone.is_empty());
265274
let tz: Tz = timezone.parse()?;
266275
match array.data_type() {
267276
DataType::Timestamp(TimeUnit::Microsecond, Some(_)) => {
@@ -289,7 +298,6 @@ pub(crate) fn cast_timestamp_to_ntz(
289298

290299
/// This takes for special pre-casting cases of Spark. E.g., Timestamp to String.
291300
fn pre_timestamp_cast(array: ArrayRef, timezone: String) -> Result<ArrayRef, ArrowError> {
292-
assert!(!timezone.is_empty());
293301
match array.data_type() {
294302
DataType::Timestamp(_, _) => {
295303
// Spark doesn't output timezone while casting timestamp to string, but arrow's cast
@@ -426,6 +434,17 @@ mod tests {
426434
Arc::new(TimestampMicrosecondArray::from(vec![ts]).with_timezone(tz.to_string()))
427435
}
428436

437+
#[test]
438+
fn test_array_with_timezone_requires_a_timezone() {
439+
let ltz = ts_with_tz("2024-01-15 10:30:00", "UTC");
440+
let err = array_with_timezone(ltz, String::new(), Some(&DataType::Utf8)).unwrap_err();
441+
assert!(err.to_string().contains("requires a timezone"), "{err}");
442+
443+
let ntz = array_containing("2024-01-15 10:30:00");
444+
let utc = DataType::Timestamp(TimeUnit::Microsecond, Some("UTC".into()));
445+
assert!(array_with_timezone(ntz, String::new(), Some(&utc)).is_err());
446+
}
447+
429448
#[test]
430449
fn test_cast_timestamp_to_ntz_utc() {
431450
// In UTC, local time == UTC time, so NTZ value == UTC epoch value

0 commit comments

Comments
 (0)