Skip to content

Commit 11e2e6c

Browse files
hhr293sunchao
authored andcommitted
[SPARK-60179][SQL] Treat a cast to STRING as time-zone dependent when the source type contains a nested TIMESTAMP_LTZ
### What changes were proposed in this pull request? `Cast.needsTimeZone` incorrectly treats complex-to-STRING casts containing nested LTZ timestamps as time-zone independent, even though `castToString` uses the session time zone. This PR fixes the classification by recursively checking the source type, including Python UDTs. TIMESTAMP_NTZ, DATE and TIME remain time-zone independent. ### Why are the changes needed? The incorrect classification can cause unresolved time-zone access, incorrect expression equivalence, and wrong results when cached queries are reused across session time zones. ### Does this PR introduce any user-facing change? Yes. Complex-to-STRING casts now correctly preserve time-zone semantics, preventing incorrect cross-time-zone cache reuse. ### How was this patch tested? - Added tests for nested types, Python UDTs, expression resolution, semantic equality, and cross-time-zone cache reuse. - Mutation testing confirmed that the affected cache tests fail without the fix, while the scalar TIMESTAMP control still passes. - All 276 Cast tests and 107 CachedTableSuite tests pass. ### Was this patch authored or co-authored using generative AI tooling? Yes. Co-authored-by: Claude Code (Claude Opus 5) Closes #59394 from hhr293/SPARK-60179. Authored-by: hu hengrui <[email protected]> Signed-off-by: Chao Sun <[email protected]>
1 parent 4bc916c commit 11e2e6c

3 files changed

Lines changed: 106 additions & 0 deletions

File tree

‎sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/Cast.scala‎

Lines changed: 22 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -380,6 +380,8 @@ object Cast extends QueryErrorsBase {
380380
* (micro and nanosecond), TIME -> TIMESTAMP_NTZ (whose date fields come from CURRENT_DATE), and
381381
* both directions of TIME <-> TIMESTAMP_LTZ (the LTZ value is an absolute instant, so extracting
382382
* its time-of-day and attaching CURRENT_DATE to a TIME both depend on the session time zone).
383+
* A cast to STRING is zone-dependent whenever one of those timestamps appears anywhere in the
384+
* source type, including under ARRAY/MAP/STRUCT.
383385
*/
384386
def needsTimeZone(from: DataType, to: DataType): Boolean = (from, to) match {
385387
case (VariantType, _) => true
@@ -429,6 +431,26 @@ object Cast extends QueryErrorsBase {
429431
case (fromField, toField) =>
430432
needsTimeZone(fromField.dataType, toField.dataType)
431433
}
434+
// `castToString` recurses into ARRAY/MAP/STRUCT and formats each LTZ leaf with `zoneId`, while
435+
// the complex-to-complex cases above only match a target of the same shape.
436+
case (from, _: StringType) => from.existsRecursively(needsTimeZoneToString)
437+
case _ => false
438+
}
439+
440+
/**
441+
* Whether this type node reads the session time zone when rendered as a string.
442+
* ARRAY/MAP/STRUCT nodes return false; the caller uses `existsRecursively` to inspect
443+
* nested types, while Python UDTs recursively check their `sqlType`.
444+
*
445+
* Only the LTZ families carry an absolute instant that has to be localized;
446+
* TIMESTAMP_NTZ, DATE and TIME render on a fixed grid, and a nested VARIANT renders
447+
* through `VariantVal.toString` at UTC (only a top-level VARIANT reads the zone,
448+
* via the `(VariantType, _)` arm).
449+
*/
450+
private def needsTimeZoneToString(dt: DataType): Boolean = dt match {
451+
case TimestampType | _: TimestampLTZNanosType => true
452+
// A Python UDT renders through its `sqlType`; a Scala UDT through its `stringifyValue`.
453+
case pudt: PythonUserDefinedType => pudt.sqlType.existsRecursively(needsTimeZoneToString)
432454
case _ => false
433455
}
434456

‎sql/catalyst/src/test/scala/org/apache/spark/sql/catalyst/expressions/CastSuiteBase.scala‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2698,6 +2698,59 @@ abstract class CastSuiteBase extends SparkFunSuite with ExpressionEvalHelper {
26982698
assert(!Cast.isTimeToTimestampLTZ(TimestampType, TimestampType))
26992699
}
27002700

2701+
test("SPARK-60179: needsTimeZone covers complex types collapsing to a string") {
2702+
// An LTZ leaf is rendered in the session time zone at any nesting depth.
2703+
Seq(
2704+
ArrayType(TimestampType),
2705+
MapType(IntegerType, TimestampType),
2706+
MapType(TimestampType, IntegerType),
2707+
new StructType().add("t", TimestampType),
2708+
new StructType().add("i", IntegerType).add("t", TimestampLTZNanosType(9)),
2709+
ArrayType(ArrayType(TimestampType)),
2710+
ArrayType(new StructType().add("t", TimestampType)),
2711+
new PythonUserDefinedType(ArrayType(TimestampType), "pyClass", "serializedClass"))
2712+
.foreach { from =>
2713+
assert(Cast.needsTimeZone(from, StringType), s"$from -> STRING")
2714+
// One level further out, the ARRAY/ARRAY case recurses into the pair above.
2715+
assert(Cast.needsTimeZone(ArrayType(from), ArrayType(StringType)),
2716+
s"ARRAY<$from> -> ARRAY<STRING>")
2717+
}
2718+
// TIMESTAMP_NTZ, DATE and TIME render on a fixed grid, and a nested VARIANT renders at UTC.
2719+
Seq(
2720+
ArrayType(TimestampNTZType),
2721+
MapType(IntegerType, DateType),
2722+
new StructType().add("t", TimestampNTZNanosType(9)).add("d", DateType),
2723+
ArrayType(ArrayType(TimeType(TimeType.MAX_PRECISION))),
2724+
ArrayType(VariantType),
2725+
MapType(IntegerType, VariantType),
2726+
new StructType().add("v", VariantType),
2727+
new PythonUserDefinedType(ArrayType(TimestampNTZType), "pyClass", "serializedClass"))
2728+
.foreach { from =>
2729+
assert(!Cast.needsTimeZone(from, StringType), s"$from -> STRING")
2730+
assert(!Cast.needsTimeZone(ArrayType(from), ArrayType(StringType)),
2731+
s"ARRAY<$from> -> ARRAY<STRING>")
2732+
}
2733+
// A top-level VARIANT still reads the zone, element-wise casts included.
2734+
assert(Cast.needsTimeZone(VariantType, StringType))
2735+
assert(Cast.needsTimeZone(ArrayType(VariantType), ArrayType(StringType)))
2736+
}
2737+
2738+
test("SPARK-60179: a complex-to-string cast needs a time zone to be resolved") {
2739+
val arrayOfTimestamp = Literal.create(Seq(0L), ArrayType(TimestampType))
2740+
// Without a time zone the cast cannot be evaluated: the leaf formatter reads `zoneId`.
2741+
assert(!Cast(arrayOfTimestamp, StringType).resolved)
2742+
assert(Cast(arrayOfTimestamp, StringType, UTC_OPT).resolved)
2743+
// `canonicalized` may no longer drop the zone, so two zones are not semantically equal.
2744+
assert(!Cast(arrayOfTimestamp, StringType, UTC_OPT)
2745+
.semanticEquals(Cast(arrayOfTimestamp, StringType, Option(LA.getId))))
2746+
checkEvaluation(Cast(arrayOfTimestamp, StringType, UTC_OPT), "[1970-01-01 00:00:00]")
2747+
checkEvaluation(
2748+
Cast(arrayOfTimestamp, StringType, Option(LA.getId)), "[1969-12-31 16:00:00]")
2749+
// The NTZ control renders on a fixed grid and stays resolvable without a zone.
2750+
val arrayOfTimestampNTZ = Literal.create(Seq(0L), ArrayType(TimestampNTZType))
2751+
assert(Cast(arrayOfTimestampNTZ, StringType).resolved)
2752+
}
2753+
27012754
test("cast timestamp_ntz to time") {
27022755
// Per ANSI rule 15.d the time-of-day fields are extracted and truncated to the target
27032756
// precision; the operation is deterministic and time-zone independent.

‎sql/core/src/test/scala/org/apache/spark/sql/CachedTableSuite.scala‎

Lines changed: 31 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2853,6 +2853,37 @@ class CachedTableSuite extends SharedSparkSession
28532853
}
28542854
}
28552855

2856+
test("SPARK-60179: cached plan is not reused across session time zones for a " +
2857+
"complex-to-string cast") {
2858+
withTempView("t") {
2859+
sql("""SELECT array(timestamp_micros(0)) AS ats,
2860+
| array(array(timestamp_micros(0))) AS aats,
2861+
| timestamp_micros(0) AS ts""".stripMargin)
2862+
.createOrReplaceTempView("t")
2863+
// All three columns hold the same instant, and rendering one as a string localizes it. The
2864+
// scalar column is the control: it was already classified as zone-dependent.
2865+
Seq(
2866+
("ats", "[1970-01-01 00:00:00]", "[1969-12-31 16:00:00]"),
2867+
("aats", "[[1970-01-01 00:00:00]]", "[[1969-12-31 16:00:00]]"),
2868+
("ts", "1970-01-01 00:00:00", "1969-12-31 16:00:00")).foreach {
2869+
case (column, inUTC, inLA) =>
2870+
val query = s"SELECT CAST($column AS STRING) AS v FROM t"
2871+
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "UTC") {
2872+
val cached = sql(query)
2873+
cached.persist()
2874+
try {
2875+
checkAnswer(cached, Row(inUTC))
2876+
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "America/Los_Angeles") {
2877+
checkAnswer(sql(query), Row(inLA))
2878+
}
2879+
} finally {
2880+
cached.unpersist()
2881+
}
2882+
}
2883+
}
2884+
}
2885+
}
2886+
28562887
private def cacheManager = spark.sharedState.cacheManager
28572888

28582889
private def pinTable(

0 commit comments

Comments
 (0)