Skip to content

Commit c8e6553

Browse files
authored
fix: count the days partition transform in UTC (#6348)
* fix: count the days partition transform in UTC CometDays cast timestamps to dates in the session timezone, while CometHours and Iceberg's days transform count in UTC. Spark cannot evaluate the transform, so there is no Spark result to match. Use UTC so the transforms agree with each other and with Iceberg. Closes #6333. * test: pin UTC flooring of days for a pre-epoch timestamp Check days one microsecond before the epoch in an Asia/Tokyo session. That instant is day -1 in UTC but already 1970-01-01 in Tokyo, so the session-timezone serde returned 0 and fails this check. The existing timestamp literal checks pass either way: they run in the default America/Los_Angeles session, away from a day boundary. * docs: note where days differs from Iceberg for pre-1970 timestamps
1 parent 46d91ef commit c8e6553

2 files changed

Lines changed: 20 additions & 8 deletions

File tree

‎spark/src/main/scala/org/apache/comet/serde/datetime.scala‎

Lines changed: 9 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -22,7 +22,6 @@ package org.apache.comet.serde
2222
import java.util.Locale
2323

2424
import org.apache.spark.sql.catalyst.expressions.{AddMonths, Attribute, Cast, ConvertTimezone, DateAdd, DateDiff, DateFormatClass, DateFromUnixDate, DateSub, DayOfMonth, DayOfWeek, DayOfYear, Days, DivideDTInterval, Expression, FromUTCTimestamp, GetDateField, GetTimestamp, Hour, Hours, LastDay, Literal, MakeDate, MakeDTInterval, MakeInterval, MakeTimestamp, MakeYMInterval, MicrosToTimestamp, MillisToTimestamp, Minute, Month, MonthsBetween, MultiplyDTInterval, MultiplyYMInterval, NextDay, PreciseTimestampConversion, Quarter, Second, SecondsToTimestamp, TimestampAdd, TimestampDiff, ToUnixTimestamp, ToUTCTimestamp, TruncDate, TruncTimestamp, UnixDate, UnixMicros, UnixMillis, UnixSeconds, UnixTimestamp, WeekDay, WeekOfYear, Year}
25-
import org.apache.spark.sql.internal.SQLConf
2625
import org.apache.spark.sql.types.{CalendarIntervalType, DataType, DateType, DoubleType, FloatType, IntegerType, LongType, StringType, TimestampNTZType, TimestampType}
2726
import org.apache.spark.unsafe.types.UTF8String
2827

@@ -822,8 +821,14 @@ object CometHours extends CometExpressionSerde[Hours] {
822821
* For DateType: dates are internally stored as days since epoch, so this is a simple cast to
823822
* integer (same as CometUnixDate).
824823
*
825-
* For TimestampType: uses a timezone-aware Cast(Timestamp to Date) followed by Cast(Date to Int).
826-
* The first cast respects the session timezone to correctly determine the date boundary.
824+
* For TimestampType: uses Cast(Timestamp to Date) in UTC followed by Cast(Date to Int). Spark
825+
* cannot evaluate `days` itself, so this counts days in UTC like [[CometHours]] and Iceberg's
826+
* `days` transform, rather than in the session timezone.
827+
*
828+
* The cast is a true floor, as [[CometHours]] is. Iceberg's `DateTimeUtil.microsToDays` differs
829+
* in one case: it puts a pre-1970 timestamp exactly 999999 microseconds past midnight, such as
830+
* `1969-01-01 00:00:00.999999`, in the previous day. Comet's native kernel for Iceberg's own
831+
* `days` function matches Iceberg there, and this cast does not.
827832
*/
828833
object CometDays extends CometExpressionSerde[Days] {
829834

@@ -846,9 +851,8 @@ object CometDays extends CometExpressionSerde[Days] {
846851
val dateExprOpt = expr.child.dataType match {
847852
case DateType => childExpr
848853
case TimestampType =>
849-
val timezone = SQLConf.get.sessionLocalTimeZone
850854
childExpr.flatMap { child =>
851-
CometCast.castToProto(expr, Some(timezone), DateType, child, CometEvalMode.LEGACY)
855+
CometCast.castToProto(expr, Some("UTC"), DateType, child, CometEvalMode.LEGACY)
852856
}
853857
case _ => None
854858
}

‎spark/src/test/scala/org/apache/comet/CometTemporalExpressionSuite.scala‎

Lines changed: 11 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -839,7 +839,7 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH
839839
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> timezone) {
840840
checkDays(
841841
tsDF.select(col("ts"), getColumnFromExpression(Days(UnresolvedAttribute("ts")))),
842-
tsDF.selectExpr("ts", "unix_date(cast(ts as date))"))
842+
tsDF.selectExpr("ts", "cast(floor(unix_micros(ts) / 86400000000D) as int)"))
843843
}
844844
}
845845
}
@@ -885,8 +885,16 @@ class CometTemporalExpressionSuite extends CometTestBase with AdaptiveSparkPlanH
885885
java.sql.Timestamp.valueOf("2024-06-15 10:30:00"),
886886
DataTypes.TimestampType)))),
887887
dummyDF.selectExpr(
888-
"unix_date(cast(TIMESTAMP('1970-01-01 00:00:00') as date))",
889-
"unix_date(cast(TIMESTAMP('2024-06-15 10:30:00') as date))"))
888+
"cast(floor(unix_micros(TIMESTAMP('1970-01-01 00:00:00')) / 86400000000D) as int)",
889+
"cast(floor(unix_micros(TIMESTAMP('2024-06-15 10:30:00')) / 86400000000D) as int)"))
890+
891+
// One microsecond before the epoch. Days are counted in UTC and floored, so this is day -1
892+
// even though it is already 1970-01-01 in Asia/Tokyo.
893+
withSQLConf(SQLConf.SESSION_LOCAL_TIMEZONE.key -> "Asia/Tokyo") {
894+
checkDays(
895+
dummyDF.select(getColumnFromExpression(Days(Literal(-1L, DataTypes.TimestampType)))),
896+
dummyDF.selectExpr("-1"))
897+
}
890898

891899
// Null handling
892900
checkDays(

0 commit comments

Comments
 (0)