Skip to content

Commit 7684c0c

Browse files
committed
[fix](types) Close timestamp wire parsing and folding gaps
### What problem does this PR solve? Historical TIMESTAMPTZ output could contain offsets rejected by its parsers. Parse wire offsets independently of session fixed-zone limits in both modes. Validate the complete UTC/GMT fixed offset and exclude rejected entries from the timezone cache. Decline FE string folding when the session-local year is outside the range accepted by the new BE formatter. ### Release note Preserve historical TIMESTAMPTZ round trips, reject invalid fixed offsets, and keep FE folding consistent with timestamp display bounds. ### Check List (For Author) - Test: 29 focused ASAN BE tests and 15 FE tests passed. Four BE tests and the new FE boundary test failed before the fixes. clang-format 16 and FE Checkstyle passed. - Behavior changed: Yes; enforce the wire, session offset, and local-year contracts. - Does this need documentation: No.
1 parent 6106e4b commit 7684c0c

6 files changed

Lines changed: 98 additions & 24 deletions

File tree

‎be/src/exprs/function/cast/cast_to_datetimev2_impl.hpp‎

Lines changed: 17 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -651,7 +651,9 @@ inline bool CastToDatetimeV2::from_string_strict_mode_internal(
651651
SET_PARAMS_RET_FALSE_IFN((consume_digit<UInt32, 2>(ptr, end, part[0])),
652652
"invalid hour offset '{}'", std::string {ptr, end});
653653
}
654-
SET_PARAMS_RET_FALSE_IFN(part[0] <= 14, "invalid hour offset '{}'", part[0]);
654+
SET_PARAMS_RET_FALSE_IFN(
655+
part[0] < (type == DataTimeCastEnumType::TIMESTAMP_TZ ? 24U : 15U),
656+
"invalid hour offset '{}'", part[0]);
655657
if (ptr < end) {
656658
if (*ptr == ':') {
657659
++ptr;
@@ -676,17 +678,15 @@ inline bool CastToDatetimeV2::from_string_strict_mode_internal(
676678
"invalid minute offset '{}'", part[1]);
677679
}
678680
}
679-
SET_PARAMS_RET_FALSE_IFN(part[0] != 14 || (part[1] == 0 && second_offset == 0),
680-
"invalid timezone offset '{}'",
681-
combine_tz_offset(sign, part[0], part[1]));
682-
683-
if (second_offset != 0) {
684-
SET_PARAMS_RET_FALSE_IFN(sign != '-' || part[0] <= 12, "invalid hour offset '{}'",
685-
part[0]);
681+
if constexpr (type == DataTimeCastEnumType::TIMESTAMP_TZ) {
682+
// Wire offsets include historical zones outside the session fixed-zone range.
683+
// Use the exact offset even when it has no seconds (for example, Guam's -14:21).
686684
const auto offset = static_cast<int>(part[0] * 3600 + part[1] * 60 + second_offset);
687685
parsed_tz = cctz::fixed_time_zone(cctz::seconds(sign == '-' ? -offset : offset));
688686
} else {
689-
// Preserve the cached lookup for ordinary minute-aligned offsets.
687+
SET_PARAMS_RET_FALSE_IFN(part[0] != 14 || part[1] == 0,
688+
"invalid timezone offset '{}'",
689+
combine_tz_offset(sign, part[0], part[1]));
690690
SET_PARAMS_RET_FALSE_IFN(
691691
TimezoneUtils::find_cctz_time_zone(
692692
combine_tz_offset(sign, part[0], part[1]), parsed_tz),
@@ -952,7 +952,9 @@ inline bool CastToDatetimeV2::from_string_non_strict_mode_internal(
952952
} else {
953953
PROPAGATE_FALSE((consume_digit<UInt32, 2>(ptr, end, hour_offset)));
954954
}
955-
SET_PARAMS_RET_FALSE_IFN(hour_offset <= 14, "invalid hour offset '{}'", hour_offset);
955+
SET_PARAMS_RET_FALSE_IFN(
956+
hour_offset < (type == DataTimeCastEnumType::TIMESTAMP_TZ ? 24U : 15U),
957+
"invalid hour offset '{}'", hour_offset);
956958
if (ptr < end) {
957959
if (*ptr == ':') {
958960
++ptr;
@@ -974,19 +976,15 @@ inline bool CastToDatetimeV2::from_string_non_strict_mode_internal(
974976
"invalid minute offset {}", minute_offset);
975977
}
976978
}
977-
SET_PARAMS_RET_FALSE_IFN(
978-
hour_offset != 14 || (minute_offset == 0 && second_offset == 0),
979-
"invalid timezone offset '{}'",
980-
combine_tz_offset(sign, hour_offset, minute_offset));
981-
982-
if (second_offset != 0) {
983-
SET_PARAMS_RET_FALSE_IFN(sign != '-' || hour_offset <= 12,
984-
"invalid hour offset '{}'", hour_offset);
979+
if constexpr (type == DataTimeCastEnumType::TIMESTAMP_TZ) {
980+
// Match strict parsing: a serialized historical offset is not a session setting.
985981
const auto offset =
986982
static_cast<int>(hour_offset * 3600 + minute_offset * 60 + second_offset);
987983
parsed_tz = cctz::fixed_time_zone(cctz::seconds(sign == '-' ? -offset : offset));
988984
} else {
989-
// Preserve the cached lookup for ordinary minute-aligned offsets.
985+
SET_PARAMS_RET_FALSE_IFN(hour_offset != 14 || minute_offset == 0,
986+
"invalid timezone offset '{}'",
987+
combine_tz_offset(sign, hour_offset, minute_offset));
990988
SET_PARAMS_RET_FALSE_IFN(
991989
TimezoneUtils::find_cctz_time_zone(
992990
combine_tz_offset(sign, hour_offset, minute_offset), parsed_tz),

‎be/src/util/timezone_utils.cpp‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -135,7 +135,10 @@ void TimezoneUtils::load_offsets_to_cache() {
135135
snprintf(min_str, sizeof(min_str), "%02d", minute);
136136
std::string offset_str = (hour >= 0 ? "+" : "") + to_hour_string(hour) + ':' + min_str;
137137
cctz::time_zone result;
138-
parse_tz_offset_string(offset_str, result);
138+
// Rejected endpoint minutes must not be cached as the default UTC zone.
139+
if (!parse_tz_offset_string(offset_str, result)) {
140+
continue;
141+
}
139142
lower_zone_cache_->emplace(offset_str, result);
140143
}
141144
}
@@ -230,7 +233,8 @@ static bool normalize_offset_string(const std::string& timezone, bool allow_hour
230233
hour = std::stoi(std::string(rest));
231234
}
232235

233-
if ((!positive && hour > 12) || (positive && hour > 14) || minute >= 60) {
236+
// Session fixed offsets include the endpoints only when their minutes are zero.
237+
if (minute >= 60 || hour * 60 + minute > (positive ? 14 : 12) * 60) {
234238
return false;
235239
}
236240

‎be/test/runtime/timestamptz_value_test.cpp‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -49,6 +49,14 @@ TEST(TimeStampTzValueTest, ToStringPreservesHistoricalOffsetSeconds) {
4949
const TestCase cases[] = {
5050
{"Asia/Shanghai", 1890, "1890-01-01 08:05:43", "+08:05:43"},
5151
{"America/New_York", 1880, "1879-12-31 19:03:58", "-04:56:02"},
52+
{.zone = "Asia/Manila",
53+
.year = 1800,
54+
.civil = "1799-12-31 08:03:52",
55+
.offset = "-15:56:08"},
56+
{.zone = "Pacific/Guam",
57+
.year = 1800,
58+
.civil = "1799-12-31 09:39:00",
59+
.offset = "-14:21"},
5260
{"Asia/Shanghai", 2024, "2024-01-01 08:00:00", "+08:00"},
5361
{"America/New_York", 2024, "2023-12-31 19:00:00", "-05:00"},
5462
{"Asia/Kathmandu", 2024, "2024-01-01 05:45:00", "+05:45"},
@@ -196,7 +204,9 @@ TEST(TimeStampTzValueTest, HistoricalOffsetsInStrictAndFallbackParsers) {
196204
for (const std::string input :
197205
{"1890-01-01 08:05:43.123456+08:05:43", "1889-12-31 19:03:58.123456-04:56:02",
198206
"1890-01-01 00:00:30.123456+00:00:30", "1889-12-31 23:59:30.123456-00:00:30",
199-
"1890-01-01 08:05:00.123456+08:05"}) {
207+
"1890-01-01 08:05:00.123456+08:05", "1889-12-31 08:03:52.123456-15:56:08",
208+
"1889-12-31 09:39:00.123456-14:21", "1890-01-01 15:00:00.123456+15:00",
209+
"1889-12-31 11:59:59.123456-12:00:01"}) {
200210
SCOPED_TRACE(input);
201211
for (const bool fallback : {false, true}) {
202212
TimestampTzValue parsed;
@@ -213,7 +223,7 @@ TEST(TimeStampTzValueTest, HistoricalOffsetsInStrictAndFallbackParsers) {
213223
}
214224
}
215225
for (const std::string offset : {"+08:60:00", "+08:05:60", "+08:05:", "+08:05:4", "+08:05:430",
216-
"+14:00:01", "+15:00:00", "-13:00:00"}) {
226+
"+24:00:00", "-24:00:00", "+99:00:00"}) {
217227
SCOPED_TRACE(offset);
218228
const auto input = "1890-01-01 00:00:00" + offset;
219229
for (const bool strict : {false, true}) {

‎be/test/util/timezone_utils_test.cpp‎

Lines changed: 20 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,10 +97,29 @@ TEST(TimezoneUtilsTest, ParseOffset) {
9797
EXPECT_FALSE(TimezoneUtils::parse_tz_offset_string("UTC+8:75", result));
9898
}
9999

100+
TEST(TimezoneUtilsTest, FixedOffsetAliasEndpoints) {
101+
TimezoneUtils::clear_timezone_caches();
102+
TimezoneUtils::load_offsets_to_cache();
103+
cctz::time_zone result;
104+
for (const auto* prefix : {"UTC", "GMT", ""}) {
105+
for (const auto* offset : {"+14:00", "-12:00", "+13:59", "-11:59"}) {
106+
EXPECT_TRUE(
107+
TimezoneUtils::parse_tz_offset_string(std::string(prefix) + offset, result));
108+
EXPECT_TRUE(TimezoneUtils::find_cctz_time_zone(std::string(prefix) + offset, result));
109+
}
110+
// Alias normalization must validate the whole offset, including endpoint minutes.
111+
for (const auto* offset : {"+14:01", "-12:01", "+14:30", "-12:30"}) {
112+
EXPECT_FALSE(
113+
TimezoneUtils::parse_tz_offset_string(std::string(prefix) + offset, result));
114+
EXPECT_FALSE(TimezoneUtils::find_cctz_time_zone(std::string(prefix) + offset, result));
115+
}
116+
}
117+
}
118+
100119
TEST(TimezoneUtilsTest, LoadOffsets) {
101120
TimezoneUtils::clear_timezone_caches();
102121
TimezoneUtils::load_offsets_to_cache();
103-
EXPECT_EQ(TimezoneUtils::cache_size(), (13 + 15) * 3);
122+
EXPECT_EQ(TimezoneUtils::cache_size(), (13 + 15) * 3 - 4);
104123

105124
TimezoneUtils::load_timezones_to_cache();
106125
EXPECT_GE(TimezoneUtils::cache_size(), 100);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteral.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,10 @@ protected Expression uncheckedCastTo(DataType targetType) throws AnalysisExcepti
200200
private String getStringValueInSessionTimeZone() {
201201
ZoneId sessionZone = DateUtils.getTimeZone();
202202
ZonedDateTime localDateTime = toJavaDateType().atZone(ZoneId.of("UTC")).withZoneSameInstant(sessionZone);
203+
if (localDateTime.getYear() < 0 || localDateTime.getYear() > 9999) {
204+
// Defer to BE in both modes; CastException would incorrectly fold non-strict casts to NULL.
205+
throw new AnalysisException("TIMESTAMPTZ local year is outside [0, 9999]");
206+
}
203207
String offset = localDateTime.getOffset().getId();
204208
if ("Z".equals(offset)) {
205209
offset = "+00:00";

‎fe/fe-core/src/test/java/org/apache/doris/nereids/trees/expressions/literal/TimestampTzLiteralTest.java‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,15 @@
1717

1818
package org.apache.doris.nereids.trees.expressions.literal;
1919

20+
import org.apache.doris.nereids.exceptions.AnalysisException;
21+
import org.apache.doris.nereids.rules.expression.rules.FoldConstantRuleOnFE;
22+
import org.apache.doris.nereids.trees.expressions.Cast;
2023
import org.apache.doris.nereids.trees.expressions.Expression;
24+
import org.apache.doris.nereids.types.CharType;
25+
import org.apache.doris.nereids.types.DataType;
26+
import org.apache.doris.nereids.types.StringType;
2127
import org.apache.doris.nereids.types.TimeStampTzType;
28+
import org.apache.doris.nereids.types.VarcharType;
2229
import org.apache.doris.qe.ConnectContext;
2330

2431
import org.junit.jupiter.api.Assertions;
@@ -440,4 +447,36 @@ void testFromTimeZoneExplicitWithOffsetString() {
440447
Assertions.assertEquals(0, literal.second);
441448
Assertions.assertEquals(0, literal.microSecond);
442449
}
450+
451+
@Test
452+
void testStringCastLocalYearBounds() {
453+
ConnectContext context = new ConnectContext();
454+
context.setThreadLocalInfo();
455+
try {
456+
for (boolean upper : new boolean[] {false, true}) {
457+
TimestampTzLiteral literal = upper
458+
? new TimestampTzLiteral(TimeStampTzType.of(6), 9999, 12, 31, 23, 59, 59, 999999)
459+
: new TimestampTzLiteral(TimeStampTzType.of(6), 0, 1, 1, 0, 0, 0, 0);
460+
context.getSessionVariable().setTimeZone(upper ? "+08:00" : "-08:00");
461+
for (boolean strict : new boolean[] {false, true}) {
462+
context.getSessionVariable().enableStrictCast = strict;
463+
for (DataType target : new DataType[] {
464+
StringType.INSTANCE, VarcharType.createVarcharType(64), CharType.createCharType(64)}) {
465+
Assertions.assertThrows(AnalysisException.class, () -> literal.checkedCastTo(target));
466+
Cast cast = new Cast(literal, target);
467+
// Defer to BE instead of folding a value whose local year cannot be displayed.
468+
Assertions.assertEquals(cast, FoldConstantRuleOnFE.evaluate(cast, null));
469+
}
470+
}
471+
context.getSessionVariable().setTimeZone("UTC");
472+
Cast validCast = new Cast(literal, StringType.INSTANCE);
473+
Expression folded = FoldConstantRuleOnFE.evaluate(validCast, null);
474+
Assertions.assertInstanceOf(StringLiteral.class, folded);
475+
Assertions.assertEquals(upper ? "9999-12-31 23:59:59.999999+00:00"
476+
: "0000-01-01 00:00:00.000000+00:00", ((StringLiteral) folded).getStringValue());
477+
}
478+
} finally {
479+
ConnectContext.remove();
480+
}
481+
}
443482
}

0 commit comments

Comments
 (0)