Skip to content

Commit 87fe5fa

Browse files
committed
[fix](iceberg) Support fractional time travel timestamps
1 parent 6fe0b19 commit 87fe5fa

3 files changed

Lines changed: 25 additions & 10 deletions

File tree

‎fe/fe-connector/fe-connector-iceberg/src/main/java/org/apache/doris/connector/iceberg/IcebergTimeUtils.java‎

Lines changed: 14 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -24,7 +24,9 @@
2424
import java.time.ZoneId;
2525
import java.time.ZoneOffset;
2626
import java.time.format.DateTimeFormatter;
27+
import java.time.format.DateTimeFormatterBuilder;
2728
import java.time.format.DateTimeParseException;
29+
import java.time.temporal.ChronoField;
2830
import java.util.Collections;
2931
import java.util.Map;
3032
import java.util.TreeMap;
@@ -44,9 +46,14 @@ public final class IcebergTimeUtils {
4446
// Doris overrides (CST/PRC -> Asia/Shanghai, UTC/GMT -> UTC = TimeUtils DEFAULT/UTC_TIME_ZONE).
4547
private static final Map<String, String> TIME_ZONE_ALIAS_MAP;
4648

47-
// Byte-parity with legacy TimeUtils.DATETIME_FORMAT (= DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss")),
48-
// the formatter TimeUtils.timeStringToLong uses for a non-digital FOR TIME AS OF datetime string.
49-
private static final DateTimeFormatter DATETIME_FORMAT = DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss");
49+
// Iceberg's committed_at value may include fractional seconds, so the time-travel parser must round-trip
50+
// that value while preserving compatibility with existing whole-second literals.
51+
private static final DateTimeFormatter DATETIME_FORMAT = new DateTimeFormatterBuilder()
52+
.appendPattern("yyyy-MM-dd HH:mm:ss")
53+
.optionalStart()
54+
.appendFraction(ChronoField.NANO_OF_SECOND, 1, 9, true)
55+
.optionalEnd()
56+
.toFormatter();
5057

5158
// Byte-parity with legacy TimeUtils.DATETIME_MS_FORMAT (= DateTimeFormatter.ofPattern("yyyy-MM-dd HH:mm:ss.SSS")),
5259
// the formatter TimeUtils.msTimeStringToLong uses. The rollback_to_timestamp EXECUTE action parses its
@@ -88,11 +95,10 @@ public static ZoneId resolveSessionZone(ConnectorSession session) {
8895
}
8996

9097
/**
91-
* Parses a {@code FOR TIME AS OF} datetime string to epoch-millis in {@code zone}, byte-faithful to legacy
92-
* {@code TimeUtils.timeStringToLong(value, sessionTZ)} (parse {@code yyyy-MM-dd HH:mm:ss} as a local
93-
* date-time, then interpret it in the session zone). Legacy returned {@code -1} on a parse failure and the
94-
* caller ({@code IcebergUtils.getQuerySpecSnapshot}) turned that into a {@code DateTimeException}; we throw
95-
* it directly (fail loud — a parse error is a user mistake, not a not-found).
98+
* Parses a {@code FOR TIME AS OF} datetime string with optional fractional seconds to epoch-millis in
99+
* {@code zone}. Legacy returned {@code -1} on a parse failure and the caller
100+
* ({@code IcebergUtils.getQuerySpecSnapshot}) turned that into a {@code DateTimeException}; we throw it
101+
* directly (fail loud — a parse error is a user mistake, not a not-found).
96102
*/
97103
static long datetimeToMillis(String value, ZoneId zone) {
98104
try {

‎fe/fe-connector/fe-connector-iceberg/src/test/java/org/apache/doris/connector/iceberg/IcebergTimeUtilsTest.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,15 @@ public void datetimeToMillisUtcZone() {
5252
IcebergTimeUtils.datetimeToMillis("2023-06-15 10:30:00", ZoneOffset.UTC));
5353
}
5454

55+
@Test
56+
public void datetimeToMillisAcceptsFractionalSeconds() {
57+
long expected = Instant.parse("2023-06-15T10:30:00.526Z").toEpochMilli();
58+
Assertions.assertEquals(expected,
59+
IcebergTimeUtils.datetimeToMillis("2023-06-15 10:30:00.526", ZoneOffset.UTC));
60+
Assertions.assertEquals(expected,
61+
IcebergTimeUtils.datetimeToMillis("2023-06-15 10:30:00.526000", ZoneOffset.UTC));
62+
}
63+
5564
@Test
5665
public void datetimeToMillisFailsLoudOnMalformedString() {
5766
// WHY: a malformed datetime is a user mistake; legacy returned -1 and the caller threw

‎regression-test/suites/external_table_p0/iceberg/test_iceberg_schema_time_travel_matrix.groovy‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -457,7 +457,7 @@ suite("test_iceberg_schema_time_travel_matrix",
457457

458458
// Scenario TC02/T03/T04: complex-field time travel uses the pre-change nested schema.
459459
List<List<Object>> dorisNestedCp0Time = sql("""
460-
select date_format(date_add(committed_at, interval 1 second), '%Y-%m-%d %H:%i:%s'),
460+
select date_format(committed_at, '%Y-%m-%d %H:%i:%s.%f'),
461461
cast(unix_timestamp(committed_at) * 1000 + 999 as bigint)
462462
from ${dorisNestedTable}\$snapshots
463463
where snapshot_id = ${dorisNestedCp0}
@@ -561,7 +561,7 @@ suite("test_iceberg_schema_time_travel_matrix",
561561

562562
// Scenario T03/T04: string and numeric epoch-millis time travel both resolve the snapshot.
563563
List<List<Object>> cp0TimeRows = sql("""
564-
select date_format(date_add(committed_at, interval 1 second), '%Y-%m-%d %H:%i:%s'),
564+
select date_format(committed_at, '%Y-%m-%d %H:%i:%s.%f'),
565565
cast(unix_timestamp(committed_at) * 1000 + 999 as bigint)
566566
from ${topTable}\$snapshots
567567
where snapshot_id = ${topCp0}

0 commit comments

Comments
 (0)