Skip to content

Commit b9874e4

Browse files
morningmanclaude
andcommitted
[fix](paimon) Resolve static LTZ partition values from the TIMESTAMPTZ binding only
### What problem does this PR solve? Issue Number: None Related PR: #65868, #68786 Problem Summary: #68786 binds every Paimon TIMESTAMP WITH LOCAL TIME ZONE column as TIMESTAMPTZ; enable.mapping.timestamp_tz no longer selects a DATETIMEV2 binding. The cast value of a static LTZ partition is therefore always the instant in UTC with its offset, and the session-zone path of PaimonWriteBinding, with its check for a session time inside a DST gap, can no longer run. That check existed because the BE turned a DATETIMEV2 row value into an instant with cctz while the FE used java.time; a TIMESTAMPTZ row value and the static value now come from the same FE cast. Remove the path and the session zone it needed, and keep the rejection of an instant whose local time the FE JVM zone repeats. PaimonConnectorTransactionTest, added by #68858 after this change was written, builds the write binding through the new create signature. ### Release note None ### Check List (For Author) - Test: Unit Test - PaimonWriteBindingTest: the LTZ zone move, the overlap rejection and the round trip through Paimon's parser, all for TIMESTAMPTZ values. - The Paimon connector (701) and fe-connector-spi (147) tests, and the fe-core binding, BindSink and insert command tests (52) pass. - Behavior changed: No (the removed path had become unreachable) - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent c2de6f9 commit b9874e4

6 files changed

Lines changed: 69 additions & 129 deletions

File tree

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonConnectorMetadata.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -946,7 +946,7 @@ private long pinnedSnapshotId(Table table, Map<String, String> resolved) {
946946
* overrides — PST and EST resolve via {@code SHORT_IDS}, so a 4-entry-only map would still
947947
* reject them (verified by JDK harness).
948948
*/
949-
static final Map<String, String> SESSION_TIME_ZONE_ALIASES;
949+
private static final Map<String, String> SESSION_TIME_ZONE_ALIASES;
950950

951951
static {
952952
Map<String, String> m = new java.util.TreeMap<>(String.CASE_INSENSITIVE_ORDER);

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonWriteBinding.java‎

Lines changed: 11 additions & 43 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,6 @@
1717

1818
package org.apache.doris.connector.paimon;
1919

20-
import org.apache.doris.connector.spi.ConnectorColumn;
2120
import org.apache.doris.connector.spi.DorisConnectorException;
2221
import org.apache.doris.connector.spi.handle.ConnectorWriteHandle;
2322

@@ -30,7 +29,6 @@
3029
import java.time.LocalDateTime;
3130
import java.time.OffsetDateTime;
3231
import java.time.ZoneId;
33-
import java.time.ZonedDateTime;
3432
import java.time.format.DateTimeFormatter;
3533
import java.util.Base64;
3634
import java.util.Collections;
@@ -63,8 +61,8 @@ private PaimonWriteBinding(String tableName, FileStoreTable table,
6361
}
6462

6563
static PaimonWriteBinding create(PaimonTableHandle handle, FileStoreTable table,
66-
Map<String, String> hadoopConfig, ConnectorWriteHandle writeHandle, String sessionTimeZone) {
67-
Map<String, String> staticPartition = resolveStaticPartition(table, writeHandle, sessionTimeZone);
64+
Map<String, String> hadoopConfig, ConnectorWriteHandle writeHandle) {
65+
Map<String, String> staticPartition = resolveStaticPartition(table, writeHandle);
6866
FileStoreTable writeTable = configureTableForWrite(table, writeHandle.isOverwrite(), staticPartition);
6967
return new PaimonWriteBinding(handle.getDatabaseName() + "." + handle.getTableName(),
7068
writeTable, hadoopConfig, writeHandle.isOverwrite(), staticPartition);
@@ -89,8 +87,7 @@ static FileStoreTable configureTableForWrite(FileStoreTable table, boolean overw
8987
* {@code partition.default-name}, and every other value is the one the written rows carry, cast to the
9088
* column type.
9189
*/
92-
static Map<String, String> resolveStaticPartition(FileStoreTable table, ConnectorWriteHandle writeHandle,
93-
String sessionTimeZone) {
90+
static Map<String, String> resolveStaticPartition(FileStoreTable table, ConnectorWriteHandle writeHandle) {
9491
Map<String, String> canonicalNames = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
9592
for (String partitionKey : table.partitionKeys()) {
9693
canonicalNames.put(partitionKey, partitionKey);
@@ -112,8 +109,7 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto
112109
() -> "missing the cast value of static partition column " + key);
113110
if (table.rowType().getField(canonicalName).type().getTypeRoot()
114111
== DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) {
115-
value = toSdkLocalTime(canonicalName, value, isTimestampTz(writeHandle, canonicalName),
116-
sessionTimeZone, writeHandle.isOverwrite());
112+
value = toSdkLocalTime(canonicalName, value, writeHandle.isOverwrite());
117113
}
118114
if (writeHandle.isOverwrite() && defaultPartitionName.equals(value)) {
119115
// Paimon's static overwrite reads this string as the NULL partition of any partition type, so
@@ -127,45 +123,17 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto
127123
return result;
128124
}
129125

130-
private static boolean isTimestampTz(ConnectorWriteHandle writeHandle, String columnName) {
131-
for (ConnectorColumn column : writeHandle.getBoundTargetColumns()) {
132-
if (column.getName().equalsIgnoreCase(columnName)) {
133-
return "TIMESTAMPTZ".equalsIgnoreCase(column.getType().getTypeName());
134-
}
135-
}
136-
throw new DorisConnectorException("Paimon partition column is missing from the write schema: "
137-
+ columnName);
138-
}
139-
140126
/**
141127
* Paimon parses a static overwrite value of a TIMESTAMP WITH LOCAL TIME ZONE column as local time in the FE
142-
* JVM's default zone, so the value is moved there from the zone it was written in: the session zone for a
143-
* DATETIMEV2 value, UTC for a TIMESTAMPTZ value, which carries its offset. Paimon's parser takes a space,
144-
* not ISO's 'T', between the date and the time.
145-
*
146-
* <p>An overwrite rejects a value that names no single instant on one side of that move. A session-local
147-
* time that a DST change skips: Java moves it past the gap, while the BE writes the rows at the instant of
148-
* the change. An instant whose local time a DST change repeats in the JVM zone: both instants of the overlap
149-
* format to the same value, which Paimon reads as the earlier one.
128+
* JVM's default zone. Doris binds such a column as TIMESTAMPTZ, so the cast value is the instant in UTC with
129+
* its offset; it is moved to the JVM zone, with a space between the date and the time as Paimon's parser
130+
* needs. An overwrite rejects an instant whose local time a DST change repeats there: both instants of the
131+
* overlap format to the same value, which Paimon reads as the earlier one.
150132
*/
151-
private static String toSdkLocalTime(String column, String value, boolean timestampTz, String sessionTimeZone,
152-
boolean overwrite) {
153-
String isoValue = value.replace(' ', 'T');
154-
ZonedDateTime instant;
155-
if (timestampTz) {
156-
instant = OffsetDateTime.parse(isoValue, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toZonedDateTime();
157-
} else {
158-
LocalDateTime sessionLocal = LocalDateTime.parse(isoValue, DateTimeFormatter.ISO_LOCAL_DATE_TIME);
159-
ZoneId sessionZone = ZoneId.of(sessionTimeZone, PaimonConnectorMetadata.SESSION_TIME_ZONE_ALIASES);
160-
if (overwrite && sessionZone.getRules().getValidOffsets(sessionLocal).isEmpty()) {
161-
throw new DorisConnectorException("Static LTZ partition value for column '" + column
162-
+ "' does not exist in session time zone " + sessionZone
163-
+ " and cannot be represented in a static overwrite");
164-
}
165-
instant = sessionLocal.atZone(sessionZone);
166-
}
133+
private static String toSdkLocalTime(String column, String value, boolean overwrite) {
167134
ZoneId sdkZone = ZoneId.systemDefault();
168-
LocalDateTime sdkLocal = instant.withZoneSameInstant(sdkZone).toLocalDateTime();
135+
LocalDateTime sdkLocal = OffsetDateTime.parse(value.replace(' ', 'T'), DateTimeFormatter.ISO_OFFSET_DATE_TIME)
136+
.atZoneSameInstant(sdkZone).toLocalDateTime();
169137
if (overwrite && sdkZone.getRules().getValidOffsets(sdkLocal).size() > 1) {
170138
throw new DorisConnectorException("Static LTZ partition value for column '" + column
171139
+ "' is ambiguous in FE JVM time zone " + sdkZone

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonWritePlanProvider.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -152,7 +152,7 @@ public ConnectorSinkPlan planWrite(ConnectorSession session, ConnectorWriteHandl
152152
}
153153
PaimonConnectorTransaction transaction = currentTransaction(session);
154154
PaimonWriteBinding binding = PaimonWriteBinding.create(
155-
tableHandle, table, buildHadoopConfig(), handle, session.getTimeZone());
155+
tableHandle, table, buildHadoopConfig(), handle);
156156
transaction.bind(binding);
157157

158158
TPaimonTableSink sink = new TPaimonTableSink();

‎fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonBackendBoundTableTest.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,7 @@ public void writeBindingKeepsCatalogOnFeButRemovesItFromBePayload(@TempDir Path
119119

120120
PaimonWriteBinding binding = PaimonWriteBinding.create(
121121
dataHandle(), table, Collections.emptyMap(),
122-
writeHandle(Collections.emptyMap(), Collections.emptySet()), "UTC");
122+
writeHandle(Collections.emptyMap(), Collections.emptySet()));
123123
FileStoreTable backendTable = deserializeTable(binding.getSerializedTable());
124124

125125
Assertions.assertNotNull(binding.getTable().catalogEnvironment().catalogLoader());
@@ -140,9 +140,9 @@ public void staticPartitionDistinguishesSqlNullFromNullString(@TempDir Path ware
140140
Map<String, String> requested = Collections.singletonMap("pt", "NULL");
141141

142142
PaimonWriteBinding literal = PaimonWriteBinding.create(dataHandle(), table,
143-
Collections.emptyMap(), writeHandle(requested, Collections.emptySet()), "UTC");
143+
Collections.emptyMap(), writeHandle(requested, Collections.emptySet()));
144144
PaimonWriteBinding sqlNull = PaimonWriteBinding.create(dataHandle(), table,
145-
Collections.emptyMap(), writeHandle(requested, Collections.singleton("pt")), "UTC");
145+
Collections.emptyMap(), writeHandle(requested, Collections.singleton("pt")));
146146

147147
Assertions.assertEquals("NULL", literal.getStaticPartition().get("pt"));
148148
Assertions.assertEquals(table.coreOptions().partitionDefaultName(),

‎fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonConnectorTransactionTest.java‎

Lines changed: 29 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,10 @@
1717

1818
package org.apache.doris.connector.paimon;
1919

20+
import org.apache.doris.connector.spi.ConnectorColumn;
2021
import org.apache.doris.connector.spi.DorisConnectorException;
22+
import org.apache.doris.connector.spi.handle.ConnectorTableHandle;
23+
import org.apache.doris.connector.spi.handle.ConnectorWriteHandle;
2124
import org.apache.doris.thrift.TPaimonCommitMessage;
2225

2326
import org.apache.paimon.CoreOptions;
@@ -204,10 +207,35 @@ private static PaimonConnectorTransaction boundTransaction(FileStoreTable target
204207
new PaimonConnectorTransaction(TRANSACTION_ID, new RecordingConnectorContext());
205208
transaction.bind(PaimonWriteBinding.create(
206209
new PaimonTableHandle("db", "tbl", Collections.emptyList(), Collections.emptyList()),
207-
target, Collections.emptyMap(), false, Collections.emptyMap(), Collections.emptySet()));
210+
target, Collections.emptyMap(), plainInsert()));
208211
return transaction;
209212
}
210213

214+
/** An INSERT without a PARTITION clause. */
215+
private static ConnectorWriteHandle plainInsert() {
216+
return new ConnectorWriteHandle() {
217+
@Override
218+
public ConnectorTableHandle getTableHandle() {
219+
return null;
220+
}
221+
222+
@Override
223+
public List<ConnectorColumn> getColumns() {
224+
return Collections.emptyList();
225+
}
226+
227+
@Override
228+
public boolean isOverwrite() {
229+
return false;
230+
}
231+
232+
@Override
233+
public Map<String, String> getStaticPartitionSpec() {
234+
return Collections.emptyMap();
235+
}
236+
};
237+
}
238+
211239
/** A DPCM-framed payload with one empty commit message, as the BE writer reports it. */
212240
private static byte[] commitPayload(int bucket) throws Exception {
213241
CommitMessage message = new CommitMessageImpl(BinaryRow.EMPTY_ROW, bucket, 2,

0 commit comments

Comments
 (0)