Skip to content

Commit 1d4999b

Browse files
morningmansuxiaogang223Gabriel39claude
committed
[fix](paimon) Keep static overwrite values on the partition the rows are written to
### What problem does this PR solve? Issue Number: None Related PR: #65868, #68532 Problem Summary: The static overwrite value of a Paimon partition could still name a partition other than the one the rows are written to. A string written into a shorter CHAR / VARCHAR column is not cast by BindSink: under the default non-strict session it is cut to the column length in code points. The static value was cast instead, which rejects a CHAR value that is too long (PARTITION (p = 'abcd') on CHAR(2) failed even for an INSERT, whose rows hold 'ab') and cuts UTF-16 units, splitting a surrogate pair that Paimon then encodes as '?'. The static value of a string column now follows the same rule as the row. A TIMESTAMP WITH LOCAL TIME ZONE value moves from the session zone to the FE JVM zone, where Paimon parses it. A session-local time skipped by a DST change is moved past the gap by Java, while the BE writes the rows at the instant of the change (cctz), and an instant whose local time the JVM zone repeats formats to a value that Paimon reads as the earlier instant. An empty overwrite then replaced an unrelated partition. An overwrite now rejects both, the second as branch-4.x #68532 does. ### Release note None ### Check List (For Author) - Test: Unit Test - PluginDrivenTableSinkBindingTest: CHAR, VARCHAR and STRING values are cut in code points like the rows, and kept as written when the session does not truncate. - PaimonWriteBindingTest: an overwrite rejects both instants of a JVM-zone overlap and a session time inside a DST gap; an INSERT keeps them; instants outside an overlap round-trip through Paimon's parser. - Behavior changed: Yes. A static overwrite of an LTZ partition value in a DST gap of the session zone, or in a DST overlap of the FE JVM zone, is rejected. - Does this need documentation: No Co-authored-by: Socrates <[email protected]> Co-authored-by: Gabriel <[email protected]> Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent fbe6a9d commit 1d4999b

6 files changed

Lines changed: 198 additions & 22 deletions

File tree

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonWriteBinding.java‎

Lines changed: 30 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -112,7 +112,8 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto
112112
() -> "missing the cast value of static partition column " + key);
113113
if (table.rowType().getField(canonicalName).type().getTypeRoot()
114114
== DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) {
115-
value = toSdkLocalTime(value, isTimestampTz(writeHandle, canonicalName), sessionTimeZone);
115+
value = toSdkLocalTime(canonicalName, value, isTimestampTz(writeHandle, canonicalName),
116+
sessionTimeZone, writeHandle.isOverwrite());
116117
}
117118
if (writeHandle.isOverwrite() && defaultPartitionName.equals(value)) {
118119
// Paimon's static overwrite reads this string as the NULL partition of any partition type, so
@@ -141,16 +142,36 @@ private static boolean isTimestampTz(ConnectorWriteHandle writeHandle, String co
141142
* JVM's default zone, so the value is moved there from the zone it was written in: the session zone for a
142143
* DATETIMEV2 value, UTC for a TIMESTAMPTZ value, which carries its offset. Paimon's parser takes a space,
143144
* not ISO's 'T', between the date and the time.
145+
*
146+
* <p>An overwrite rejects a value that names no single instant on one side of that move. A session-local
147+
* time that a DST change skips: Java moves it past the gap, while the BE writes the rows at the instant of
148+
* the change. An instant whose local time a DST change repeats in the JVM zone: both instants of the overlap
149+
* format to the same value, which Paimon reads as the earlier one.
144150
*/
145-
private static String toSdkLocalTime(String value, boolean timestampTz, String sessionTimeZone) {
151+
private static String toSdkLocalTime(String column, String value, boolean timestampTz, String sessionTimeZone,
152+
boolean overwrite) {
146153
String isoValue = value.replace(' ', 'T');
147-
ZonedDateTime instant = timestampTz
148-
? OffsetDateTime.parse(isoValue, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toZonedDateTime()
149-
: LocalDateTime.parse(isoValue, DateTimeFormatter.ISO_LOCAL_DATE_TIME)
150-
.atZone(ZoneId.of(sessionTimeZone, PaimonConnectorMetadata.SESSION_TIME_ZONE_ALIASES));
151-
return instant.withZoneSameInstant(ZoneId.systemDefault()).toLocalDateTime()
152-
.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME)
153-
.replace('T', ' ');
154+
ZonedDateTime instant;
155+
if (timestampTz) {
156+
instant = OffsetDateTime.parse(isoValue, DateTimeFormatter.ISO_OFFSET_DATE_TIME).toZonedDateTime();
157+
} else {
158+
LocalDateTime sessionLocal = LocalDateTime.parse(isoValue, DateTimeFormatter.ISO_LOCAL_DATE_TIME);
159+
ZoneId sessionZone = ZoneId.of(sessionTimeZone, PaimonConnectorMetadata.SESSION_TIME_ZONE_ALIASES);
160+
if (overwrite && sessionZone.getRules().getValidOffsets(sessionLocal).isEmpty()) {
161+
throw new DorisConnectorException("Static LTZ partition value for column '" + column
162+
+ "' does not exist in session time zone " + sessionZone
163+
+ " and cannot be represented in a static overwrite");
164+
}
165+
instant = sessionLocal.atZone(sessionZone);
166+
}
167+
ZoneId sdkZone = ZoneId.systemDefault();
168+
LocalDateTime sdkLocal = instant.withZoneSameInstant(sdkZone).toLocalDateTime();
169+
if (overwrite && sdkZone.getRules().getValidOffsets(sdkLocal).size() > 1) {
170+
throw new DorisConnectorException("Static LTZ partition value for column '" + column
171+
+ "' is ambiguous in FE JVM time zone " + sdkZone
172+
+ " and cannot be represented in a static overwrite");
173+
}
174+
return sdkLocal.format(DateTimeFormatter.ISO_LOCAL_DATE_TIME).replace('T', ' ');
154175
}
155176

156177
private static String serialize(FileStoreTable table) {

‎fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonWriteBindingTest.java‎

Lines changed: 73 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939

4040
import java.nio.file.Path;
4141
import java.time.Instant;
42+
import java.time.OffsetDateTime;
4243
import java.util.Arrays;
4344
import java.util.Collections;
4445
import java.util.List;
@@ -138,6 +139,71 @@ public void staticTimestampTzPartitionMovesTheInstantToTheSdkZone(@TempDir Path
138139
Assertions.assertEquals(Instant.parse("2024-01-15T00:30:45.123456Z"), parsedByPaimon(table, value));
139140
}
140141

142+
@Test
143+
public void staticLtzOverwriteRejectsAnInstantTheSdkZoneRepeats(@TempDir Path warehouse) {
144+
// 2023-11-05 08:30Z and 09:30Z are both 01:30 in Los Angeles, the JVM zone here. Either value would name
145+
// the same partition, which Paimon reads as the earlier instant. MUTATION: dropping the check -> red.
146+
FileStoreTable table = partitionedTable(warehouse, Collections.emptyMap(),
147+
new DataField(1, "part", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(6)));
148+
for (String instant : Arrays.asList("2023-11-05 08:30:00.123456+00:00",
149+
"2023-11-05 09:30:00.123456+00:00")) {
150+
DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
151+
() -> PaimonWriteBinding.resolveStaticPartition(table,
152+
write(true, "part", instant, instant, ConnectorType.of("TIMESTAMPTZ", 6, 0)), "UTC"));
153+
Assertions.assertTrue(error.getMessage().contains("ambiguous"), error.getMessage());
154+
Assertions.assertTrue(error.getMessage().contains("America/Los_Angeles"), error.getMessage());
155+
156+
// An INSERT does not overwrite anything, so it keeps the value.
157+
Assertions.assertEquals("2023-11-05 01:30:00.123456", PaimonWriteBinding.resolveStaticPartition(table,
158+
write(false, "part", instant, instant, ConnectorType.of("TIMESTAMPTZ", 6, 0)), "UTC")
159+
.get("part"));
160+
}
161+
162+
// A session-local DATETIMEV2 value lands in the same overlap: 09:30:45 UTC is the second 01:30:45.
163+
DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
164+
() -> PaimonWriteBinding.resolveStaticPartition(table,
165+
write(true, "part", "2024-11-03 09:30:45", "2024-11-03 09:30:45",
166+
ConnectorType.of("DATETIMEV2", 0, 0)), "UTC"));
167+
Assertions.assertTrue(error.getMessage().contains("ambiguous"), error.getMessage());
168+
}
169+
170+
@Test
171+
public void staticLtzPartitionOutsideAnOverlapRoundTripsThroughPaimon(@TempDir Path warehouse) {
172+
// Both instants that overlap in Los Angeles are distinct in UTC, and an instant after the overlap is
173+
// distinct in Los Angeles. MUTATION: rejecting every instant near a DST change -> red.
174+
FileStoreTable table = partitionedTable(warehouse, Collections.emptyMap(),
175+
new DataField(1, "part", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(6)));
176+
TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
177+
for (String instant : Arrays.asList("2023-11-05 08:30:00.123456+00:00",
178+
"2023-11-05 09:30:00.123456+00:00")) {
179+
assertStaticTimestampTzRoundTrip(table, instant);
180+
}
181+
TimeZone.setDefault(TimeZone.getTimeZone("America/Los_Angeles"));
182+
assertStaticTimestampTzRoundTrip(table, "2023-11-05 10:30:00.123456+00:00");
183+
}
184+
185+
@Test
186+
public void staticLtzOverwriteRejectsASessionTimeADstChangeSkips(@TempDir Path warehouse) {
187+
// 02:30:45 on 2024-03-10 does not exist in Los Angeles. Java moves it to 03:30:45, while the BE writes the
188+
// rows at the instant of the change (cctz), so no overwrite value names the rows' partition.
189+
// MUTATION: dropping the check -> red.
190+
TimeZone.setDefault(TimeZone.getTimeZone("UTC"));
191+
FileStoreTable table = partitionedTable(warehouse, Collections.emptyMap(),
192+
new DataField(1, "part", DataTypes.TIMESTAMP_WITH_LOCAL_TIME_ZONE(6)));
193+
194+
DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
195+
() -> PaimonWriteBinding.resolveStaticPartition(table,
196+
write(true, "part", "2024-03-10 02:30:45", "2024-03-10 02:30:45",
197+
ConnectorType.of("DATETIMEV2", 0, 0)), "America/Los_Angeles"));
198+
Assertions.assertTrue(error.getMessage().contains("does not exist"), error.getMessage());
199+
Assertions.assertTrue(error.getMessage().contains("America/Los_Angeles"), error.getMessage());
200+
201+
// An INSERT does not overwrite anything, so it keeps the value.
202+
Assertions.assertNotNull(PaimonWriteBinding.resolveStaticPartition(table,
203+
write(false, "part", "2024-03-10 02:30:45", "2024-03-10 02:30:45",
204+
ConnectorType.of("DATETIMEV2", 0, 0)), "America/Los_Angeles").get("part"));
205+
}
206+
141207
@Test
142208
public void staticPartitionNullUsesTheDefaultPartitionName(@TempDir Path warehouse) {
143209
FileStoreTable table = partitionedTable(warehouse,
@@ -151,6 +217,13 @@ public void staticPartitionNullUsesTheDefaultPartitionName(@TempDir Path warehou
151217
PaimonWriteBinding.resolveStaticPartition(table, write, "UTC").get("part"));
152218
}
153219

220+
private static void assertStaticTimestampTzRoundTrip(FileStoreTable table, String instant) {
221+
String value = PaimonWriteBinding.resolveStaticPartition(table,
222+
write(true, "part", instant, instant, ConnectorType.of("TIMESTAMPTZ", 6, 0)), "UTC").get("part");
223+
Assertions.assertEquals(OffsetDateTime.parse(instant.replace(' ', 'T')).toInstant(),
224+
parsedByPaimon(table, value), instant);
225+
}
226+
154227
private static Instant parsedByPaimon(FileStoreTable table, String value) {
155228
return ((Timestamp) TypeUtils.castFromString(value, table.rowType().getTypeAt(1))).toInstant();
156229
}

‎fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/handle/ConnectorWriteHandle.java‎

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -77,9 +77,10 @@ default Set<String> getStaticPartitionNullKeys() {
7777
}
7878

7979
/**
80-
* The static partition values cast to the types of their columns, the cast a written row gets, in the
81-
* string form of the cast value and keyed like {@link #getStaticPartitionSpec}, which keeps each literal as
82-
* written: {@code PARTITION (p = true)} on an INT column is {@code "true"} there and {@code "1"} here. The
80+
* The static partition values cast to the types of their columns, as the written rows carry them (a string
81+
* longer than its CHAR / VARCHAR column is cut as the INSERT cuts it), in the string form of the cast value
82+
* and keyed like {@link #getStaticPartitionSpec}, which keeps each literal as written:
83+
* {@code PARTITION (p = true)} on an INT column is {@code "true"} there and {@code "1"} here. The
8384
* {@link #getStaticPartitionNullKeys null keys} are absent, and a key that names no column (an Iceberg
8485
* partition field) keeps its literal. A value that cannot be cast to its column type fails the call. The
8586
* default, for a handle that does not cast, is the spec as written without its null keys.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/rules/analysis/BindSink.java‎

Lines changed: 11 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -310,10 +310,7 @@ private LogicalProject<?> getOutputProjectByCoercion(List<Column> tableSchema, L
310310

311311
// add cast project
312312
List<NamedExpression> castExprs = Lists.newArrayList();
313-
ConnectContext connCtx = ConnectContext.get();
314-
final boolean truncateString = needTruncateStringWhenInsert
315-
&& (connCtx == null || connCtx.getSessionVariable().enableInsertValueAutoCast)
316-
&& !SessionVariable.enableStrictCast();
313+
final boolean truncateString = needTruncateStringWhenInsert && truncatesStringOnInsert();
317314
for (int i = 0; i < tableSchema.size(); ++i) {
318315
Column col = tableSchema.get(i);
319316
NamedExpression expr = columnToOutput.get(col.getName()); // relative outputExpr
@@ -359,6 +356,16 @@ private LogicalProject<?> getOutputProjectByCoercion(List<Column> tableSchema, L
359356
return fullOutputProject;
360357
}
361358

359+
/**
360+
* Whether the session lets an INSERT write a string into a shorter CHAR / VARCHAR column cut to the column
361+
* length, counted in code points (the {@link Substring} above), instead of as is.
362+
*/
363+
public static boolean truncatesStringOnInsert() {
364+
ConnectContext connCtx = ConnectContext.get();
365+
return (connCtx == null || connCtx.getSessionVariable().enableInsertValueAutoCast)
366+
&& !SessionVariable.enableStrictCast();
367+
}
368+
362369
@VisibleForTesting
363370
static Expression coerceSinkExpression(Expression expression, DataType targetType) {
364371
return TypeCoercionUtils.castIfNotSameType(expression, targetType);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertCommandContext.java‎

Lines changed: 27 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,10 +18,15 @@
1818
package org.apache.doris.nereids.trees.plans.commands.insert;
1919

2020
import org.apache.doris.catalog.Column;
21+
import org.apache.doris.nereids.rules.analysis.BindSink;
2122
import org.apache.doris.nereids.trees.expressions.Expression;
23+
import org.apache.doris.nereids.trees.expressions.functions.executable.StringArithmetic;
24+
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
2225
import org.apache.doris.nereids.trees.expressions.literal.Literal;
2326
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
27+
import org.apache.doris.nereids.trees.expressions.literal.StringLikeLiteral;
2428
import org.apache.doris.nereids.types.DataType;
29+
import org.apache.doris.nereids.types.coercion.CharacterType;
2530

2631
import com.google.common.base.Preconditions;
2732

@@ -90,9 +95,9 @@ public void setBoundTargetSchema(List<Column> boundTargetSchema) {
9095

9196
/**
9297
* Casts the static partition values to the types of their columns in the bound target schema; see
93-
* {@code ConnectorWriteHandle#getCastStaticPartitionSpec}. The cast is the one constant folding applies to
94-
* the value BindSink materializes into each row, but a value that does not fit its type fails here instead
95-
* of becoming NULL.
98+
* {@code ConnectorWriteHandle#getCastStaticPartitionSpec}. Each value is the one BindSink materializes into
99+
* the rows, after constant folding, but a value that does not fit its type fails here instead of becoming
100+
* NULL.
96101
*/
97102
public Map<String, String> castStaticPartitionSpec() {
98103
Preconditions.checkState(staticPartitionLiterals.isEmpty() || !boundTargetSchema.isEmpty(),
@@ -107,15 +112,31 @@ public Map<String, String> castStaticPartitionSpec() {
107112
.filter(candidate -> candidate.getName().equalsIgnoreCase(entry.getKey()))
108113
.findFirst();
109114
if (column.isPresent()) {
110-
Expression cast = value.checkedCastTo(DataType.fromCatalogType(column.get().getType()));
111-
Preconditions.checkState(cast instanceof Literal, "cast of literal %s is not a literal", value);
112-
value = (Literal) cast;
115+
value = writtenValue(value, DataType.fromCatalogType(column.get().getType()));
113116
}
114117
spec.put(entry.getKey(), value.getStringValue());
115118
}
116119
return spec;
117120
}
118121

122+
private static Literal writtenValue(Literal value, DataType columnType) {
123+
if (!value.getDataType().isStringLikeType() || !columnType.isStringLikeType()) {
124+
Expression cast = value.checkedCastTo(columnType);
125+
Preconditions.checkState(cast instanceof Literal, "cast of literal %s is not a literal", value);
126+
return (Literal) cast;
127+
}
128+
// BindSink does not cast a string written into a string column. It keeps the value, cut to the length of
129+
// a CHAR / VARCHAR column in code points when the session truncates strings on insert; a cast would cut
130+
// UTF-16 units instead, or reject a CHAR value that is too long.
131+
int length = ((CharacterType) columnType).getLen();
132+
if (length < 0 || !BindSink.truncatesStringOnInsert()) {
133+
return value;
134+
}
135+
Preconditions.checkState(value instanceof StringLikeLiteral, "string literal %s has no string value", value);
136+
return (Literal) StringArithmetic.substringVarcharIntInt((StringLikeLiteral) value,
137+
new IntegerLiteral(1), new IntegerLiteral(length));
138+
}
139+
119140
public Optional<String> getBranchName() {
120141
return branchName;
121142
}

0 commit comments

Comments
 (0)