Skip to content

Commit cc5818e

Browse files
morningmanclaude
andcommitted
[fix](paimon) Reject a static overwrite of a binary Paimon partition
### What problem does this PR solve? Issue Number: None Related PR: #65868 Problem Summary: Paimon's static overwrite turns each value of the partition spec into the partition key with TypeUtils.castFromString, which for BINARY / VARBINARY takes the UTF-8 bytes of the string. Doris cannot name a binary partition that way: the cast values spelled a binary value as String.valueOf(byte[]) ([B@...), and the spec as written spells it 0x4142, which Paimon reads as the six bytes "0x4142" rather than the two bytes 41 42 the rows carry. An overwrite of PARTITION (p = X'4142') whose query returns no rows therefore committed against another partition and left the requested one in place while reporting success. Reject a static overwrite that gives a binary partition column a value other than NULL, before the values are cast. SQL NULL still names the partition by partition.default-name. The cast values now spell a binary value as the spec as written does (0x and its bytes in hex), so they never carry an array identity. ### Release note None ### Check List (For Author) - Test: Unit Test - PaimonWriteBindingTest.staticOverwriteRejectsABinaryPartitionValue: BYTES, VARBINARY(2) and BINARY(2) partitions reject 0x4142 without asking for the cast values; it also pins Paimon reading "0x4142" as its six UTF-8 bytes. staticPartitionNullUsesTheDefaultPartitionName now covers a binary column too. - PluginDrivenTableSinkBindingTest.staticPartitionBinaryValueKeepsItsBytesWhenCast: the cast value of X'4142' on a VARBINARY(2147483647) column is 0x4142. - Behavior changed: Yes. INSERT OVERWRITE ... PARTITION (p = <binary>) on a Paimon table now fails with "Static overwrite of binary partition column 'p' is not supported" instead of overwriting another partition. - Does this need documentation: No Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent 454af09 commit cc5818e

5 files changed

Lines changed: 77 additions & 17 deletions

File tree

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonWriteBinding.java‎

Lines changed: 19 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -89,22 +89,37 @@ static FileStoreTable configureTableForWrite(FileStoreTable table, boolean overw
8989
/**
9090
* The static partition of an overwrite, as Paimon's static overwrite parses it: SQL NULL becomes the table's
9191
* {@code partition.default-name}, and every other value is the one the written rows carry, cast to the
92-
* column type.
92+
* column type. A binary column can only be statically overwritten with NULL.
9393
*/
9494
static Map<String, String> resolveStaticPartition(FileStoreTable table, ConnectorWriteHandle writeHandle) {
9595
Map<String, String> canonicalNames = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
9696
for (String partitionKey : table.partitionKeys()) {
9797
canonicalNames.put(partitionKey, partitionKey);
9898
}
99-
String defaultPartitionName = CoreOptions.fromMap(table.options()).partitionDefaultName();
100-
Map<String, String> castValues = writeHandle.getCastStaticPartitionSpec();
101-
Map<String, String> result = new LinkedHashMap<>();
99+
Map<String, String> requestedColumns = new LinkedHashMap<>();
102100
for (String key : writeHandle.getStaticPartitionSpec().keySet()) {
103101
String canonicalName = canonicalNames.get(key);
104102
if (canonicalName == null) {
105103
throw new DorisConnectorException("Column '" + key
106104
+ "' is not a partition column of Paimon table");
107105
}
106+
DataTypeRoot typeRoot = table.rowType().getField(canonicalName).type().getTypeRoot();
107+
if ((typeRoot == DataTypeRoot.BINARY || typeRoot == DataTypeRoot.VARBINARY)
108+
&& !writeHandle.getStaticPartitionNullKeys().contains(key)) {
109+
// Paimon's static overwrite takes the UTF-8 bytes of a value's string, which cannot name every
110+
// binary partition, and a wrong name overwrites another partition without an error. Checked
111+
// before the values are cast.
112+
throw new DorisConnectorException("Static overwrite of binary partition column '"
113+
+ canonicalName + "' is not supported");
114+
}
115+
requestedColumns.put(key, canonicalName);
116+
}
117+
String defaultPartitionName = CoreOptions.fromMap(table.options()).partitionDefaultName();
118+
Map<String, String> castValues = writeHandle.getCastStaticPartitionSpec();
119+
Map<String, String> result = new LinkedHashMap<>();
120+
for (Map.Entry<String, String> requested : requestedColumns.entrySet()) {
121+
String key = requested.getKey();
122+
String canonicalName = requested.getValue();
108123
if (writeHandle.getStaticPartitionNullKeys().contains(key)) {
109124
result.put(canonicalName, defaultPartitionName);
110125
continue;

‎fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonWriteBindingTest.java‎

Lines changed: 35 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.paimon.table.FileStoreTable;
2929
import org.apache.paimon.table.FileStoreTableFactory;
3030
import org.apache.paimon.types.DataField;
31+
import org.apache.paimon.types.DataType;
3132
import org.apache.paimon.types.DataTypes;
3233
import org.apache.paimon.utils.TypeUtils;
3334
import org.junit.jupiter.api.AfterEach;
@@ -36,6 +37,7 @@
3637
import org.junit.jupiter.api.Test;
3738
import org.junit.jupiter.api.io.TempDir;
3839

40+
import java.nio.charset.StandardCharsets;
3941
import java.nio.file.Path;
4042
import java.time.Instant;
4143
import java.time.OffsetDateTime;
@@ -148,15 +150,40 @@ public void staticLtzPartitionOutsideAnOverlapRoundTripsThroughPaimon(@TempDir P
148150
}
149151

150152
@Test
151-
public void staticPartitionNullUsesTheDefaultPartitionName(@TempDir Path warehouse) {
152-
FileStoreTable table = partitionedTable(warehouse,
153-
Collections.singletonMap("partition.default-name", DEFAULT_NAME),
154-
new DataField(1, "part", DataTypes.INT()));
155-
ConnectorWriteHandle write = handle(true, Collections.singletonMap("part", "NULL"),
156-
Collections::emptyMap, Collections.singleton("part"));
153+
public void staticOverwriteRejectsABinaryPartitionValue(@TempDir Path warehouse) {
154+
// Paimon's static overwrite takes the UTF-8 bytes of a value's string: "0x4142" names the partition of
155+
// those six bytes, not of the two bytes 41 42 the rows carry, so an overwrite without rows would report
156+
// success and leave the partition it was given in place. The check comes before the values are cast,
157+
// which a binary value may not survive. MUTATION: dropping the check -> red.
158+
Assertions.assertArrayEquals("0x4142".getBytes(StandardCharsets.UTF_8),
159+
(byte[]) TypeUtils.castFromString("0x4142", DataTypes.BYTES()));
160+
for (DataType type : Arrays.asList(DataTypes.BYTES(), DataTypes.VARBINARY(2), DataTypes.BINARY(2))) {
161+
FileStoreTable table = partitionedTable(warehouse, Collections.emptyMap(),
162+
new DataField(1, "part", type));
163+
ConnectorWriteHandle overwrite = handle(true, Collections.singletonMap("part", "0x4142"), () -> {
164+
throw new AssertionError("a binary static partition value must be rejected before it is cast");
165+
}, Collections.emptySet());
157166

158-
Assertions.assertEquals(DEFAULT_NAME,
159-
PaimonWriteBinding.resolveStaticPartition(table, write).get("part"));
167+
DorisConnectorException error = Assertions.assertThrows(DorisConnectorException.class,
168+
() -> PaimonWriteBinding.resolveStaticPartition(table, overwrite));
169+
Assertions.assertTrue(error.getMessage().contains("binary partition column 'part'"),
170+
error.getMessage());
171+
}
172+
}
173+
174+
@Test
175+
public void staticPartitionNullUsesTheDefaultPartitionName(@TempDir Path warehouse) {
176+
// A binary column too: SQL NULL names its partition by partition.default-name, not by bytes.
177+
for (DataType type : Arrays.asList(DataTypes.INT(), DataTypes.BYTES())) {
178+
FileStoreTable table = partitionedTable(warehouse,
179+
Collections.singletonMap("partition.default-name", DEFAULT_NAME),
180+
new DataField(1, "part", type));
181+
ConnectorWriteHandle write = handle(true, Collections.singletonMap("part", "NULL"),
182+
Collections::emptyMap, Collections.singleton("part"));
183+
184+
Assertions.assertEquals(DEFAULT_NAME,
185+
PaimonWriteBinding.resolveStaticPartition(table, write).get("part"), type.toString());
186+
}
160187
}
161188

162189
@Test

‎fe/fe-connector/fe-connector-spi/src/main/java/org/apache/doris/connector/spi/handle/ConnectorWriteHandle.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,8 @@ default Set<String> getStaticPartitionNullKeys() {
8080
* The static partition values cast to the types of their columns, as the written rows carry them (a string
8181
* longer than its CHAR / VARCHAR column is cut as the INSERT cuts it), in the string form of the cast value
8282
* and keyed like {@link #getStaticPartitionSpec}, which keeps each literal as written:
83-
* {@code PARTITION (p = true)} on an INT column is {@code "true"} there and {@code "1"} here. The
83+
* {@code PARTITION (p = true)} on an INT column is {@code "true"} there and {@code "1"} here. A binary value
84+
* is {@code 0x} followed by its bytes in hex, as in {@link #getStaticPartitionSpec}. The
8485
* {@link #getStaticPartitionNullKeys null keys} are absent, and a key that names no column (an Iceberg
8586
* partition field) keeps its literal. A value that cannot be cast to its column type fails the call. The
8687
* default, for a handle that does not cast, is the spec as written without its null keys.

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/PluginDrivenInsertCommandContext.java‎

Lines changed: 7 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -91,9 +91,7 @@ public void setStaticPartitionSpecFromExpressions(Map<String, Expression> partit
9191
// Commit metadata must use the same semantic bytes as the materialized partition row.
9292
literal = (Literal) ConnectorWriteValueConverter.convert(column, literal);
9393
}
94-
// Binary partition keys must bypass character decoding, including empty and non-UTF-8 bytes.
95-
spec.put(entry.getKey(), literal instanceof VarBinaryLiteral
96-
? "0x" + literal.toString() : literal.getStringValue());
94+
spec.put(entry.getKey(), specValue(literal));
9795
literals.put(entry.getKey(), literal);
9896
if (entry.getValue() instanceof NullLiteral) {
9997
nullKeys.add(entry.getKey());
@@ -130,11 +128,16 @@ public Map<String, String> castStaticPartitionSpec() {
130128
if (column.isPresent()) {
131129
value = writtenValue(value, DataType.fromCatalogType(column.get().getType()));
132130
}
133-
spec.put(entry.getKey(), value.getStringValue());
131+
spec.put(entry.getKey(), specValue(value));
134132
}
135133
return spec;
136134
}
137135

136+
private static String specValue(Literal literal) {
137+
// Binary partition keys must bypass character decoding, including empty and non-UTF-8 bytes.
138+
return literal instanceof VarBinaryLiteral ? "0x" + literal.toString() : literal.getStringValue();
139+
}
140+
138141
private static Literal writtenValue(Literal value, DataType columnType) {
139142
if (!value.getDataType().isStringLikeType() || !columnType.isStringLikeType()) {
140143
Expression cast = value.checkedCastTo(columnType);

‎fe/fe-core/src/test/java/org/apache/doris/planner/PluginDrivenTableSinkBindingTest.java‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import org.apache.doris.nereids.trees.expressions.literal.IntegerLiteral;
4040
import org.apache.doris.nereids.trees.expressions.literal.NullLiteral;
4141
import org.apache.doris.nereids.trees.expressions.literal.StringLiteral;
42+
import org.apache.doris.nereids.trees.expressions.literal.VarBinaryLiteral;
4243
import org.apache.doris.nereids.trees.expressions.literal.VarcharLiteral;
4344
import org.apache.doris.nereids.trees.plans.commands.insert.PluginDrivenInsertCommandContext;
4445
import org.apache.doris.qe.ConnectContext;
@@ -199,6 +200,19 @@ public void staticPartitionStringValueIsCutLikeTheWrittenRows() throws AnalysisE
199200
}
200201
}
201202

203+
@Test
204+
public void staticPartitionBinaryValueKeepsItsBytesWhenCast() throws AnalysisException {
205+
// A binary value is 0x and its bytes in hex in both maps. Its string value would be the identity of the
206+
// byte array ([B@...), which names no partition. MUTATION: getStringValue() for a cast binary value -> red.
207+
Map<String, Expression> partition = Collections.singletonMap("b",
208+
new VarBinaryLiteral(new byte[] {0x41, 0x42}));
209+
// The type Paimon's VARBINARY maps to.
210+
List<Column> schema = Collections.singletonList(
211+
new Column("b", ScalarType.createVarbinaryType(ScalarType.MAX_VARBINARY_LENGTH)));
212+
213+
Assertions.assertEquals(Collections.singletonMap("b", "0x4142"), castStaticPartitionSpec(partition, schema));
214+
}
215+
202216
@Test
203217
public void staticPartitionValueThatDoesNotFitItsTypeFailsOnlyWhenCast() throws AnalysisException {
204218
// Only a connector that reads the cast values may fail on them; the others keep reading the literal.

0 commit comments

Comments
 (0)