Skip to content

Commit 45e6e5d

Browse files
morningmanclaude
andcommitted
[fix](paimon) Reject padding a short CHAR key that MERGE inserts
MERGE matches its source rows to a Paimon table, and checks the keys it inserts for duplicates, by the key as Doris holds it: Doris compares CHAR values unpadded, so 'k' and 'k ' are two keys and neither matches a stored 'k '. The writer padded the keys of the rows MERGE inserts afterwards. On a local cluster, a NOT MATCHED MERGE of 'k' and 'k ' into an empty CHAR(3)-keyed table committed one row, and a NOT MATCHED-only MERGE of 'k' replaced the stored row 'k ' that it had not matched. The writer now fails such a row when a CHAR primary-key value is shorter than the column, the only case in which padding changes a key, and the message says to pad the key in the MERGE with rpad or to turn padding off. A full-length key is written as it is. The static partition value of the table's rowkind.field is no longer padded either, as the writer already leaves the rows' value alone: with a CHAR rowkind.field partition column, a nonempty static overwrite of '+I' failed with "does not belong to this partition", and an empty one returned OK without clearing the partition. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent b584341 commit 45e6e5d

7 files changed

Lines changed: 239 additions & 7 deletions

File tree

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,8 @@ public void open(String serializedTable, Map<String, String> hadoopConfig,
184184

185185
CoreOptions coreOptions = CoreOptions.fromMap(table.options());
186186
this.writeSchema = PaimonWriteSchema.create(table.rowType(), columnNames, changelogWrite,
187-
PaimonWriteSchema.charPadding(table.options()), coreOptions.rowkindField().orElse(null));
187+
PaimonWriteSchema.charPadding(table.options()), coreOptions.rowkindField().orElse(null),
188+
table.primaryKeys());
188189
this.arrowAdapter = new PaimonArrowBatchAdapter(
189190
writeSchema.inputType(), ZoneId.of(timeZone), allocator);
190191
validateWriteColumnsForMergeEngine(

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonWriteSchema.java‎

Lines changed: 40 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.paimon.types.CharType;
2828
import org.apache.paimon.types.DataField;
2929
import org.apache.paimon.types.DataType;
30+
import org.apache.paimon.types.DataTypeRoot;
3031
import org.apache.paimon.types.MapType;
3132
import org.apache.paimon.types.RowKind;
3233
import org.apache.paimon.types.RowType;
@@ -36,6 +37,7 @@
3637

3738
import java.util.ArrayList;
3839
import java.util.Arrays;
40+
import java.util.Collections;
3941
import java.util.List;
4042
import java.util.Map;
4143

@@ -64,6 +66,12 @@ final class PaimonWriteSchema {
6466
* table's rowkind.field.
6567
*/
6668
private final CharPadder[] charPadders;
69+
/**
70+
* Per input column, the length of a padded CHAR primary-key column of a changelog write, or 0. Padding a
71+
* shorter key that MERGE inserts would change the key after MERGE matched it and checked it for duplicates;
72+
* see {@link #tableRow}.
73+
*/
74+
private final int[] insertedKeyCharLengths;
6775
private final RowType inputType;
6876
/** Maps Doris input-column position → Paimon table-schema position. */
6977
private final int[] tableFieldIndexes;
@@ -73,7 +81,8 @@ final class PaimonWriteSchema {
7381
private final int tableFieldCount;
7482

7583
private PaimonWriteSchema(int[] tableFieldIndexes, int[] omittedDefaultFieldIndexes,
76-
Object[] omittedDefaultValues, int tableFieldCount, RowType inputType, CharPadder[] charPadders) {
84+
Object[] omittedDefaultValues, int tableFieldCount, RowType inputType, CharPadder[] charPadders,
85+
int[] insertedKeyCharLengths) {
7786
this.fieldGetters = new InternalRow.FieldGetter[inputType.getFieldCount()];
7887
for (int i = 0; i < fieldGetters.length; i++) {
7988
// Getters describe the transport row, not the target table. Transport fields are
@@ -82,6 +91,7 @@ private PaimonWriteSchema(int[] tableFieldIndexes, int[] omittedDefaultFieldInde
8291
this.fieldGetters[i] = InternalRow.createFieldGetter(inputType.getTypeAt(i), i);
8392
}
8493
this.charPadders = charPadders;
94+
this.insertedKeyCharLengths = insertedKeyCharLengths;
8595
this.inputType = inputType;
8696
this.tableFieldIndexes = tableFieldIndexes;
8797
this.omittedDefaultFieldIndexes = omittedDefaultFieldIndexes;
@@ -112,21 +122,29 @@ static PaimonWriteSchema create(
112122
return create(tableType, columnNames, changelogWrite, charPadding, null);
113123
}
114124

125+
static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean changelogWrite,
126+
boolean charPadding, String rowKindField) {
127+
return create(tableType, columnNames, changelogWrite, charPadding, rowKindField, Collections.emptyList());
128+
}
129+
115130
/**
116131
* @param charPadding whether to pad the CHAR values of inserted rows as Paimon's cast to CHAR does; see
117132
* {@link #charPadder}
118133
* @param rowKindField the table's {@code rowkind.field}, or null. Its value is not padded: Paimon takes the row
119134
* kind from it as it is ({@code RowKind.fromShortString}), and a padded '+I' is no row kind.
135+
* @param primaryKeys the table's primary keys. In a changelog write, a CHAR key that padding would change
136+
* fails the row; see {@link #tableRow}.
120137
*/
121138
static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean changelogWrite,
122-
boolean charPadding, String rowKindField) {
139+
boolean charPadding, String rowKindField, List<String> primaryKeys) {
123140
if (columnNames == null || columnNames.length == 0) {
124141
throw new IllegalArgumentException(
125142
"PaimonJniWriter requires explicit column names");
126143
}
127144

128145
int[] tableFieldIndexes = new int[columnNames.length];
129146
CharPadder[] charPadders = new CharPadder[columnNames.length];
147+
int[] insertedKeyCharLengths = new int[columnNames.length];
130148
List<DataField> inputFields = new ArrayList<>(columnNames.length);
131149
boolean[] specifiedFields = new boolean[tableType.getFieldCount()];
132150
for (int i = 0; i < columnNames.length; i++) {
@@ -159,6 +177,10 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
159177
inputFields.add(field.newType(field.type().nullable()));
160178
if (charPadding && !field.name().equals(rowKindField)) {
161179
charPadders[i] = charPadder(field.type());
180+
if (changelogWrite && field.type().getTypeRoot() == DataTypeRoot.CHAR
181+
&& primaryKeys.contains(field.name())) {
182+
insertedKeyCharLengths[i] = ((CharType) field.type()).getLength();
183+
}
162184
}
163185
}
164186

@@ -182,7 +204,8 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
182204
Arrays.copyOf(omittedDefaultValues, omittedDefaultCount),
183205
tableType.getFieldCount(),
184206
new RowType(inputFields),
185-
charPadders);
207+
charPadders,
208+
insertedKeyCharLengths);
186209
}
187210

188211
/**
@@ -218,6 +241,20 @@ GenericRow tableRow(InternalRow columnValues) {
218241
// DELETE row carries the key Doris read back from the table, which has to stay byte-equal to the
219242
// stored key for Paimon to match it, padded or not.
220243
if (charPadders[i] != null && value != null && row.getRowKind() == RowKind.INSERT) {
244+
// In a changelog write only MERGE inserts rows, and MERGE has matched them to the table and
245+
// checked them for duplicate keys by the key as it is: Doris compares CHAR values unpadded.
246+
// Padding a shorter key here would make 'k' and 'k ' one key, or let a NOT MATCHED row replace
247+
// the stored 'k ' it did not match. A key as long as the column is not changed by padding.
248+
if (insertedKeyCharLengths[i] > 0 && ((BinaryString) value).numChars() < insertedKeyCharLengths[i]) {
249+
throw new IllegalArgumentException(String.format(
250+
"MERGE cannot insert '%s' into the CHAR(%d) primary key column '%s' of a Paimon table: "
251+
+ "the value is shorter than the column, and padding it to the column "
252+
+ "length would change the key that MERGE matched. Pad the key in the "
253+
+ "MERGE, with rpad(<key>, %d, ' ') in both the ON condition and the "
254+
+ "inserted values, or set enable_paimon_write_char_padding = false",
255+
value, insertedKeyCharLengths[i], inputType.getFields().get(i).name(),
256+
insertedKeyCharLengths[i]));
257+
}
221258
value = charPadders[i].pad(value);
222259
}
223260
// Actual Doris input is applied last so an explicit NULL remains distinct

‎fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonWriteSchemaTest.java‎

Lines changed: 66 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -355,6 +355,66 @@ public void testCharPaddingLeavesTheRowKindFieldAlone() {
355355
Assertions.assertEquals("a ", tableRow.getString(2).toString());
356356
}
357357

358+
@Test
359+
public void testMergeInsertOfAShortCharKeyFails() {
360+
// MERGE matches its source rows to the table and checks the keys it inserts for duplicates by the key as
361+
// Doris holds it, unpadded: 'k' and 'k ' are two keys there, and neither matches a stored 'k '. Padding
362+
// a shorter key afterwards would make them one key, or let a NOT MATCHED row replace the stored row it
363+
// did not match, so such a row fails. Code points are counted, as padding counts them.
364+
// MUTATION: no check, or counting bytes -> red; checking UPDATE / DELETE rows too -> red below.
365+
PaimonWriteSchema schema = PaimonWriteSchema.create(charKeyTableType(),
366+
new String[] {PaimonWriteSchema.ROW_KIND_COLUMN, "code", "label"}, true, true, null,
367+
Collections.singletonList("code"));
368+
369+
for (String shortKey : Arrays.asList("k", "k ", "中中")) {
370+
IllegalArgumentException e = Assertions.assertThrows(IllegalArgumentException.class,
371+
() -> tableRow(schema, PaimonWriteSchema.INSERT_OPERATION, string(shortKey), string("x")));
372+
Assertions.assertTrue(e.getMessage().contains("CHAR(3) primary key column 'code'"), e.getMessage());
373+
Assertions.assertTrue(e.getMessage().contains("rpad(<key>, 3, ' ')"), e.getMessage());
374+
Assertions.assertTrue(e.getMessage().contains("enable_paimon_write_char_padding"), e.getMessage());
375+
}
376+
}
377+
378+
@Test
379+
public void testMergeInsertPadsTheRestOfAFullLengthKeyRow() {
380+
// Padding does not change a key as long as the column, or a longer one Doris kept, so that row is written
381+
// with the key as MERGE saw it, and its CHAR value columns are padded as in any inserted row.
382+
PaimonWriteSchema schema = PaimonWriteSchema.create(charKeyTableType(),
383+
new String[] {PaimonWriteSchema.ROW_KIND_COLUMN, "code", "label"}, true, true, null,
384+
Collections.singletonList("code"));
385+
386+
InternalRow tableRow = tableRow(schema, PaimonWriteSchema.INSERT_OPERATION, string("abc"), string("x"));
387+
Assertions.assertEquals("abc", tableRow.getString(0).toString());
388+
Assertions.assertEquals("x ", tableRow.getString(1).toString());
389+
Assertions.assertEquals("中中中", tableRow(schema, PaimonWriteSchema.INSERT_OPERATION,
390+
string("中中中"), null).getString(0).toString());
391+
Assertions.assertEquals("abcd", tableRow(schema, PaimonWriteSchema.INSERT_OPERATION,
392+
string("abcd"), null).getString(0).toString());
393+
}
394+
395+
@Test
396+
public void testShortCharKeysThatMergeDoesNotInsertAreWrittenAsBefore() {
397+
// UPDATE and DELETE rows carry the stored key and are never padded; with padding off nothing is padded;
398+
// and an INSERT statement is no changelog write, so its keys are padded and nothing fails.
399+
PaimonWriteSchema merge = PaimonWriteSchema.create(charKeyTableType(),
400+
new String[] {PaimonWriteSchema.ROW_KIND_COLUMN, "code", "label"}, true, true, null,
401+
Collections.singletonList("code"));
402+
Assertions.assertEquals("k", tableRow(merge, PaimonWriteSchema.UPDATE_OPERATION, string("k"), string("x"))
403+
.getString(0).toString());
404+
Assertions.assertEquals("k", tableRow(merge, PaimonWriteSchema.DELETE_OPERATION, string("k"), string("x"))
405+
.getString(0).toString());
406+
407+
PaimonWriteSchema unpadded = PaimonWriteSchema.create(charKeyTableType(),
408+
new String[] {PaimonWriteSchema.ROW_KIND_COLUMN, "code", "label"}, true, false, null,
409+
Collections.singletonList("code"));
410+
Assertions.assertEquals("k", tableRow(unpadded, PaimonWriteSchema.INSERT_OPERATION, string("k"),
411+
string("x")).getString(0).toString());
412+
413+
PaimonWriteSchema insert = PaimonWriteSchema.create(charKeyTableType(),
414+
new String[] {"code", "label"}, false, true, null, Collections.singletonList("code"));
415+
Assertions.assertEquals("k ", tableRow(insert, string("k"), string("x")).getString(0).toString());
416+
}
417+
358418
@Test
359419
public void testCharPaddingIsReadFromTheTableOptionsTheFeSets() {
360420
// FE puts doris.write.char-padding on the table it serializes for the writer (PaimonWriteBinding). An FE
@@ -383,6 +443,12 @@ private static RowType tableType() {
383443
new DataField(3, "region", new VarCharType())));
384444
}
385445

446+
private static RowType charKeyTableType() {
447+
return new RowType(Arrays.asList(
448+
new DataField(0, "code", new CharType(false, 3)),
449+
new DataField(1, "label", new CharType(3))));
450+
}
451+
386452
private static RowType charTableType() {
387453
return new RowType(Arrays.asList(
388454
new DataField(0, "id", new IntType()),

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonWriteBinding.java‎

Lines changed: 6 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -115,7 +115,8 @@ static FileStoreTable configureTableForWrite(FileStoreTable table, boolean overw
115115
* The static partition of an overwrite, as Paimon's static overwrite parses it: SQL NULL becomes the table's
116116
* {@code partition.default-name}, and every other value is the one the written rows carry, cast to the
117117
* column type, and padded as the BE writer pads the rows' CHAR values when {@code charPadding} is on: Paimon
118-
* parses a partition value without padding it. A binary column can only be statically overwritten with NULL.
118+
* parses a partition value without padding it. Like the writer, this leaves the table's {@code rowkind.field}
119+
* unpadded. A binary column can only be statically overwritten with NULL.
119120
*/
120121
static Map<String, String> resolveStaticPartition(FileStoreTable table, ConnectorWriteHandle writeHandle,
121122
boolean charPadding) {
@@ -141,7 +142,9 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto
141142
}
142143
requestedColumns.put(key, canonicalName);
143144
}
144-
String defaultPartitionName = CoreOptions.fromMap(table.options()).partitionDefaultName();
145+
CoreOptions options = CoreOptions.fromMap(table.options());
146+
String defaultPartitionName = options.partitionDefaultName();
147+
String rowKindField = options.rowkindField().orElse(null);
145148
Map<String, String> castValues = writeHandle.getCastStaticPartitionSpec();
146149
Map<String, String> result = new LinkedHashMap<>();
147150
for (Map.Entry<String, String> requested : requestedColumns.entrySet()) {
@@ -157,7 +160,7 @@ static Map<String, String> resolveStaticPartition(FileStoreTable table, Connecto
157160
if (type.getTypeRoot() == DataTypeRoot.TIMESTAMP_WITH_LOCAL_TIME_ZONE) {
158161
value = toSdkLocalTime(canonicalName, value);
159162
}
160-
if (charPadding && type.getTypeRoot() == DataTypeRoot.CHAR) {
163+
if (charPadding && type.getTypeRoot() == DataTypeRoot.CHAR && !canonicalName.equals(rowKindField)) {
161164
value = padChar(value, ((CharType) type).getLength());
162165
}
163166
if (defaultPartitionName.equals(value)) {

‎fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonWriteBindingTest.java‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,7 @@
4444
import java.time.OffsetDateTime;
4545
import java.util.Arrays;
4646
import java.util.Collections;
47+
import java.util.LinkedHashMap;
4748
import java.util.List;
4849
import java.util.Map;
4950
import java.util.Set;
@@ -253,6 +254,29 @@ public void staticCharPartitionIsPaddedAsTheRowsAre(@TempDir Path warehouse) {
253254
write(true, "part", "a", "a"), false).get("part"));
254255
}
255256

257+
@Test
258+
public void staticRowKindFieldPartitionIsLeftUnpaddedAsTheRowsAre(@TempDir Path warehouse) {
259+
// The BE writer does not pad the value of the table's rowkind.field, which Paimon reads as a row kind as it
260+
// is, so the rows of a CHAR(3) rowkind.field partition column carry '+I'. A padded '+I ' would name
261+
// another partition: a nonempty overwrite would fail and an empty one would clear nothing. The other CHAR
262+
// partition column is padded as before. MUTATION: padding the rowkind.field value -> red.
263+
List<DataField> fields = Arrays.asList(new DataField(0, "id", DataTypes.INT()),
264+
new DataField(1, "kind", DataTypes.CHAR(3)), new DataField(2, "code", DataTypes.CHAR(3)));
265+
TableSchema schema = new TableSchema(0L, fields, 2, Arrays.asList("kind", "code"),
266+
Collections.emptyList(), Collections.singletonMap(CoreOptions.ROWKIND_FIELD.key(), "kind"), "");
267+
FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(),
268+
new org.apache.paimon.fs.Path("file://" + warehouse + "/db.db/tbl"), schema);
269+
Map<String, String> spec = new LinkedHashMap<>();
270+
spec.put("kind", "+I");
271+
spec.put("code", "a");
272+
273+
Map<String, String> partition = PaimonWriteBinding.resolveStaticPartition(table,
274+
handle(true, spec, () -> spec, Collections.emptySet()), true);
275+
276+
Assertions.assertEquals("+I", partition.get("kind"));
277+
Assertions.assertEquals("a ", partition.get("code"));
278+
}
279+
256280
@Test
257281
public void insertLeavesTheStaticPartitionValuesAlone(@TempDir Path warehouse) {
258282
// Only a static overwrite reads the static partition. An INSERT writes each row to the partition its own

‎regression-test/data/external_table_p0/paimon/write/test_paimon_write_char_padding.out‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -40,3 +40,18 @@
4040
-- !rowkind_field_not_padded --
4141
2 [+I] [b ]
4242

43+
-- !merge_short_key_fails --
44+
45+
-- !merge_padded_key --
46+
[k ] 3 11
47+
48+
-- !merge_padding_off --
49+
[k ] 3 11
50+
[m ] 2 2
51+
[m] 1 1
52+
53+
-- !rowkind_partition_overwrite --
54+
3 [+I]
55+
56+
-- !rowkind_partition_empty_overwrite --
57+

0 commit comments

Comments
 (0)