Skip to content

Commit 0037d01

Browse files
morningmanclaude
andcommitted
[fix](paimon) Index padded keys and keep MERGE checks where Paimon gives CHAR keys back padded
### What problem does this PR solve? Issue Number: None Related PR: #69083 Problem Summary: The previous commit pads every row written to a Paimon table that writes ORC files, and on such a table lets a MERGE insert a short CHAR key, failing only one that ends with a space. That went wrong in three cases, all reproduced on a local cluster: - A cross-partition table (a primary key without the partition column, bucket = -1) bootstraps its global key index from the keys its files hold, which ORC gives back without their trailing spaces. The index held 'k' while the rows written carry 'k ', so 'k' written into p2 after it was written into p1, by Doris or by Spark, missed the stored row and left two rows of 'k'; master (#69083) kept one. The writer now pads the bootstrapped keys of such a table as it pads its rows, and keeps the partition values, which are read from the partitions as they are stored. - A CHAR partition column in the primary key: Paimon keeps the value in the partition, so Doris reads it back padded from an ORC table too. A MERGE whose source partition value 'x' did not match the stored 'x ' inserted 'x', padded it, and replaced that row. A partition key keeps the rule of #69083 again: a MERGE insert shorter than the column fails. - A table that writes ORC files at some levels only, by file.format.per.level (such as 'orc' with '0:avro'), gives a key back padded from its other files and trimmed from its ORC files. A MERGE insert is then safe only when it is as long as the column and does not end with a space, and the others now fail. The formats of a table are read per level, from 0 to num-levels - 1. ### Release note Extends the note of the previous commit: a MERGE into a Paimon table that writes ORC files at some levels only fails when it would insert a CHAR primary key shorter than the column or ending with a space, and a MERGE into an ORC table fails when it would insert a CHAR partition key shorter than the column. ### Check List (For Author) - Test: Regression test (test_paimon_write_char_padding: cross-partition upserts by Doris and Spark, MERGE with a CHAR partition key, MERGE into an 'orc' + '0:avro' table; fails with the previous commit's jar) and Unit Test (PaimonWriteSchemaTest; a mutation of each new rule turns it red) - Behavior changed: Yes, the MERGE failures above - Does this need documentation: Yes, with the documentation of enable_paimon_write_char_padding from #69083 Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent c8a30d2 commit 0037d01

5 files changed

Lines changed: 451 additions & 45 deletions

File tree

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,7 @@
6464
import java.util.concurrent.ExecutorService;
6565
import java.util.concurrent.Executors;
6666
import java.util.concurrent.TimeUnit;
67+
import java.util.function.UnaryOperator;
6768

6869
/**
6970
* JNI entry point for Paimon write operations.
@@ -185,7 +186,8 @@ public void open(String serializedTable, Map<String, String> hadoopConfig,
185186
CoreOptions coreOptions = CoreOptions.fromMap(table.options());
186187
this.writeSchema = PaimonWriteSchema.create(table.rowType(), columnNames, changelogWrite,
187188
PaimonWriteSchema.charPadding(table.options()), coreOptions.rowkindField().orElse(null),
188-
table.primaryKeys(), PaimonWriteSchema.writesOrcFiles(table.options()), overwrite);
189+
table.primaryKeys(), table.partitionKeys(), PaimonWriteSchema.orcFiles(table.options()),
190+
overwrite);
189191
this.arrowAdapter = new PaimonArrowBatchAdapter(
190192
writeSchema.inputType(), ZoneId.of(timeZone), allocator);
191193
validateWriteColumnsForMergeEngine(
@@ -470,8 +472,10 @@ private void openKeyDynamicBucketAssigner(FileStoreTable table,
470472
long writerMemoryLimitBytes) throws Exception {
471473
globalIndexAssigner = new GlobalIndexAssigner(table, writerMemoryLimitBytes);
472474
globalIndexAssigner.open(1, 0, this::writeAssignedRow);
475+
// The index has to hold the stored keys as the rows written carry them, which ORC does not give back.
476+
UnaryOperator<InternalRow> bootstrapKeyPadder = writeSchema.bootstrapKeyPadder(table.schema());
473477
new IndexBootstrap(table).bootstrap(
474-
1, 0, this::bootstrapGlobalIndexKey);
478+
1, 0, row -> bootstrapGlobalIndexKey(bootstrapKeyPadder.apply(row)));
475479
globalIndexAssigner.finishBootstrap();
476480
}
477481

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonWriteSchema.java‎

Lines changed: 137 additions & 25 deletions
Original file line numberDiff line numberDiff line change
@@ -21,6 +21,7 @@
2121
import org.apache.paimon.casting.CastExecutor;
2222
import org.apache.paimon.casting.CastExecutors;
2323
import org.apache.paimon.casting.DefaultValueRow;
24+
import org.apache.paimon.crosspartition.IndexBootstrap;
2425
import org.apache.paimon.data.BinaryRow;
2526
import org.apache.paimon.data.BinaryString;
2627
import org.apache.paimon.data.GenericArray;
@@ -29,6 +30,7 @@
2930
import org.apache.paimon.data.InternalMap;
3031
import org.apache.paimon.data.InternalRow;
3132
import org.apache.paimon.data.serializer.InternalRowSerializer;
33+
import org.apache.paimon.schema.TableSchema;
3234
import org.apache.paimon.table.sink.RowKindGenerator;
3335
import org.apache.paimon.types.ArrayType;
3436
import org.apache.paimon.types.CharType;
@@ -48,6 +50,7 @@
4850
import java.util.HashMap;
4951
import java.util.List;
5052
import java.util.Map;
53+
import java.util.function.UnaryOperator;
5154

5255
/**
5356
* Immutable mapping from Doris input columns to a Paimon table row.
@@ -80,8 +83,10 @@ final class PaimonWriteSchema {
8083
* that MERGE inserts can change it after MERGE matched it and checked it for duplicates; see {@link #padRow}.
8184
*/
8285
private final int[] insertedKeyCharLengths;
83-
/** Whether the table writes ORC files ({@link #writesOrcFiles}); see {@link #padsRow} and {@link #padRow}. */
84-
private final boolean orcFiles;
86+
/** Per input column with an inserted key length, how Doris reads its stored keys back; see {@link #padRow}. */
87+
private final KeyReadBack[] insertedKeyReadBacks;
88+
/** Which data files of the table are ORC files ({@link #orcFiles}); see {@link #padsRow}. */
89+
private final OrcFiles orcFiles;
8590
/** Decodes the kind of a row from the table's rowkind.field as Paimon does when it writes the row, or null. */
8691
private final RowKindGenerator rowKindGenerator;
8792
/**
@@ -101,8 +106,8 @@ final class PaimonWriteSchema {
101106

102107
private PaimonWriteSchema(int[] tableFieldIndexes, int[] omittedDefaultFieldIndexes,
103108
Object[] omittedDefaultValues, int tableFieldCount, RowType inputType, CharPadder[] charPadders,
104-
int[] insertedKeyCharLengths, boolean orcFiles, RowKindGenerator rowKindGenerator,
105-
DefaultValueRow defaultValueRow, boolean overwrite) {
109+
int[] insertedKeyCharLengths, KeyReadBack[] insertedKeyReadBacks, OrcFiles orcFiles,
110+
RowKindGenerator rowKindGenerator, DefaultValueRow defaultValueRow, boolean overwrite) {
106111
this.fieldGetters = new InternalRow.FieldGetter[inputType.getFieldCount()];
107112
for (int i = 0; i < fieldGetters.length; i++) {
108113
// Getters describe the transport row, not the target table. Transport fields are
@@ -112,6 +117,7 @@ private PaimonWriteSchema(int[] tableFieldIndexes, int[] omittedDefaultFieldInde
112117
}
113118
this.charPadders = charPadders;
114119
this.insertedKeyCharLengths = insertedKeyCharLengths;
120+
this.insertedKeyReadBacks = insertedKeyReadBacks;
115121
this.orcFiles = orcFiles;
116122
this.rowKindGenerator = rowKindGenerator;
117123
this.defaultValueRow = defaultValueRow;
@@ -153,7 +159,8 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
153159

154160
static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean changelogWrite,
155161
boolean charPadding, String rowKindField, List<String> primaryKeys) {
156-
return create(tableType, columnNames, changelogWrite, charPadding, rowKindField, primaryKeys, false, false);
162+
return create(tableType, columnNames, changelogWrite, charPadding, rowKindField, primaryKeys,
163+
Collections.emptyList(), OrcFiles.NONE, false);
157164
}
158165

159166
/**
@@ -163,13 +170,14 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
163170
* kind from it ({@code RowKind.fromShortString}), and a padded '+I ' is no row kind.
164171
* @param primaryKeys the table's primary keys. In a changelog write, a CHAR key that MERGE inserts fails the
165172
* row when padding could make it a key that MERGE did not see; see {@link #padRow}.
166-
* @param orcFiles whether the table writes ORC files ({@link #writesOrcFiles}), every row of which is
167-
* padded; see {@link #padsRow}
173+
* @param partitionKeys the table's partition keys, whose values Doris reads back as they are stored
174+
* @param orcFiles which data files of the table are ORC files ({@link #orcFiles}); every row of a table
175+
* that writes some is padded, see {@link #padsRow}
168176
* @param overwrite whether the write overwrites the partitions it writes; see {@link #padsRow}
169177
*/
170178
static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean changelogWrite,
171-
boolean charPadding, String rowKindField, List<String> primaryKeys, boolean orcFiles,
172-
boolean overwrite) {
179+
boolean charPadding, String rowKindField, List<String> primaryKeys, List<String> partitionKeys,
180+
OrcFiles orcFiles, boolean overwrite) {
173181
if (columnNames == null || columnNames.length == 0) {
174182
throw new IllegalArgumentException(
175183
"PaimonJniWriter requires explicit column names");
@@ -178,6 +186,7 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
178186
int[] tableFieldIndexes = new int[columnNames.length];
179187
CharPadder[] charPadders = new CharPadder[columnNames.length];
180188
int[] insertedKeyCharLengths = new int[columnNames.length];
189+
KeyReadBack[] insertedKeyReadBacks = new KeyReadBack[columnNames.length];
181190
List<DataField> inputFields = new ArrayList<>(columnNames.length);
182191
boolean[] specifiedFields = new boolean[tableType.getFieldCount()];
183192
for (int i = 0; i < columnNames.length; i++) {
@@ -213,6 +222,9 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
213222
if (changelogWrite && field.type().getTypeRoot() == DataTypeRoot.CHAR
214223
&& primaryKeys.contains(field.name())) {
215224
insertedKeyCharLengths[i] = ((CharType) field.type()).getLength();
225+
insertedKeyReadBacks[i] = partitionKeys.contains(field.name()) || orcFiles == OrcFiles.NONE
226+
? KeyReadBack.PADDED
227+
: orcFiles == OrcFiles.ALL ? KeyReadBack.TRIMMED : KeyReadBack.PADDED_OR_TRIMMED;
216228
}
217229
}
218230
}
@@ -239,6 +251,7 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
239251
new RowType(inputFields),
240252
charPadders,
241253
insertedKeyCharLengths,
254+
insertedKeyReadBacks,
242255
orcFiles,
243256
rowKindField == null ? null : new RowKindGenerator(rowKindField, tableType),
244257
DefaultValueRow.create(tableType),
@@ -253,11 +266,23 @@ static boolean charPadding(Map<String, String> tableOptions) {
253266
return Boolean.parseBoolean(tableOptions.get(DORIS_WRITE_CHAR_PADDING));
254267
}
255268

256-
/** Whether the table writes ORC files, by {@code file.format} or at some level by {@code file.format.per.level}. */
257-
static boolean writesOrcFiles(Map<String, String> tableOptions) {
269+
/**
270+
* Which data files of the table are ORC files: Paimon writes a file of a level in the format that
271+
* {@code file.format.per.level} names for the level, or else in {@code file.format}.
272+
*/
273+
static OrcFiles orcFiles(Map<String, String> tableOptions) {
258274
CoreOptions options = CoreOptions.fromMap(tableOptions);
259-
return CoreOptions.FILE_FORMAT_ORC.equals(options.formatType())
260-
|| options.fileFormatPerLevel().containsValue(CoreOptions.FILE_FORMAT_ORC);
275+
Map<Integer, String> formatPerLevel = options.fileFormatPerLevel();
276+
boolean orc = false;
277+
boolean other = false;
278+
for (int level = 0; level < options.numLevels(); level++) {
279+
if (CoreOptions.FILE_FORMAT_ORC.equals(formatPerLevel.getOrDefault(level, options.formatType()))) {
280+
orc = true;
281+
} else {
282+
other = true;
283+
}
284+
}
285+
return !orc ? OrcFiles.NONE : other ? OrcFiles.SOME : OrcFiles.ALL;
261286
}
262287

263288
/** Paimon input fields in the exact order transported by Arrow C Data. */
@@ -292,11 +317,52 @@ GenericRow tableRow(InternalRow columnValues) {
292317
}
293318

294319
/**
295-
* Whether to pad {@code row}. Every row of a table that writes ORC files is padded. ORC pads every CHAR value it
296-
* writes and strips the trailing spaces from one it reads, so Doris reads back 'k' for the 'k ' that Spark and
297-
* Flink write. Paimon puts a row in the bucket of its key as the writer holds it, and merges only the data files
298-
* whose key ranges, also kept as the writer held the keys, overlap: an UPDATE or DELETE row that wrote that key
299-
* back as 'k' would land in another bucket, or below the range of the file holding 'k ', and miss the row.
320+
* Returns what gives a row that Paimon's IndexBootstrap reads from the key-dynamic table of {@code schema}, to
321+
* index the keys stored in it, those keys as this writer writes them. Such a row holds the primary keys that are
322+
* no partition keys as the data files give them back, then the partition and the bucket
323+
* ({@code IndexBootstrap#bootstrapType}). ORC gives a CHAR key back without the trailing spaces that every row
324+
* this writer writes into a table that writes ORC files carries ({@link #padsRow}), so an index of the keys as
325+
* read would hold 'k' for the stored 'k ', and a row of 'k' written into another partition would miss it and
326+
* leave the stored row in place, two rows of one key. Those keys are padded as the rows are; the other files give
327+
* the keys back as they are stored, and the partition is the stored one.
328+
*/
329+
UnaryOperator<InternalRow> bootstrapKeyPadder(TableSchema schema) {
330+
List<String> keys = schema.trimmedPrimaryKeys();
331+
CharPadder[] keyPadders = new CharPadder[keys.size()];
332+
boolean padsKeys = false;
333+
if (orcFiles != OrcFiles.NONE) {
334+
for (int i = 0; i < keyPadders.length; i++) {
335+
// A key omitted from the input takes its default value unpadded, as the rows written do.
336+
int column = inputType.getFieldIndex(keys.get(i));
337+
keyPadders[i] = column < 0 ? null : charPadders[column];
338+
padsKeys |= keyPadders[i] != null;
339+
}
340+
}
341+
if (!padsKeys) {
342+
return UnaryOperator.identity();
343+
}
344+
RowType bootstrapType = IndexBootstrap.bootstrapType(schema);
345+
InternalRow.FieldGetter[] bootstrapFieldGetters = new InternalRow.FieldGetter[bootstrapType.getFieldCount()];
346+
for (int i = 0; i < bootstrapFieldGetters.length; i++) {
347+
bootstrapFieldGetters[i] = InternalRow.createFieldGetter(bootstrapType.getTypeAt(i), i);
348+
}
349+
return bootstrapRow -> {
350+
GenericRow padded = new GenericRow(bootstrapFieldGetters.length);
351+
for (int i = 0; i < bootstrapFieldGetters.length; i++) {
352+
Object value = bootstrapFieldGetters[i].getFieldOrNull(bootstrapRow);
353+
padded.setField(i, i < keyPadders.length && keyPadders[i] != null ? keyPadders[i].pad(value) : value);
354+
}
355+
return padded;
356+
};
357+
}
358+
359+
/**
360+
* Whether to pad {@code row}. Every row of a table that writes ORC files, at some levels or at all, is padded.
361+
* ORC pads every CHAR value it writes and strips the trailing spaces from one it reads, so Doris reads back 'k'
362+
* for the 'k ' that Spark and Flink write. Paimon puts a row in the bucket of its key as the writer holds it,
363+
* and merges only the data files whose key ranges, also kept as the writer held the keys, overlap: an UPDATE or
364+
* DELETE row that wrote that key back as 'k' would land in another bucket, or below the range of the file
365+
* holding 'k ', and miss the row.
300366
*
301367
* <p>On any other table, a row is padded when it inserts, outside an overwrite. Its row kind, if any, is the
302368
* first input column. An UPDATE or DELETE row carries the key Doris read back from the table as it is stored,
@@ -311,7 +377,7 @@ GenericRow tableRow(InternalRow columnValues) {
311377
* (p = 'x') has to land in 'x ', and a '+I' and a '-D' of one key in the statement still cancel out.
312378
*/
313379
private boolean padsRow(GenericRow row) {
314-
if (orcFiles) {
380+
if (orcFiles != OrcFiles.NONE) {
315381
return true;
316382
}
317383
if (row.getRowKind() != RowKind.INSERT) {
@@ -343,16 +409,36 @@ private void padRow(GenericRow row) {
343409

344410
/**
345411
* In a changelog write only MERGE inserts rows, and MERGE has matched them to the table and checked them for
346-
* duplicate keys by the key as it is: Doris compares CHAR values unpadded. A table that writes no ORC files gives
347-
* Doris its keys back padded, so padding a shorter key here would make 'k' and 'k ' one key, or let a NOT MATCHED
348-
* row replace the stored 'k ' it did not match; a key as long as the column is not changed by padding. ORC gives
349-
* the keys back without their trailing spaces, so there MERGE matched 'k' to the stored 'k ', and only a key
350-
* that ends with a space, such as 'k ', matches no stored key yet pads to one.
412+
* duplicate keys by the key as it is: Doris compares CHAR values unpadded. So a key that padding could make a
413+
* stored key that MERGE did not match, or the same key as another one MERGE inserts, fails the row, and which
414+
* keys those are depends on how Doris reads the stored keys back ({@link KeyReadBack}). Read back padded, as the
415+
* values of a partition column and the keys of a table that writes no ORC files are, a key shorter than the
416+
* column, such as 'k', matched no stored 'k ' yet pads to it, and pads to the key 'k ' pads to; a key as long as
417+
* the column is not changed by padding. ORC gives the keys back without their trailing spaces, so there MERGE
418+
* matched 'k' to the stored 'k ', and only a key that ends with a space, such as 'k ', matches no stored key yet
419+
* pads to one. A table that writes ORC files at some levels only gives a key back in either form, by the file
420+
* that holds it, so both such keys fail the row.
351421
*/
352422
private void checkInsertedKey(BinaryString key, int column) {
353423
int length = insertedKeyCharLengths[column];
354424
String name = inputType.getFields().get(column).name();
355-
if (orcFiles) {
425+
KeyReadBack readBack = insertedKeyReadBacks[column];
426+
if (readBack == KeyReadBack.PADDED_OR_TRIMMED) {
427+
if (key.numChars() < length || key.endsWith(SPACE)) {
428+
// The remedy comes first: the client gets a long error message of the writer cut short.
429+
throw new IllegalArgumentException(String.format(
430+
"MERGE cannot insert '%s' into the CHAR(%d) primary key column '%s' of a Paimon table "
431+
+ "that writes ORC files at some levels only. Insert keys as long as the column "
432+
+ "that do not end with a space, or set enable_paimon_write_char_padding = false: "
433+
+ "the table gives its keys back padded from its other files and without their "
434+
+ "trailing spaces from its ORC files, so MERGE matched this value against keys in "
435+
+ "either form, and padding it can turn it into a stored key that MERGE did not "
436+
+ "match",
437+
key, length, name));
438+
}
439+
return;
440+
}
441+
if (readBack == KeyReadBack.TRIMMED) {
356442
if (key.endsWith(SPACE)) {
357443
// The remedy comes first: the client gets a long error message of the writer cut short.
358444
throw new IllegalArgumentException(String.format(
@@ -657,6 +743,32 @@ private interface CharPadder {
657743
Object pad(Object value);
658744
}
659745

746+
/**
747+
* Which data files of a table are ORC files. ORC pads every CHAR value it writes and strips the trailing spaces
748+
* from one it reads, while the other formats give a value back as it is stored.
749+
*/
750+
enum OrcFiles {
751+
/** No level writes ORC files. */
752+
NONE,
753+
/** Some levels write ORC files, and the others another format. */
754+
SOME,
755+
/** Every level writes ORC files. */
756+
ALL
757+
}
758+
759+
/** How Doris reads back the stored values of a CHAR primary-key column, against which MERGE matched its keys. */
760+
private enum KeyReadBack {
761+
/**
762+
* As they are stored, padded: the values of a partition column, which Paimon keeps in the partition, and the
763+
* keys of a table that writes no ORC files.
764+
*/
765+
PADDED,
766+
/** Without their trailing spaces: the keys of a table that writes only ORC files. */
767+
TRIMMED,
768+
/** Padded from the files of some levels and trimmed from the ORC files of the others. */
769+
PADDED_OR_TRIMMED
770+
}
771+
660772
/**
661773
* Copies one non-null value of a type that holds a MAP, FLOAT or DOUBLE, with each NaN in it made the canonical
662774
* NaN and the entries of each MAP in it sorted, one per key.

0 commit comments

Comments
 (0)