2121import org .apache .paimon .casting .CastExecutor ;
2222import org .apache .paimon .casting .CastExecutors ;
2323import org .apache .paimon .casting .DefaultValueRow ;
24+ import org .apache .paimon .crosspartition .IndexBootstrap ;
2425import org .apache .paimon .data .BinaryRow ;
2526import org .apache .paimon .data .BinaryString ;
2627import org .apache .paimon .data .GenericArray ;
2930import org .apache .paimon .data .InternalMap ;
3031import org .apache .paimon .data .InternalRow ;
3132import org .apache .paimon .data .serializer .InternalRowSerializer ;
33+ import org .apache .paimon .schema .TableSchema ;
3234import org .apache .paimon .table .sink .RowKindGenerator ;
3335import org .apache .paimon .types .ArrayType ;
3436import org .apache .paimon .types .CharType ;
4850import java .util .HashMap ;
4951import java .util .List ;
5052import java .util .Map ;
53+ import java .util .function .UnaryOperator ;
5154
5255/**
5356 * Immutable mapping from Doris input columns to a Paimon table row.
@@ -80,8 +83,10 @@ final class PaimonWriteSchema {
8083 * that MERGE inserts can change it after MERGE matched it and checked it for duplicates; see {@link #padRow}.
8184 */
8285 private final int [] insertedKeyCharLengths ;
83- /** Whether the table writes ORC files ({@link #writesOrcFiles}); see {@link #padsRow} and {@link #padRow}. */
84- private final boolean orcFiles ;
86+ /** Per input column with an inserted key length, how Doris reads its stored keys back; see {@link #padRow}. */
87+ private final KeyReadBack [] insertedKeyReadBacks ;
88+ /** Which data files of the table are ORC files ({@link #orcFiles}); see {@link #padsRow}. */
89+ private final OrcFiles orcFiles ;
8590 /** Decodes the kind of a row from the table's rowkind.field as Paimon does when it writes the row, or null. */
8691 private final RowKindGenerator rowKindGenerator ;
8792 /**
@@ -101,8 +106,8 @@ final class PaimonWriteSchema {
101106
102107 private PaimonWriteSchema (int [] tableFieldIndexes , int [] omittedDefaultFieldIndexes ,
103108 Object [] omittedDefaultValues , int tableFieldCount , RowType inputType , CharPadder [] charPadders ,
104- int [] insertedKeyCharLengths , boolean orcFiles , RowKindGenerator rowKindGenerator ,
105- DefaultValueRow defaultValueRow , boolean overwrite ) {
109+ int [] insertedKeyCharLengths , KeyReadBack [] insertedKeyReadBacks , OrcFiles orcFiles ,
110+ RowKindGenerator rowKindGenerator , DefaultValueRow defaultValueRow , boolean overwrite ) {
106111 this .fieldGetters = new InternalRow .FieldGetter [inputType .getFieldCount ()];
107112 for (int i = 0 ; i < fieldGetters .length ; i ++) {
108113 // Getters describe the transport row, not the target table. Transport fields are
@@ -112,6 +117,7 @@ private PaimonWriteSchema(int[] tableFieldIndexes, int[] omittedDefaultFieldInde
112117 }
113118 this .charPadders = charPadders ;
114119 this .insertedKeyCharLengths = insertedKeyCharLengths ;
120+ this .insertedKeyReadBacks = insertedKeyReadBacks ;
115121 this .orcFiles = orcFiles ;
116122 this .rowKindGenerator = rowKindGenerator ;
117123 this .defaultValueRow = defaultValueRow ;
@@ -153,7 +159,8 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
153159
154160 static PaimonWriteSchema create (RowType tableType , String [] columnNames , boolean changelogWrite ,
155161 boolean charPadding , String rowKindField , List <String > primaryKeys ) {
156- return create (tableType , columnNames , changelogWrite , charPadding , rowKindField , primaryKeys , false , false );
162+ return create (tableType , columnNames , changelogWrite , charPadding , rowKindField , primaryKeys ,
163+ Collections .emptyList (), OrcFiles .NONE , false );
157164 }
158165
159166 /**
@@ -163,13 +170,14 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
163170 * kind from it ({@code RowKind.fromShortString}), and a padded '+I ' is no row kind.
164171 * @param primaryKeys the table's primary keys. In a changelog write, a CHAR key that MERGE inserts fails the
165172 * row when padding could make it a key that MERGE did not see; see {@link #padRow}.
166- * @param orcFiles whether the table writes ORC files ({@link #writesOrcFiles}), every row of which is
167- * padded; see {@link #padsRow}
173+ * @param partitionKeys the table's partition keys, whose values Doris reads back as they are stored
174+ * @param orcFiles which data files of the table are ORC files ({@link #orcFiles}); every row of a table
175+ * that writes some is padded, see {@link #padsRow}
168176 * @param overwrite whether the write overwrites the partitions it writes; see {@link #padsRow}
169177 */
170178 static PaimonWriteSchema create (RowType tableType , String [] columnNames , boolean changelogWrite ,
171- boolean charPadding , String rowKindField , List <String > primaryKeys , boolean orcFiles ,
172- boolean overwrite ) {
179+ boolean charPadding , String rowKindField , List <String > primaryKeys , List < String > partitionKeys ,
180+ OrcFiles orcFiles , boolean overwrite ) {
173181 if (columnNames == null || columnNames .length == 0 ) {
174182 throw new IllegalArgumentException (
175183 "PaimonJniWriter requires explicit column names" );
@@ -178,6 +186,7 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
178186 int [] tableFieldIndexes = new int [columnNames .length ];
179187 CharPadder [] charPadders = new CharPadder [columnNames .length ];
180188 int [] insertedKeyCharLengths = new int [columnNames .length ];
189+ KeyReadBack [] insertedKeyReadBacks = new KeyReadBack [columnNames .length ];
181190 List <DataField > inputFields = new ArrayList <>(columnNames .length );
182191 boolean [] specifiedFields = new boolean [tableType .getFieldCount ()];
183192 for (int i = 0 ; i < columnNames .length ; i ++) {
@@ -213,6 +222,9 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
213222 if (changelogWrite && field .type ().getTypeRoot () == DataTypeRoot .CHAR
214223 && primaryKeys .contains (field .name ())) {
215224 insertedKeyCharLengths [i ] = ((CharType ) field .type ()).getLength ();
225+ insertedKeyReadBacks [i ] = partitionKeys .contains (field .name ()) || orcFiles == OrcFiles .NONE
226+ ? KeyReadBack .PADDED
227+ : orcFiles == OrcFiles .ALL ? KeyReadBack .TRIMMED : KeyReadBack .PADDED_OR_TRIMMED ;
216228 }
217229 }
218230 }
@@ -239,6 +251,7 @@ static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean
239251 new RowType (inputFields ),
240252 charPadders ,
241253 insertedKeyCharLengths ,
254+ insertedKeyReadBacks ,
242255 orcFiles ,
243256 rowKindField == null ? null : new RowKindGenerator (rowKindField , tableType ),
244257 DefaultValueRow .create (tableType ),
@@ -253,11 +266,23 @@ static boolean charPadding(Map<String, String> tableOptions) {
253266 return Boolean .parseBoolean (tableOptions .get (DORIS_WRITE_CHAR_PADDING ));
254267 }
255268
256- /** Whether the table writes ORC files, by {@code file.format} or at some level by {@code file.format.per.level}. */
257- static boolean writesOrcFiles (Map <String , String > tableOptions ) {
269+ /**
270+ * Which data files of the table are ORC files: Paimon writes a file of a level in the format that
271+ * {@code file.format.per.level} names for the level, or else in {@code file.format}.
272+ */
273+ static OrcFiles orcFiles (Map <String , String > tableOptions ) {
258274 CoreOptions options = CoreOptions .fromMap (tableOptions );
259- return CoreOptions .FILE_FORMAT_ORC .equals (options .formatType ())
260- || options .fileFormatPerLevel ().containsValue (CoreOptions .FILE_FORMAT_ORC );
275+ Map <Integer , String > formatPerLevel = options .fileFormatPerLevel ();
276+ boolean orc = false ;
277+ boolean other = false ;
278+ for (int level = 0 ; level < options .numLevels (); level ++) {
279+ if (CoreOptions .FILE_FORMAT_ORC .equals (formatPerLevel .getOrDefault (level , options .formatType ()))) {
280+ orc = true ;
281+ } else {
282+ other = true ;
283+ }
284+ }
285+ return !orc ? OrcFiles .NONE : other ? OrcFiles .SOME : OrcFiles .ALL ;
261286 }
262287
263288 /** Paimon input fields in the exact order transported by Arrow C Data. */
@@ -292,11 +317,52 @@ GenericRow tableRow(InternalRow columnValues) {
292317 }
293318
294319 /**
295- * Whether to pad {@code row}. Every row of a table that writes ORC files is padded. ORC pads every CHAR value it
296- * writes and strips the trailing spaces from one it reads, so Doris reads back 'k' for the 'k ' that Spark and
297- * Flink write. Paimon puts a row in the bucket of its key as the writer holds it, and merges only the data files
298- * whose key ranges, also kept as the writer held the keys, overlap: an UPDATE or DELETE row that wrote that key
299- * back as 'k' would land in another bucket, or below the range of the file holding 'k ', and miss the row.
320+ * Returns what gives a row that Paimon's IndexBootstrap reads from the key-dynamic table of {@code schema}, to
321+ * index the keys stored in it, those keys as this writer writes them. Such a row holds the primary keys that are
322+ * no partition keys as the data files give them back, then the partition and the bucket
323+ * ({@code IndexBootstrap#bootstrapType}). ORC gives a CHAR key back without the trailing spaces that every row
324+ * this writer writes into a table that writes ORC files carries ({@link #padsRow}), so an index of the keys as
325+ * read would hold 'k' for the stored 'k ', and a row of 'k' written into another partition would miss it and
326+ * leave the stored row in place, two rows of one key. Those keys are padded as the rows are; the other files give
327+ * the keys back as they are stored, and the partition is the stored one.
328+ */
329+ UnaryOperator <InternalRow > bootstrapKeyPadder (TableSchema schema ) {
330+ List <String > keys = schema .trimmedPrimaryKeys ();
331+ CharPadder [] keyPadders = new CharPadder [keys .size ()];
332+ boolean padsKeys = false ;
333+ if (orcFiles != OrcFiles .NONE ) {
334+ for (int i = 0 ; i < keyPadders .length ; i ++) {
335+ // A key omitted from the input takes its default value unpadded, as the rows written do.
336+ int column = inputType .getFieldIndex (keys .get (i ));
337+ keyPadders [i ] = column < 0 ? null : charPadders [column ];
338+ padsKeys |= keyPadders [i ] != null ;
339+ }
340+ }
341+ if (!padsKeys ) {
342+ return UnaryOperator .identity ();
343+ }
344+ RowType bootstrapType = IndexBootstrap .bootstrapType (schema );
345+ InternalRow .FieldGetter [] bootstrapFieldGetters = new InternalRow .FieldGetter [bootstrapType .getFieldCount ()];
346+ for (int i = 0 ; i < bootstrapFieldGetters .length ; i ++) {
347+ bootstrapFieldGetters [i ] = InternalRow .createFieldGetter (bootstrapType .getTypeAt (i ), i );
348+ }
349+ return bootstrapRow -> {
350+ GenericRow padded = new GenericRow (bootstrapFieldGetters .length );
351+ for (int i = 0 ; i < bootstrapFieldGetters .length ; i ++) {
352+ Object value = bootstrapFieldGetters [i ].getFieldOrNull (bootstrapRow );
353+ padded .setField (i , i < keyPadders .length && keyPadders [i ] != null ? keyPadders [i ].pad (value ) : value );
354+ }
355+ return padded ;
356+ };
357+ }
358+
359+ /**
360+ * Whether to pad {@code row}. Every row of a table that writes ORC files, at some levels or at all, is padded.
361+ * ORC pads every CHAR value it writes and strips the trailing spaces from one it reads, so Doris reads back 'k'
362+ * for the 'k ' that Spark and Flink write. Paimon puts a row in the bucket of its key as the writer holds it,
363+ * and merges only the data files whose key ranges, also kept as the writer held the keys, overlap: an UPDATE or
364+ * DELETE row that wrote that key back as 'k' would land in another bucket, or below the range of the file
365+ * holding 'k ', and miss the row.
300366 *
301367 * <p>On any other table, a row is padded when it inserts, outside an overwrite. Its row kind, if any, is the
302368 * first input column. An UPDATE or DELETE row carries the key Doris read back from the table as it is stored,
@@ -311,7 +377,7 @@ GenericRow tableRow(InternalRow columnValues) {
311377 * (p = 'x') has to land in 'x ', and a '+I' and a '-D' of one key in the statement still cancel out.
312378 */
313379 private boolean padsRow (GenericRow row ) {
314- if (orcFiles ) {
380+ if (orcFiles != OrcFiles . NONE ) {
315381 return true ;
316382 }
317383 if (row .getRowKind () != RowKind .INSERT ) {
@@ -343,16 +409,36 @@ private void padRow(GenericRow row) {
343409
344410 /**
345411 * In a changelog write only MERGE inserts rows, and MERGE has matched them to the table and checked them for
346- * duplicate keys by the key as it is: Doris compares CHAR values unpadded. A table that writes no ORC files gives
347- * Doris its keys back padded, so padding a shorter key here would make 'k' and 'k ' one key, or let a NOT MATCHED
348- * row replace the stored 'k ' it did not match; a key as long as the column is not changed by padding. ORC gives
349- * the keys back without their trailing spaces, so there MERGE matched 'k' to the stored 'k ', and only a key
350- * that ends with a space, such as 'k ', matches no stored key yet pads to one.
412+ * duplicate keys by the key as it is: Doris compares CHAR values unpadded. So a key that padding could make a
413+ * stored key that MERGE did not match, or the same key as another one MERGE inserts, fails the row, and which
414+ * keys those are depends on how Doris reads the stored keys back ({@link KeyReadBack}). Read back padded, as the
415+ * values of a partition column and the keys of a table that writes no ORC files are, a key shorter than the
416+ * column, such as 'k', matched no stored 'k ' yet pads to it, and pads to the key 'k ' pads to; a key as long as
417+ * the column is not changed by padding. ORC gives the keys back without their trailing spaces, so there MERGE
418+ * matched 'k' to the stored 'k ', and only a key that ends with a space, such as 'k ', matches no stored key yet
419+ * pads to one. A table that writes ORC files at some levels only gives a key back in either form, by the file
420+ * that holds it, so both such keys fail the row.
351421 */
352422 private void checkInsertedKey (BinaryString key , int column ) {
353423 int length = insertedKeyCharLengths [column ];
354424 String name = inputType .getFields ().get (column ).name ();
355- if (orcFiles ) {
425+ KeyReadBack readBack = insertedKeyReadBacks [column ];
426+ if (readBack == KeyReadBack .PADDED_OR_TRIMMED ) {
427+ if (key .numChars () < length || key .endsWith (SPACE )) {
428+ // The remedy comes first: the client gets a long error message of the writer cut short.
429+ throw new IllegalArgumentException (String .format (
430+ "MERGE cannot insert '%s' into the CHAR(%d) primary key column '%s' of a Paimon table "
431+ + "that writes ORC files at some levels only. Insert keys as long as the column "
432+ + "that do not end with a space, or set enable_paimon_write_char_padding = false: "
433+ + "the table gives its keys back padded from its other files and without their "
434+ + "trailing spaces from its ORC files, so MERGE matched this value against keys in "
435+ + "either form, and padding it can turn it into a stored key that MERGE did not "
436+ + "match" ,
437+ key , length , name ));
438+ }
439+ return ;
440+ }
441+ if (readBack == KeyReadBack .TRIMMED ) {
356442 if (key .endsWith (SPACE )) {
357443 // The remedy comes first: the client gets a long error message of the writer cut short.
358444 throw new IllegalArgumentException (String .format (
@@ -657,6 +743,32 @@ private interface CharPadder {
657743 Object pad (Object value );
658744 }
659745
746+ /**
747+ * Which data files of a table are ORC files. ORC pads every CHAR value it writes and strips the trailing spaces
748+ * from one it reads, while the other formats give a value back as it is stored.
749+ */
750+ enum OrcFiles {
751+ /** No level writes ORC files. */
752+ NONE ,
753+ /** Some levels write ORC files, and the others another format. */
754+ SOME ,
755+ /** Every level writes ORC files. */
756+ ALL
757+ }
758+
759+ /** How Doris reads back the stored values of a CHAR primary-key column, against which MERGE matched its keys. */
760+ private enum KeyReadBack {
761+ /**
762+ * As they are stored, padded: the values of a partition column, which Paimon keeps in the partition, and the
763+ * keys of a table that writes no ORC files.
764+ */
765+ PADDED ,
766+ /** Without their trailing spaces: the keys of a table that writes only ORC files. */
767+ TRIMMED ,
768+ /** Padded from the files of some levels and trimmed from the ORC files of the others. */
769+ PADDED_OR_TRIMMED
770+ }
771+
660772 /**
661773 * Copies one non-null value of a type that holds a MAP, FLOAT or DOUBLE, with each NaN in it made the canonical
662774 * NaN and the entries of each MAP in it sorted, one per key.
0 commit comments