Skip to content

Commit b584341

Browse files
morningmanclaude
andcommitted
[fix](paimon) Keep CHAR routing fields off the native bucket route while padding
With padding on, the writer pads a CHAR value before Paimon computes the partition and the bucket of the row, but the fixed-bucket route (paimon_fixed_bucket) hashes the routing values that BE sends, unpadded. Rows of one bucket then reached several writers. In a full-compaction table each of them compacted the bucket, and Paimon gave up committing the compaction after the rows were committed (LSM conflicts, then file deletion conflicts), so the INSERT reported success without its changelog. A CHAR partition or bucket field is now routed only while padding is off; with padding on the table gets the distribution of a routing field BE cannot hash, a single writer for a primary-key table. The writer also leaves the value of the table's rowkind.field alone: Paimon takes the row kind from it as it is, and a CHAR(3) '+I' padded to '+I ' failed the write. Co-Authored-By: Claude Opus 5.5 (1M context) <[email protected]>
1 parent f6b658b commit b584341

7 files changed

Lines changed: 147 additions & 22 deletions

File tree

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -184,7 +184,7 @@ public void open(String serializedTable, Map<String, String> hadoopConfig,
184184

185185
CoreOptions coreOptions = CoreOptions.fromMap(table.options());
186186
this.writeSchema = PaimonWriteSchema.create(table.rowType(), columnNames, changelogWrite,
187-
PaimonWriteSchema.charPadding(table.options()));
187+
PaimonWriteSchema.charPadding(table.options()), coreOptions.rowkindField().orElse(null));
188188
this.arrowAdapter = new PaimonArrowBatchAdapter(
189189
writeSchema.inputType(), ZoneId.of(timeZone), allocator);
190190
validateWriteColumnsForMergeEngine(

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonWriteSchema.java‎

Lines changed: 22 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -59,7 +59,10 @@ final class PaimonWriteSchema {
5959
static final String DORIS_WRITE_CHAR_PADDING = "doris.write.char-padding";
6060

6161
private final InternalRow.FieldGetter[] fieldGetters;
62-
/** Per input column, pads the CHAR values it holds; null when it holds none or padding is off. */
62+
/**
63+
* Per input column, pads the CHAR values it holds; null when it holds none, when padding is off, or for the
64+
* table's rowkind.field.
65+
*/
6366
private final CharPadder[] charPadders;
6467
private final RowType inputType;
6568
/** Maps Doris input-column position → Paimon table-schema position. */
@@ -70,16 +73,15 @@ final class PaimonWriteSchema {
7073
private final int tableFieldCount;
7174

7275
private PaimonWriteSchema(int[] tableFieldIndexes, int[] omittedDefaultFieldIndexes,
73-
Object[] omittedDefaultValues, int tableFieldCount, RowType inputType, boolean charPadding) {
76+
Object[] omittedDefaultValues, int tableFieldCount, RowType inputType, CharPadder[] charPadders) {
7477
this.fieldGetters = new InternalRow.FieldGetter[inputType.getFieldCount()];
75-
this.charPadders = new CharPadder[inputType.getFieldCount()];
7678
for (int i = 0; i < fieldGetters.length; i++) {
7779
// Getters describe the transport row, not the target table. Transport fields are
7880
// nullable so an invalid Doris NULL survives projection and reaches Paimon's
7981
// authoritative table-schema validation.
8082
this.fieldGetters[i] = InternalRow.createFieldGetter(inputType.getTypeAt(i), i);
81-
this.charPadders[i] = charPadding ? charPadder(inputType.getTypeAt(i)) : null;
8283
}
84+
this.charPadders = charPadders;
8385
this.inputType = inputType;
8486
this.tableFieldIndexes = tableFieldIndexes;
8587
this.omittedDefaultFieldIndexes = omittedDefaultFieldIndexes;
@@ -105,18 +107,26 @@ static PaimonWriteSchema create(
105107
return create(tableType, columnNames, changelogWrite, false);
106108
}
107109

108-
/**
109-
* @param charPadding whether to pad the CHAR values of inserted rows as Paimon's cast to CHAR does; see
110-
* {@link #charPadder}
111-
*/
112110
static PaimonWriteSchema create(
113111
RowType tableType, String[] columnNames, boolean changelogWrite, boolean charPadding) {
112+
return create(tableType, columnNames, changelogWrite, charPadding, null);
113+
}
114+
115+
/**
116+
* @param charPadding whether to pad the CHAR values of inserted rows as Paimon's cast to CHAR does; see
117+
* {@link #charPadder}
118+
* @param rowKindField the table's {@code rowkind.field}, or null. Its value is not padded: Paimon takes the row
119+
* kind from it as it is ({@code RowKind.fromShortString}), and a padded '+I' is no row kind.
120+
*/
121+
static PaimonWriteSchema create(RowType tableType, String[] columnNames, boolean changelogWrite,
122+
boolean charPadding, String rowKindField) {
114123
if (columnNames == null || columnNames.length == 0) {
115124
throw new IllegalArgumentException(
116125
"PaimonJniWriter requires explicit column names");
117126
}
118127

119128
int[] tableFieldIndexes = new int[columnNames.length];
129+
CharPadder[] charPadders = new CharPadder[columnNames.length];
120130
List<DataField> inputFields = new ArrayList<>(columnNames.length);
121131
boolean[] specifiedFields = new boolean[tableType.getFieldCount()];
122132
for (int i = 0; i < columnNames.length; i++) {
@@ -147,6 +157,9 @@ static PaimonWriteSchema create(
147157
// Paimon writer return its normal constraint diagnostic instead of reading a default
148158
// primitive value from a non-null getter.
149159
inputFields.add(field.newType(field.type().nullable()));
160+
if (charPadding && !field.name().equals(rowKindField)) {
161+
charPadders[i] = charPadder(field.type());
162+
}
150163
}
151164

152165
int[] omittedDefaultFieldIndexes = new int[tableType.getFieldCount()];
@@ -169,7 +182,7 @@ static PaimonWriteSchema create(
169182
Arrays.copyOf(omittedDefaultValues, omittedDefaultCount),
170183
tableType.getFieldCount(),
171184
new RowType(inputFields),
172-
charPadding);
185+
charPadders);
173186
}
174187

175188
/**

‎fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonWriteSchemaTest.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -338,6 +338,23 @@ public void testCharPaddingLeavesUpdateAndDeleteRowsAlone() {
338338
.getString(1).toString());
339339
}
340340

341+
@Test
342+
public void testCharPaddingLeavesTheRowKindFieldAlone() {
343+
// Paimon takes the row kind of a row from the value of its rowkind.field as it is (RowKind.fromShortString),
344+
// so a CHAR(3) '+I' padded to '+I ' would fail the write. The other CHAR columns are padded.
345+
// MUTATION: padding the rowkind.field column -> red.
346+
RowType tableType = new RowType(Arrays.asList(
347+
new DataField(0, "id", new IntType()),
348+
new DataField(1, "kind", new CharType(3)),
349+
new DataField(2, "code", new CharType(3))));
350+
PaimonWriteSchema schema = PaimonWriteSchema.create(tableType,
351+
new String[] {"id", "kind", "code"}, false, true, "kind");
352+
353+
InternalRow tableRow = tableRow(schema, 1, string("+I"), string("a"));
354+
Assertions.assertEquals("+I", tableRow.getString(1).toString());
355+
Assertions.assertEquals("a ", tableRow.getString(2).toString());
356+
}
357+
341358
@Test
342359
public void testCharPaddingIsReadFromTheTableOptionsTheFeSets() {
343360
// FE puts doris.write.char-padding on the table it serializes for the writer (PaimonWriteBinding). An FE

‎fe/fe-connector/fe-connector-paimon/src/main/java/org/apache/doris/connector/paimon/PaimonWritePlanProvider.java‎

Lines changed: 16 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -347,7 +347,7 @@ public boolean requiresMaterializeStaticPartitionValues() {
347347
public ConnectorWriteDistribution getWriteDistribution(ConnectorSession session,
348348
ConnectorTableHandle connectorHandle) {
349349
FileStoreTable table = resolveTable((PaimonTableHandle) connectorHandle);
350-
ConnectorWriteDistribution fixedBucket = fixedBucketDistribution(table);
350+
ConnectorWriteDistribution fixedBucket = fixedBucketDistribution(table, charPaddingEnabled(session));
351351
if (fixedBucket != null) {
352352
return fixedBucket;
353353
}
@@ -365,7 +365,14 @@ public ConnectorWriteDistribution getWriteDistribution(ConnectorSession session,
365365
ConnectorWriteDistribution.Mode.EXTERNAL_UNPARTITIONED);
366366
}
367367

368-
private ConnectorWriteDistribution fixedBucketDistribution(FileStoreTable table) {
368+
/**
369+
* Routes each row to the writer that owns its partition and bucket, which BE computes from the row's routing
370+
* values. With {@code charPadding} on, the writer pads a CHAR value before Paimon computes the partition and the
371+
* bucket of the row, while BE would hash the unpadded value, and several writers would write and compact the
372+
* same bucket. A CHAR routing field then cannot be routed natively: this returns null, and the table gets the
373+
* distribution of a table whose routing fields BE cannot hash, a single writer for a primary-key table.
374+
*/
375+
private ConnectorWriteDistribution fixedBucketDistribution(FileStoreTable table, boolean charPadding) {
369376
if (table.bucketMode() != BucketMode.HASH_FIXED) {
370377
return null;
371378
}
@@ -382,9 +389,9 @@ private ConnectorWriteDistribution fixedBucketDistribution(FileStoreTable table)
382389
List<String> routeColumns = new ArrayList<>();
383390
Map<String, Integer> routeIndexes = new TreeMap<>(String.CASE_INSENSITIVE_ORDER);
384391
List<Integer> partitionIndexes = appendRouteFields(
385-
schema.partitionKeys(), fields, routeColumns, routeIndexes);
392+
schema.partitionKeys(), fields, routeColumns, routeIndexes, charPadding);
386393
List<Integer> bucketIndexes = appendRouteFields(
387-
schema.bucketKeys(), fields, routeColumns, routeIndexes);
394+
schema.bucketKeys(), fields, routeColumns, routeIndexes, charPadding);
388395
if (partitionIndexes == null || bucketIndexes == null || bucketIndexes.isEmpty()) {
389396
return null;
390397
}
@@ -402,12 +409,12 @@ private String joinIndexes(List<Integer> indexes) {
402409

403410
private List<Integer> appendRouteFields(List<String> fieldNames,
404411
Map<String, DataField> fields, List<String> routeColumns,
405-
Map<String, Integer> routeIndexes) {
412+
Map<String, Integer> routeIndexes, boolean charPadding) {
406413
List<Integer> indexes = new ArrayList<>();
407414
for (String fieldName : fieldNames) {
408415
DataField field = fields.get(fieldName);
409416
if (field == null || field.defaultValue() != null
410-
|| !supportsNativeRouting(field.type().getTypeRoot())) {
417+
|| !supportsNativeRouting(field.type().getTypeRoot(), charPadding)) {
411418
return null;
412419
}
413420
Integer index = routeIndexes.get(fieldName);
@@ -421,16 +428,17 @@ private List<Integer> appendRouteFields(List<String> fieldNames,
421428
return indexes;
422429
}
423430

424-
private boolean supportsNativeRouting(DataTypeRoot type) {
431+
private boolean supportsNativeRouting(DataTypeRoot type, boolean charPadding) {
425432
switch (type) {
433+
case CHAR:
434+
return !charPadding;
426435
case BOOLEAN:
427436
case TINYINT:
428437
case SMALLINT:
429438
case INTEGER:
430439
case BIGINT:
431440
case FLOAT:
432441
case DOUBLE:
433-
case CHAR:
434442
case VARCHAR:
435443
case BINARY:
436444
case VARBINARY:

‎fe/fe-connector/fe-connector-paimon/src/test/java/org/apache/doris/connector/paimon/PaimonWritePlanProviderTest.java‎

Lines changed: 37 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -262,37 +262,70 @@ public void defaultedFixedBucketRouteFieldsFallBackToSingleWriter() {
262262
Collections.singletonList("part"), Collections.emptyMap()).getMode());
263263
}
264264

265+
@Test
266+
public void charRouteFieldsFallBackToSingleWriterWhenCharPaddingIsOn() {
267+
// The writer pads a CHAR value before Paimon computes the partition and the bucket of the row, while the
268+
// BE exchange would hash the unpadded value. Several writers would then write and compact one bucket, and
269+
// Paimon gives up committing the compaction (LSM conflicts) after the rows are committed, so a
270+
// full-compaction changelog goes missing. MUTATION: routing CHAR natively with padding on -> red.
271+
List<DataField> charKey = Arrays.asList(
272+
new DataField(0, "c", DataTypes.CHAR(3).notNull()),
273+
new DataField(1, "v", DataTypes.INT()));
274+
Map<String, String> options = Collections.singletonMap(CoreOptions.BUCKET.key(), "4");
275+
Assertions.assertEquals(ConnectorWriteDistribution.Mode.GATHER, writeDistribution(charKey,
276+
Collections.emptyList(), Collections.singletonList("c"), options, BucketMode.HASH_FIXED, true)
277+
.getMode());
278+
Assertions.assertEquals("paimon_fixed_bucket", writeDistribution(charKey,
279+
Collections.emptyList(), Collections.singletonList("c"), options, BucketMode.HASH_FIXED, false)
280+
.getPartitionFunction());
281+
// The partition fields are routing fields as well.
282+
List<DataField> charPartition = Arrays.asList(
283+
new DataField(0, "id", DataTypes.INT()),
284+
new DataField(1, "part", DataTypes.CHAR(3)));
285+
Assertions.assertEquals(ConnectorWriteDistribution.Mode.GATHER, fixedBucketDistribution(charPartition,
286+
Collections.singletonList("part"), Collections.emptyMap(), true).getMode());
287+
Assertions.assertEquals("paimon_fixed_bucket", fixedBucketDistribution(charPartition,
288+
Collections.singletonList("part"), Collections.emptyMap(), false).getPartitionFunction());
289+
}
290+
265291
private static ConnectorWriteDistribution.Mode writeDistribution(List<String> partitionKeys,
266292
List<String> primaryKeys, Map<String, String> options, BucketMode expectedBucketMode) {
267293
List<DataField> fields = Arrays.asList(
268294
new DataField(0, "id", DataTypes.INT().notNull()),
269295
new DataField(1, "dt", DataTypes.DATE().notNull()),
270296
new DataField(2, "region", DataTypes.STRING().notNull()));
271-
return writeDistribution(fields, partitionKeys, primaryKeys, options, expectedBucketMode)
297+
return writeDistribution(fields, partitionKeys, primaryKeys, options, expectedBucketMode, true)
272298
.getMode();
273299
}
274300

275301
private static ConnectorWriteDistribution writeDistribution(List<DataField> fields,
276302
List<String> partitionKeys, List<String> primaryKeys, Map<String, String> options,
277-
BucketMode expectedBucketMode) {
303+
BucketMode expectedBucketMode, boolean charPadding) {
278304
int highestFieldId = fields.stream().mapToInt(DataField::id).max().getAsInt();
279305
FileStoreTable table = FileStoreTableFactory.create(LocalFileIO.create(),
280306
new Path("file:///tmp/paimon-write-distribution/tbl"),
281307
new TableSchema(0L, fields, highestFieldId, partitionKeys, primaryKeys, options, ""));
282308
Assertions.assertEquals(expectedBucketMode, table.bucketMode(), "fixture bucket mode");
283309
PaimonTableHandle handle = new PaimonTableHandle("db", "tbl", partitionKeys, primaryKeys);
284310
handle.setPaimonTable(table);
285-
return new PaimonWritePlanProvider(null, null, null).getWriteDistribution(null, handle);
311+
return new PaimonWritePlanProvider(null, null, null).getWriteDistribution(new PropertySession(
312+
Collections.singletonMap(PaimonWritePlanProvider.ENABLE_CHAR_PADDING, Boolean.toString(charPadding))),
313+
handle);
286314
}
287315

288316
/** A compacting append table with 4 fixed buckets keyed by {@code id}. */
289317
private static ConnectorWriteDistribution fixedBucketDistribution(List<DataField> fields,
290318
List<String> partitionKeys, Map<String, String> extraOptions) {
319+
return fixedBucketDistribution(fields, partitionKeys, extraOptions, true);
320+
}
321+
322+
private static ConnectorWriteDistribution fixedBucketDistribution(List<DataField> fields,
323+
List<String> partitionKeys, Map<String, String> extraOptions, boolean charPadding) {
291324
Map<String, String> options = new HashMap<>(extraOptions);
292325
options.put(CoreOptions.BUCKET.key(), "4");
293326
options.put(CoreOptions.BUCKET_KEY.key(), "id");
294327
return writeDistribution(fields, partitionKeys, Collections.emptyList(), options,
295-
BucketMode.HASH_FIXED);
328+
BucketMode.HASH_FIXED, charPadding);
296329
}
297330

298331
private static Map<String, String> fixedBucketOptions(boolean writeOnly) {

‎regression-test/data/external_table_p0/paimon/write/test_paimon_write_char_padding.out‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -30,3 +30,13 @@
3030
[k ] 3 2
3131
[w] 1 40
3232

33+
-- !bucketed_padded --
34+
100 100 3 3
35+
36+
-- !bucketed_compaction_committed --
37+
1 APPEND \N
38+
2 COMPACT 100
39+
40+
-- !rowkind_field_not_padded --
41+
2 [+I] [b ]
42+

‎regression-test/suites/external_table_p0/paimon/write/test_paimon_write_char_padding.groovy‎

Lines changed: 44 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -56,6 +56,26 @@ suite("test_paimon_write_char_padding", "p0,external,paimon") {
5656
id INT, p CHAR(3)
5757
) USING paimon
5858
PARTITIONED BY (p);
59+
60+
DROP TABLE IF EXISTS paimon.${dbName}.t_char_bucketed;
61+
CREATE TABLE paimon.${dbName}.t_char_bucketed (
62+
c CHAR(3), v INT
63+
) USING paimon
64+
TBLPROPERTIES (
65+
'primary-key' = 'c',
66+
'bucket' = '4',
67+
'changelog-producer' = 'full-compaction'
68+
);
69+
70+
DROP TABLE IF EXISTS paimon.${dbName}.t_char_rowkind;
71+
CREATE TABLE paimon.${dbName}.t_char_rowkind (
72+
id INT, rk CHAR(3), code CHAR(3)
73+
) USING paimon
74+
TBLPROPERTIES (
75+
'primary-key' = 'id',
76+
'bucket' = '1',
77+
'rowkind.field' = 'rk'
78+
);
5979
"""
6080

6181
sql """drop catalog if exists ${catalogName}"""
@@ -112,7 +132,31 @@ suite("test_paimon_write_char_padding", "p0,external,paimon") {
112132
sql """DELETE FROM t_char_pk WHERE v = 3"""
113133
sql """UPDATE t_char_pk SET v = 40 WHERE v = 4"""
114134
order_qt_dml_matches_unpadded_keys """SELECT concat('[', c, ']'), char_length(c), v FROM t_char_pk"""
135+
136+
// BE cannot route a CHAR bucket key by Paimon's bucket while padding is on, since it would hash the
137+
// unpadded value. Rows of one bucket would then reach several of the four writers, each would compact
138+
// the bucket, and Paimon would give up committing the compaction, and with it the changelog, after the
139+
// rows were committed. The write goes to one writer instead, and both snapshots are committed.
140+
sql """set parallel_pipeline_task_num = 4"""
141+
sql """INSERT INTO t_char_bucketed
142+
SELECT cast(number AS STRING), cast(number AS INT) FROM numbers("number" = "100")"""
143+
sql """set parallel_pipeline_task_num = 0"""
144+
order_qt_bucketed_padded """
145+
SELECT count(*), count(DISTINCT c), min(char_length(c)), max(char_length(c)) FROM t_char_bucketed
146+
"""
147+
order_qt_bucketed_compaction_committed """
148+
SELECT snapshot_id, commit_kind, changelog_record_count FROM t_char_bucketed\$snapshots
149+
"""
150+
151+
// Paimon takes the row kind from the value of the rowkind.field as it is, so that value is not padded:
152+
// '-D' deletes the row of key 1.
153+
sql """INSERT INTO t_char_rowkind VALUES (1, '+I', 'a'), (2, '+I', 'b')"""
154+
sql """INSERT INTO t_char_rowkind VALUES (1, '-D', 'a')"""
155+
order_qt_rowkind_field_not_padded """
156+
SELECT id, concat('[', rk, ']'), concat('[', code, ']') FROM t_char_rowkind
157+
"""
115158
} finally {
116159
sql """set enable_paimon_write_char_padding = true"""
160+
sql """set parallel_pipeline_task_num = 0"""
117161
}
118162
}

0 commit comments

Comments
 (0)