Skip to content

Commit e7bb9c6

Browse files
committed
[test](paimon) Add Paimon write unit coverage
### What problem does this PR solve? Issue Number: part of apache#65086 Related PR: apache#64924 Problem Summary: Add focused unit coverage for the Paimon write P0 path. The new tests cover JNI writer schema and commit payload boundaries, FE transaction commit-message aggregation and failure handling, and physical sink distribution requirements for bucket-unaware, fixed bucket, and unsupported bucket modes. ### Release note None ### Check List (For Author) - Test: Unit Test - mvn test -pl be-java-extensions/paimon-scanner -am -Dcheckstyle.skip=true -DfailIfNoTests=false -Dmaven.build.cache.enabled=false -Dtest=org.apache.doris.paimon.PaimonJniWriterTest - ./run-fe-ut.sh --run org.apache.doris.datasource.paimon.PaimonTransactionTest - ./run-fe-ut.sh --run org.apache.doris.nereids.trees.plans.physical.PhysicalPaimonTableSinkTest - git diff --check - Behavior changed: No - Does this need documentation: No
1 parent cf8f827 commit e7bb9c6

3 files changed

Lines changed: 323 additions & 0 deletions

File tree

‎fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniWriterTest.java‎

Lines changed: 94 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,11 @@
2424
import org.junit.Test;
2525

2626
import java.io.IOException;
27+
import java.lang.reflect.Field;
28+
import java.lang.reflect.Method;
29+
import java.util.HashMap;
30+
import java.util.Map;
31+
import java.util.stream.Collectors;
2732

2833
public class PaimonJniWriterTest {
2934
@Test
@@ -39,6 +44,56 @@ public void testFullRowWriteRequiresSameFieldOrder() {
3944
Assert.assertFalse(PaimonJniWriter.isFullRowWrite(tableType, differentOrder));
4045
}
4146

47+
@Test
48+
public void testFullRowWriteRejectsMissingColumns() {
49+
RowType tableType = RowType.of(new DataType[] {
50+
DataTypes.INT(), DataTypes.STRING()}, new String[] {"k", "v"});
51+
RowType partialWrite = RowType.of(new DataType[] {
52+
DataTypes.INT()}, new String[] {"k"});
53+
54+
Assert.assertFalse(PaimonJniWriter.isFullRowWrite(tableType, partialWrite));
55+
}
56+
57+
@Test
58+
public void testBuildWriteTypeUsesRequestedColumnOrder() throws Exception {
59+
RowType tableType = RowType.of(new DataType[] {
60+
DataTypes.INT(), DataTypes.STRING()}, new String[] {"k", "v"});
61+
PaimonJniWriter writer = writerWithSchema(tableType);
62+
63+
RowType writeType = invokeBuildWriteType(writer, new String[] {"v", "k"});
64+
65+
Assert.assertEquals("[v, k]", writeType.getFieldNames().toString());
66+
Assert.assertEquals(tableType.getFields().get(1).type(), writeType.getFields().get(0).type());
67+
Assert.assertEquals(tableType.getFields().get(0).type(), writeType.getFields().get(1).type());
68+
closeAllocator(writer);
69+
}
70+
71+
@Test
72+
public void testBuildWriteTypeRequiresExplicitColumns() throws Exception {
73+
RowType tableType = RowType.of(new DataType[] {
74+
DataTypes.INT()}, new String[] {"k"});
75+
PaimonJniWriter writer = writerWithSchema(tableType);
76+
77+
IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class,
78+
() -> invokeBuildWriteType(writer, new String[0]));
79+
80+
Assert.assertTrue(exception.getMessage().contains("explicit column names"));
81+
closeAllocator(writer);
82+
}
83+
84+
@Test
85+
public void testBuildWriteTypeRejectsUnknownColumn() throws Exception {
86+
RowType tableType = RowType.of(new DataType[] {
87+
DataTypes.INT()}, new String[] {"k"});
88+
PaimonJniWriter writer = writerWithSchema(tableType);
89+
90+
IllegalArgumentException exception = Assert.assertThrows(IllegalArgumentException.class,
91+
() -> invokeBuildWriteType(writer, new String[] {"missing"}));
92+
93+
Assert.assertTrue(exception.getMessage().contains("Cannot find Paimon column 'missing'"));
94+
closeAllocator(writer);
95+
}
96+
4297
@Test
4398
public void testOversizedCommitPayloadReducesMultiMessageChunk() throws Exception {
4499
byte[] payload = new byte[PaimonJniWriter.MAX_COMMIT_PAYLOAD_BYTES + 1];
@@ -54,4 +109,43 @@ public void testOversizedSingleCommitPayloadFails() {
54109
() -> PaimonJniWriter.shouldReduceCommitChunk(payload, 1));
55110
Assert.assertTrue(exception.getMessage().contains("exceeds limit"));
56111
}
112+
113+
@Test
114+
public void testCommitPayloadWithinLimitDoesNotReduceChunk() throws Exception {
115+
byte[] payload = new byte[PaimonJniWriter.MAX_COMMIT_PAYLOAD_BYTES];
116+
117+
Assert.assertFalse(PaimonJniWriter.shouldReduceCommitChunk(payload, 512));
118+
Assert.assertFalse(PaimonJniWriter.shouldReduceCommitChunk(payload, 1));
119+
}
120+
121+
private static PaimonJniWriter writerWithSchema(RowType tableType) throws Exception {
122+
PaimonJniWriter writer = new PaimonJniWriter();
123+
Map<String, org.apache.paimon.types.DataField> fieldMap = tableType.getFields().stream()
124+
.collect(Collectors.toMap(org.apache.paimon.types.DataField::name,
125+
java.util.function.Function.identity(), (left, right) -> left, HashMap::new));
126+
Field field = PaimonJniWriter.class.getDeclaredField("paimonFieldMap");
127+
field.setAccessible(true);
128+
field.set(writer, fieldMap);
129+
return writer;
130+
}
131+
132+
private static RowType invokeBuildWriteType(PaimonJniWriter writer, String[] columnNames) throws Exception {
133+
Method method = PaimonJniWriter.class.getDeclaredMethod("buildWriteType", String[].class);
134+
method.setAccessible(true);
135+
try {
136+
return (RowType) method.invoke(writer, new Object[] {columnNames});
137+
} catch (java.lang.reflect.InvocationTargetException e) {
138+
Throwable cause = e.getCause();
139+
if (cause instanceof RuntimeException) {
140+
throw (RuntimeException) cause;
141+
}
142+
throw e;
143+
}
144+
}
145+
146+
private static void closeAllocator(PaimonJniWriter writer) throws Exception {
147+
Field field = PaimonJniWriter.class.getDeclaredField("allocator");
148+
field.setAccessible(true);
149+
((AutoCloseable) field.get(writer)).close();
150+
}
57151
}

‎fe/fe-core/src/test/java/org/apache/doris/datasource/paimon/PaimonTransactionTest.java‎

Lines changed: 74 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,20 @@ public void testDeserializeCommitMessagePayload() throws Exception {
6767
Assert.assertEquals(11, messages.get(0).bucket());
6868
}
6969

70+
@Test
71+
public void testDeserializeCommitMessagePayloadsAggregatesChunks() throws Exception {
72+
byte[] firstPayload = serializePayload(Collections.singletonList(newCommitMessage(11)));
73+
byte[] secondPayload = serializePayload(Arrays.asList(newCommitMessage(12), newCommitMessage(13)));
74+
75+
List<CommitMessage> messages = PaimonTransaction.deserializeCommitMessagePayloads(
76+
Arrays.asList(firstPayload, secondPayload));
77+
78+
Assert.assertEquals(3, messages.size());
79+
Assert.assertEquals(11, messages.get(0).bucket());
80+
Assert.assertEquals(12, messages.get(1).bucket());
81+
Assert.assertEquals(13, messages.get(2).bucket());
82+
}
83+
7084
@Test
7185
public void testDeserializeCommitMessagePayloadRejectsInvalidHeader() {
7286
IOException exception = Assert.assertThrows(IOException.class,
@@ -105,6 +119,25 @@ public void testCommitUsesDecodedMessagesAndMarksCommitted() throws Exception {
105119
Assert.assertEquals(3, executor.committedMessages.get(0).bucket());
106120
}
107121

122+
@Test
123+
public void testCommitAggregatesMessagesFromMultiplePayloads() throws Exception {
124+
RecordingCommitExecutor executor = new RecordingCommitExecutor();
125+
PaimonTransaction transaction = new PaimonTransaction(executor);
126+
transaction.beginInsert(mockTable(), Optional.empty());
127+
transaction.setTransactionId(142L);
128+
transaction.updateCommitMessages(Arrays.asList(
129+
newThriftMessage(serializePayload(Collections.singletonList(newCommitMessage(30)))),
130+
newThriftMessage(serializePayload(Arrays.asList(newCommitMessage(31), newCommitMessage(32))))));
131+
132+
transaction.commit();
133+
134+
Assert.assertEquals(1, executor.commitCalls);
135+
Assert.assertEquals(3, executor.committedMessages.size());
136+
Assert.assertEquals(30, executor.committedMessages.get(0).bucket());
137+
Assert.assertEquals(31, executor.committedMessages.get(1).bucket());
138+
Assert.assertEquals(32, executor.committedMessages.get(2).bucket());
139+
}
140+
108141
@Test
109142
public void testCommitWithEmptyPayloadIsNoop() throws Exception {
110143
RecordingCommitExecutor executor = new RecordingCommitExecutor();
@@ -115,6 +148,34 @@ public void testCommitWithEmptyPayloadIsNoop() throws Exception {
115148
Assert.assertEquals(0, executor.commitCalls);
116149
}
117150

151+
@Test
152+
public void testCommitWithPayloadRequiresTable() throws Exception {
153+
RecordingCommitExecutor executor = new RecordingCommitExecutor();
154+
PaimonTransaction transaction = new PaimonTransaction(executor);
155+
transaction.setTransactionId(143L);
156+
transaction.updateCommitMessages(Collections.singletonList(
157+
newThriftMessage(serializePayload(Collections.singletonList(newCommitMessage(33))))));
158+
159+
UserException exception = Assert.assertThrows(UserException.class, transaction::commit);
160+
161+
Assert.assertTrue(exception.getMessage().contains("Missing paimon table"));
162+
Assert.assertEquals(0, executor.commitCalls);
163+
}
164+
165+
@Test
166+
public void testCommitWithPayloadRequiresTransactionId() throws Exception {
167+
RecordingCommitExecutor executor = new RecordingCommitExecutor();
168+
PaimonTransaction transaction = new PaimonTransaction(executor);
169+
transaction.beginInsert(mockTable(), Optional.empty());
170+
transaction.updateCommitMessages(Collections.singletonList(
171+
newThriftMessage(serializePayload(Collections.singletonList(newCommitMessage(34))))));
172+
173+
UserException exception = Assert.assertThrows(UserException.class, transaction::commit);
174+
175+
Assert.assertTrue(exception.getMessage().contains("Missing transaction id"));
176+
Assert.assertEquals(0, executor.commitCalls);
177+
}
178+
118179
@Test
119180
public void testRollbackAbortsDecodedMessages() throws Exception {
120181
RecordingCommitExecutor executor = new RecordingCommitExecutor();
@@ -153,6 +214,19 @@ public void testRollbackAfterCommitIsNoop() throws Exception {
153214
Assert.assertEquals(0, executor.abortCalls);
154215
}
155216

217+
@Test
218+
public void testRollbackWithPayloadAndMissingTableSkipsAbort() throws Exception {
219+
RecordingCommitExecutor executor = new RecordingCommitExecutor();
220+
PaimonTransaction transaction = new PaimonTransaction(executor);
221+
transaction.setTransactionId(144L);
222+
transaction.updateCommitMessages(Collections.singletonList(
223+
newThriftMessage(serializePayload(Collections.singletonList(newCommitMessage(35))))));
224+
225+
transaction.rollback();
226+
227+
Assert.assertEquals(0, executor.abortCalls);
228+
}
229+
156230
@Test
157231
public void testRollbackDoesNotThrowWhenAbortFails() throws Exception {
158232
RecordingCommitExecutor executor = new RecordingCommitExecutor();
Lines changed: 155 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,155 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.trees.plans.physical;
19+
20+
import org.apache.doris.catalog.Column;
21+
import org.apache.doris.catalog.PrimitiveType;
22+
import org.apache.doris.datasource.paimon.PaimonExternalDatabase;
23+
import org.apache.doris.datasource.paimon.PaimonExternalTable;
24+
import org.apache.doris.nereids.properties.DataTrait;
25+
import org.apache.doris.nereids.properties.DistributionSpecHiveTableSinkHashPartitioned;
26+
import org.apache.doris.nereids.properties.LogicalProperties;
27+
import org.apache.doris.nereids.properties.PhysicalProperties;
28+
import org.apache.doris.nereids.trees.expressions.NamedExpression;
29+
import org.apache.doris.nereids.trees.expressions.Slot;
30+
import org.apache.doris.nereids.trees.expressions.SlotReference;
31+
import org.apache.doris.nereids.trees.plans.Plan;
32+
import org.apache.doris.nereids.types.IntegerType;
33+
import org.apache.doris.nereids.types.StringType;
34+
35+
import org.apache.paimon.schema.TableSchema;
36+
import org.apache.paimon.table.BucketMode;
37+
import org.apache.paimon.table.FileStoreTable;
38+
import org.junit.Assert;
39+
import org.junit.Test;
40+
import org.mockito.Mockito;
41+
42+
import java.util.Arrays;
43+
import java.util.BitSet;
44+
import java.util.Collections;
45+
import java.util.List;
46+
import java.util.Optional;
47+
import java.util.stream.Collectors;
48+
49+
public class PhysicalPaimonTableSinkTest {
50+
51+
@Test
52+
public void testBucketUnawareWithoutDistributionColumnsUsesRandomPartitioned() {
53+
SlotReference valueSlot = new SlotReference("v", StringType.INSTANCE);
54+
PhysicalPaimonTableSink<Plan> sink = newSink(
55+
BucketMode.BUCKET_UNAWARE,
56+
Collections.emptyList(),
57+
Collections.emptyList(),
58+
Collections.singletonList(new Column("v", PrimitiveType.STRING)),
59+
Collections.singletonList(valueSlot));
60+
61+
Assert.assertSame(PhysicalProperties.SINK_RANDOM_PARTITIONED, sink.getRequirePhysicalProperties());
62+
}
63+
64+
@Test
65+
public void testFixedBucketUsesPartitionAndBucketKeyExprIds() {
66+
SlotReference partitionSlot = new SlotReference("pt", IntegerType.INSTANCE);
67+
SlotReference bucketSlot = new SlotReference("bucket_key", IntegerType.INSTANCE);
68+
SlotReference valueSlot = new SlotReference("v", StringType.INSTANCE);
69+
PhysicalPaimonTableSink<Plan> sink = newSink(
70+
BucketMode.HASH_FIXED,
71+
Collections.singletonList("bucket_key"),
72+
Collections.singletonList(new Column("pt", PrimitiveType.INT)),
73+
Arrays.asList(
74+
new Column("pt", PrimitiveType.INT),
75+
new Column("bucket_key", PrimitiveType.INT),
76+
new Column("v", PrimitiveType.STRING)),
77+
Arrays.asList(partitionSlot, bucketSlot, valueSlot));
78+
79+
PhysicalProperties properties = sink.getRequirePhysicalProperties();
80+
81+
Assert.assertTrue(properties.getDistributionSpec()
82+
instanceof DistributionSpecHiveTableSinkHashPartitioned);
83+
DistributionSpecHiveTableSinkHashPartitioned distribution =
84+
(DistributionSpecHiveTableSinkHashPartitioned) properties.getDistributionSpec();
85+
Assert.assertEquals(Arrays.asList(partitionSlot.getExprId(), bucketSlot.getExprId()),
86+
distribution.getOutputColExprIds());
87+
}
88+
89+
@Test
90+
public void testFixedBucketRequiresBucketKeyInSinkOutput() {
91+
SlotReference partitionSlot = new SlotReference("pt", IntegerType.INSTANCE);
92+
SlotReference valueSlot = new SlotReference("v", StringType.INSTANCE);
93+
PhysicalPaimonTableSink<Plan> sink = newSink(
94+
BucketMode.HASH_FIXED,
95+
Collections.singletonList("bucket_key"),
96+
Collections.singletonList(new Column("pt", PrimitiveType.INT)),
97+
Arrays.asList(new Column("pt", PrimitiveType.INT), new Column("v", PrimitiveType.STRING)),
98+
Arrays.asList(partitionSlot, valueSlot));
99+
100+
UnsupportedOperationException exception = Assert.assertThrows(UnsupportedOperationException.class,
101+
sink::getRequirePhysicalProperties);
102+
103+
Assert.assertTrue(exception.getMessage().contains("requires distribution column"));
104+
}
105+
106+
@Test
107+
public void testUnsupportedBucketModeFailsFast() {
108+
SlotReference valueSlot = new SlotReference("v", StringType.INSTANCE);
109+
PhysicalPaimonTableSink<Plan> sink = newSink(
110+
BucketMode.HASH_DYNAMIC,
111+
Collections.emptyList(),
112+
Collections.emptyList(),
113+
Collections.singletonList(new Column("v", PrimitiveType.STRING)),
114+
Collections.singletonList(valueSlot));
115+
116+
UnsupportedOperationException exception = Assert.assertThrows(UnsupportedOperationException.class,
117+
sink::getRequirePhysicalProperties);
118+
119+
Assert.assertTrue(exception.getMessage().contains("Unsupported Paimon bucket mode"));
120+
}
121+
122+
private static PhysicalPaimonTableSink<Plan> newSink(BucketMode bucketMode, List<String> bucketKeys,
123+
List<Column> partitionColumns, List<Column> columns, List<SlotReference> outputSlots) {
124+
PaimonExternalTable table = Mockito.mock(PaimonExternalTable.class);
125+
FileStoreTable fileStoreTable = Mockito.mock(FileStoreTable.class);
126+
TableSchema schema = Mockito.mock(TableSchema.class);
127+
Mockito.when(table.getPaimonTable(Mockito.any(Optional.class))).thenReturn(fileStoreTable);
128+
Mockito.when(table.getPartitionColumns(Mockito.any(Optional.class))).thenReturn(partitionColumns);
129+
Mockito.when(fileStoreTable.bucketMode()).thenReturn(bucketMode);
130+
Mockito.when(fileStoreTable.schema()).thenReturn(schema);
131+
Mockito.when(schema.bucketKeys()).thenReturn(bucketKeys);
132+
133+
Plan child = Mockito.mock(Plan.class);
134+
Mockito.when(child.getAllChildrenTypes()).thenReturn(new BitSet());
135+
Mockito.when(child.depth()).thenReturn(1);
136+
Mockito.when(child.getOutput()).thenReturn(outputSlots.stream()
137+
.map(Slot.class::cast)
138+
.collect(Collectors.toList()));
139+
List<Slot> output = outputSlots.stream()
140+
.map(Slot.class::cast)
141+
.collect(Collectors.toList());
142+
List<NamedExpression> outputExprs = outputSlots.stream()
143+
.map(NamedExpression.class::cast)
144+
.collect(Collectors.toList());
145+
146+
return new PhysicalPaimonTableSink<>(
147+
Mockito.mock(PaimonExternalDatabase.class),
148+
table,
149+
columns,
150+
outputExprs,
151+
Optional.empty(),
152+
new LogicalProperties(() -> output, () -> DataTrait.EMPTY_TRAIT),
153+
child);
154+
}
155+
}

0 commit comments

Comments
 (0)