Skip to content

Commit bebb70b

Browse files
committed
[refactor](paimon) Clarify SDK writer boundary and extend coverage
### What problem does this PR solve? Issue Number: close apache#65086 Related PR: apache#65381 Problem Summary: The Java-side Paimon writer lifecycle abstraction mixed SDK resource ownership with externally exposed writer operations, while its unit tests exercised weak lifecycle checks instead of stable configuration and delegation contracts. The JNI bridge also reconstructed DirectByteBuffer through reflective Java internals, and the Paimon table writer retained an unused initialization hook. Rename and narrow the SDK writer boundary, keep routing, bundle, commit, and spill ownership together, validate partial primary-key writes before allocating SDK resources, construct the direct buffer through JNI, and remove the empty hook. Replace the old tests with mock-based contract tests that perform no table writes, and add regression coverage for string bucket routing, spill, and failed-write snapshot isolation. ### Release note None ### Check List (For Author) - Test: Unit Test - EXTRA_FE_MODULES=paimon=be-java-extensions/paimon-connector ./run-fe-ut.sh --run org.apache.doris.paimon.PaimonSdkWriterTest: 5 tests passed - paimon-connector production and test sources compiled successfully as part of the unit-test run - Behavior changed: No - Does this need documentation: No
1 parent 92703b1 commit bebb70b

12 files changed

Lines changed: 361 additions & 223 deletions

File tree

‎be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp‎

Lines changed: 11 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -150,7 +150,7 @@ Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* s
150150
// Cache method IDs
151151
jmethodID open_id = env->GetMethodID(_jni_writer_cls, "open",
152152
"(Ljava/lang/String;Ljava/util/Map;[Ljava/lang/String;)V");
153-
_write_id = env->GetMethodID(_jni_writer_cls, "write", "(JI)V");
153+
_write_id = env->GetMethodID(_jni_writer_cls, "write", "(Ljava/nio/ByteBuffer;)V");
154154
_prepare_commit_id = env->GetMethodID(_jni_writer_cls, "prepareCommit", "()[[B");
155155
_abort_id = env->GetMethodID(_jni_writer_cls, "abort", "()V");
156156
_close_id = env->GetMethodID(_jni_writer_cls, "close", "()V");
@@ -288,12 +288,16 @@ Status JniPaimonWriter::_write_projected_block(RuntimeState* state, Block& block
288288
JNIEnv* env = nullptr;
289289
RETURN_IF_ERROR(_get_jni_env(&env));
290290

291-
auto address = reinterpret_cast<jlong>(buffer->data());
292-
jint length = static_cast<jint>(buffer->size());
293-
env->CallVoidMethod(_jni_writer_obj, _write_id, address, length);
294-
RETURN_IF_ERROR(
295-
Jni::Env::GetJniExceptionMsg(env, false, "JNI exception in JniPaimonWriter::write: "));
296-
return Status::OK();
291+
jobject direct_buffer =
292+
env->NewDirectByteBuffer(buffer->mutable_data(), static_cast<jlong>(buffer->size()));
293+
RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg(
294+
env, false, "JNI exception in NewDirectByteBuffer for PaimonJniWriter::write: "));
295+
296+
env->CallVoidMethod(_jni_writer_obj, _write_id, direct_buffer);
297+
Status write_status =
298+
Jni::Env::GetJniExceptionMsg(env, false, "JNI exception in JniPaimonWriter::write: ");
299+
env->DeleteLocalRef(direct_buffer);
300+
return write_status;
297301
}
298302

299303
Status JniPaimonWriter::write(RuntimeState* state, Block& block) {

‎be/src/exec/sink/writer/paimon/vpaimon_table_writer.h‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -31,7 +31,6 @@
3131

3232
namespace doris {
3333

34-
class ObjectPool;
3534
class RuntimeState;
3635

3736
/// Each PaimonTableSinkLocalState owns one VPaimonTableWriter and one SDK
@@ -68,8 +67,6 @@ class VPaimonTableWriter final : public AsyncResultWriter {
6867

6968
~VPaimonTableWriter() override = default;
7069

71-
Status init_properties(ObjectPool* pool) { return Status::OK(); }
72-
7370
Status open(RuntimeState* state, RuntimeProfile* profile) override;
7471

7572
Status write(RuntimeState* state, Block& block) override;

‎fe/be-java-extensions/paimon-connector/pom.xml‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -66,6 +66,12 @@ under the License.
6666
<artifactId>arrow-vector</artifactId>
6767
</dependency>
6868

69+
<dependency>
70+
<groupId>org.mockito</groupId>
71+
<artifactId>mockito-core</artifactId>
72+
<scope>test</scope>
73+
</dependency>
74+
6975
</dependencies>
7076

7177
<build>

‎fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java‎

Lines changed: 16 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -30,22 +30,17 @@
3030
import org.apache.paimon.catalog.CatalogContext;
3131
import org.apache.paimon.catalog.CatalogFactory;
3232
import org.apache.paimon.catalog.Identifier;
33-
import org.apache.paimon.data.BinaryRow;
3433
import org.apache.paimon.data.GenericRow;
3534
import org.apache.paimon.data.InternalRow;
36-
import org.apache.paimon.io.BundleRecords;
3735
import org.apache.paimon.options.Options;
3836
import org.apache.paimon.table.Table;
39-
import org.apache.paimon.table.sink.BatchTableWrite;
4037
import org.apache.paimon.table.sink.CommitMessage;
4138
import org.slf4j.Logger;
4239
import org.slf4j.LoggerFactory;
4340

4441
import java.io.InputStream;
45-
import java.lang.reflect.Constructor;
4642
import java.nio.ByteBuffer;
4743
import java.util.ArrayList;
48-
import java.util.Iterator;
4944
import java.util.LinkedHashMap;
5045
import java.util.List;
5146
import java.util.Map;
@@ -57,7 +52,7 @@
5752
* Called from C++ (VPaimonTableWriter) via JNI. Data path:
5853
*
5954
* C++ Block → Arrow IPC Stream → direct memory address
60-
* → PaimonJniWriter.write(address, length)
55+
* → PaimonJniWriter.write(directBuffer)
6156
* → ArrowStreamReader → VectorSchemaRoot
6257
* → PaimonArrowConverter (column-major typed extraction)
6358
* → group by writer.getPartition(row) + writer.getBucket(row)
@@ -80,16 +75,14 @@ public class PaimonJniWriter {
8075
private final ClassLoader classLoader;
8176
private final PaimonArrowConverter arrowConverter = new PaimonArrowConverter();
8277
private final PaimonCommitCodec commitCodec = new PaimonCommitCodec();
83-
private final PaimonWriterLifecycle writerLifecycle = new PaimonWriterLifecycle();
78+
private final PaimonSdkWriter sdkWriter = new PaimonSdkWriter();
8479

8580
private String tableLocation;
8681
private PreExecutionAuthenticator preExecutionAuthenticator;
8782

8883
private PaimonWriteSchema writeSchema;
8984
private long commitIdentifier;
9085

91-
private static volatile Constructor<?> directByteBufferConstructor;
92-
9386
public PaimonJniWriter() {
9487
this.allocator = new RootAllocator(Long.MAX_VALUE);
9588
this.classLoader = this.getClass().getClassLoader();
@@ -131,7 +124,7 @@ public void open(String tableLocation, Map<String, String> options,
131124
options.getOrDefault(KEY_COMMIT_IDENTIFIER, "-1"));
132125

133126
this.writeSchema = PaimonWriteSchema.create(table.rowType(), columnNames);
134-
writerLifecycle.open(table, options, writeSchema.writeType());
127+
sdkWriter.open(table, options, writeSchema.writeType());
135128
return null;
136129
} catch (Throwable t) {
137130
throw new RuntimeException("PaimonJniWriter open failed: location="
@@ -143,13 +136,11 @@ public void open(String tableLocation, Map<String, String> options,
143136
/**
144137
* Write a batch of rows. Called once per Arrow IPC buffer from C++.
145138
*
146-
* @param address native memory address of the Arrow IPC Stream bytes
147-
* @param length number of bytes
139+
* @param directBuffer direct view of the native Arrow IPC Stream bytes
148140
*/
149-
public void write(long address, int length) throws Exception {
141+
public void write(ByteBuffer directBuffer) throws Exception {
150142
preExecutionAuthenticator.execute(() -> {
151143
try {
152-
ByteBuffer directBuffer = getDirectBuffer(address, length);
153144
try (ArrowStreamReader reader = new ArrowStreamReader(
154145
new DirectBufInputStream(directBuffer), allocator)) {
155146
VectorSchemaRoot root = reader.getVectorSchemaRoot();
@@ -159,8 +150,8 @@ public void write(long address, int length) throws Exception {
159150
}
160151
return null;
161152
} catch (Throwable t) {
162-
throw new RuntimeException("PaimonJniWriter write failed: addr="
163-
+ address + ", len=" + length, t);
153+
throw new RuntimeException("PaimonJniWriter write failed: bytes="
154+
+ directBuffer.capacity(), t);
164155
}
165156
});
166157
}
@@ -171,12 +162,9 @@ public void write(long address, int length) throws Exception {
171162
* @return byte[][] each element is a DPCM-framed serialized CommitMessage chunk
172163
*/
173164
public byte[][] prepareCommit() throws Exception {
174-
if (!writerLifecycle.isOpen()) {
175-
throw new IllegalStateException("Paimon writer is not open");
176-
}
177165
return preExecutionAuthenticator.execute(() -> {
178166
try {
179-
List<CommitMessage> messages = writerLifecycle.prepareCommit(commitIdentifier);
167+
List<CommitMessage> messages = sdkWriter.prepareCommit(commitIdentifier);
180168
if (messages.isEmpty()) {
181169
LOG.info("PaimonJniWriter prepareCommit: empty, location={}", tableLocation);
182170
return new byte[0][];
@@ -197,11 +185,11 @@ public void abort() throws Exception {
197185
try {
198186
if (preExecutionAuthenticator != null) {
199187
preExecutionAuthenticator.execute(() -> {
200-
writerLifecycle.close();
188+
sdkWriter.close();
201189
return null;
202190
});
203191
} else {
204-
writerLifecycle.close();
192+
sdkWriter.close();
205193
}
206194
} catch (Exception e) {
207195
LOG.error("PaimonJniWriter abort failed", e);
@@ -264,11 +252,9 @@ private void writeBatch(VectorSchemaRoot root) throws Exception {
264252
}
265253
int colCount = root.getFieldVectors().size();
266254
Object[][] columnValues = arrowConverter.convert(root, writeSchema.targetTypes());
267-
BatchTableWrite writer = writerLifecycle.writer();
268-
269255
// Group rows by (partition, bucket) using the Paimon SDK's routing methods,
270256
// then write each group via writeBundle for batch-level I/O optimization.
271-
Map<PartitionBucketKey, List<InternalRow>> groups = new LinkedHashMap<>();
257+
Map<PaimonSdkWriter.RoutingKey, List<InternalRow>> groups = new LinkedHashMap<>();
272258
for (int r = 0; r < rowCount; r++) {
273259
GenericRow row = new GenericRow(colCount);
274260
for (int c = 0; c < colCount; c++) {
@@ -277,15 +263,13 @@ private void writeBatch(VectorSchemaRoot root) throws Exception {
277263
// withWriteType controls the file row layout, but the SDK routing
278264
// extractors still read fields in table schema order.
279265
InternalRow routingRow = writeSchema.routingRow(row, columnValues, r);
280-
BinaryRow partition = writer.getPartition(routingRow);
281-
int bucket = writer.getBucket(routingRow);
282-
groups.computeIfAbsent(new PartitionBucketKey(partition, bucket),
266+
PaimonSdkWriter.RoutingKey routingKey = sdkWriter.route(routingRow);
267+
groups.computeIfAbsent(routingKey,
283268
k -> new ArrayList<>()).add(row);
284269
}
285270

286-
for (Map.Entry<PartitionBucketKey, List<InternalRow>> entry : groups.entrySet()) {
287-
PartitionBucketKey key = entry.getKey();
288-
writer.writeBundle(key.partition, key.bucket, new ListBundleRecords(entry.getValue()));
271+
for (Map.Entry<PaimonSdkWriter.RoutingKey, List<InternalRow>> entry : groups.entrySet()) {
272+
sdkWriter.writeBundle(entry.getKey(), entry.getValue());
289273
}
290274
}
291275

@@ -295,7 +279,7 @@ private void writeBatch(VectorSchemaRoot root) throws Exception {
295279

296280
private void closeResources() throws Exception {
297281
try {
298-
writerLifecycle.close();
282+
sdkWriter.close();
299283
} finally {
300284
writeSchema = null;
301285
allocator.close();
@@ -306,17 +290,6 @@ private void closeResources() throws Exception {
306290
// Utilities
307291
// ────────────────────────────────────────────────────────────
308292

309-
private static ByteBuffer getDirectBuffer(long address, int length) throws Exception {
310-
Constructor<?> ctor = directByteBufferConstructor;
311-
if (ctor == null) {
312-
Class<?> cls = Class.forName("java.nio.DirectByteBuffer");
313-
ctor = cls.getDeclaredConstructor(long.class, int.class);
314-
ctor.setAccessible(true);
315-
directByteBufferConstructor = ctor;
316-
}
317-
return (ByteBuffer) ctor.newInstance(address, length);
318-
}
319-
320293
private static Catalog createCatalog(Map<String, String> paimonOpts,
321294
Map<String, String> hadoopOpts) {
322295
Options options = new Options();
@@ -359,59 +332,6 @@ private static void configureLogLevels() {
359332
}
360333
}
361334

362-
// ────────────────────────────────────────────────────────────
363-
// Bundle helpers
364-
// ────────────────────────────────────────────────────────────
365-
366-
/** Compound key for grouping rows by (partition, bucket). */
367-
static final class PartitionBucketKey {
368-
final BinaryRow partition;
369-
final int bucket;
370-
371-
PartitionBucketKey(BinaryRow partition, int bucket) {
372-
// Paimon projections reuse their BinaryRow result across records.
373-
// Keep an immutable snapshot because this object is also a map key.
374-
this.partition = partition.copy();
375-
this.bucket = bucket;
376-
}
377-
378-
@Override
379-
public int hashCode() {
380-
return partition.hashCode() * 31 + bucket;
381-
}
382-
383-
@Override
384-
public boolean equals(Object obj) {
385-
if (this == obj) {
386-
return true;
387-
}
388-
if (!(obj instanceof PartitionBucketKey)) {
389-
return false;
390-
}
391-
PartitionBucketKey other = (PartitionBucketKey) obj;
392-
return bucket == other.bucket && partition.equals(other.partition);
393-
}
394-
}
395-
396-
/** BundleRecords backed by a List of pre-built InternalRows. */
397-
private static final class ListBundleRecords implements BundleRecords {
398-
private final List<InternalRow> rows;
399-
400-
ListBundleRecords(List<InternalRow> rows) {
401-
this.rows = rows;
402-
}
403-
404-
@Override
405-
public long rowCount() {
406-
return rows.size();
407-
}
408-
409-
@Override
410-
public Iterator<InternalRow> iterator() {
411-
return rows.iterator();
412-
}
413-
}
414-
415335
/** InputStream over a direct ByteBuffer (no copy). */
416336
private static class DirectBufInputStream extends InputStream {
417337
private final ByteBuffer buf;

0 commit comments

Comments
 (0)