Skip to content

Commit 8eebab9

Browse files
committed
[fix](paimon) Adapt Paimon writes to current master APIs
Issue Number: close apache#65086 Related PR: apache#67395 Problem Summary: The forward-ported Paimon writer still loaded its concrete Java class through the removed scanner classloader API, while current master requires isolated JNI plugins to publish writer factories. Register the Paimon writer in the plugin registry, use the shared writer lifecycle for close, and keep the specialized Arrow write ABI on the concrete implementation. Also align the connector SPI surface/version and Paimon 1.4.2 benchmark constructor with current master. Forward-port Paimon table write support to master. - Test: Manual test - ./build.sh --fe --clean - build-support/check-format.sh - build-support/check-build-hygiene.sh - Behavior changed: Yes (Paimon JNI writes are loaded through the isolated plugin registry) - Does this need documentation: No
1 parent 3aae283 commit 8eebab9

9 files changed

Lines changed: 153 additions & 95 deletions

File tree

‎be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp‎

Lines changed: 36 additions & 62 deletions
Original file line numberDiff line numberDiff line change
@@ -41,6 +41,7 @@
4141
#include "runtime/runtime_state.h"
4242
#include "util/defer_op.h"
4343
#include "util/jni-util.h"
44+
#include "util/jni_plugin_registry.h"
4445
#include "util/pretty_printer.h"
4546

4647
namespace doris {
@@ -192,7 +193,6 @@ void retain_resources_after_failed_close(std::unique_ptr<PaimonJniMemoryManager>
192193
// JNI helpers — class loading
193194
// ────────────────────────────────────────────────────────────
194195

195-
static constexpr const char* PAIMON_JNI_WRITER_CLASS = "org/apache/doris/paimon/PaimonJniWriter";
196196
const char* const PAIMON_JNI_WRITER_OPEN_SIGNATURE =
197197
"(Ljava/lang/String;Ljava/util/Map;[Ljava/lang/String;JLjava/lang/String;ZZLjava/lang/"
198198
"String;JJJ)V";
@@ -213,7 +213,7 @@ JniPaimonWriteBackend::~JniPaimonWriteBackend() {
213213
}
214214

215215
Status JniPaimonWriteBackend::close() {
216-
if (_jni_writer_obj == nullptr && _jni_writer_cls == nullptr) {
216+
if (_jni_writer_obj.uninitialized()) {
217217
_memory_manager.reset();
218218
_arrow_schema.reset();
219219
_spill_session.reset();
@@ -224,38 +224,23 @@ Status JniPaimonWriteBackend::close() {
224224
JNIEnv* env = nullptr;
225225
Status env_status = Jni::Env::Get(&env);
226226
if (!env_status.ok()) {
227-
bool java_users_may_exist = _jni_writer_obj != nullptr;
228-
// JNI global references cannot be released without an environment.
229-
// Deliberately abandon the handles so the Java writer remains alive.
230-
_jni_writer_obj = nullptr;
231-
_jni_writer_cls = nullptr;
232-
if (java_users_may_exist) {
233-
retain_resources_after_failed_close(std::move(_memory_manager),
234-
std::move(_spill_session));
235-
} else {
236-
_memory_manager.reset();
237-
_spill_session.reset();
238-
}
227+
// JNI global references cannot be released without an environment. Leave the wrapper
228+
// intact so its destructor can release the reference if the thread can attach later.
229+
retain_resources_after_failed_close(std::move(_memory_manager), std::move(_spill_session));
239230
_arrow_schema.reset();
240231
_opened = false;
241232
return env_status;
242233
}
243234

244235
Status close_status = Status::OK();
245-
if (_jni_writer_obj != nullptr) {
236+
if (!_jni_writer_obj.uninitialized()) {
246237
_refresh_memory_profile();
247-
if (_close_id == nullptr) {
248-
close_status = Status::InternalError("PaimonJniWriter.close method is unavailable");
238+
if (_writer_api == nullptr) {
239+
close_status = Status::InternalError("Paimon JniWriter API is unavailable");
249240
} else {
250-
env->CallVoidMethod(_jni_writer_obj, _close_id);
251-
close_status = _check_jni_exception(env, "close PaimonJniWriter");
241+
close_status = _jni_writer_obj.call_void_method(env, _writer_api->close).call();
252242
}
253-
env->DeleteGlobalRef(_jni_writer_obj);
254-
_jni_writer_obj = nullptr;
255-
}
256-
if (_jni_writer_cls != nullptr) {
257-
env->DeleteGlobalRef(_jni_writer_cls);
258-
_jni_writer_cls = nullptr;
243+
_jni_writer_obj.reset(env);
259244
}
260245

261246
if (close_status.ok()) {
@@ -349,40 +334,28 @@ Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* s
349334
}
350335
Defer pop_local_frame([&]() { env->PopLocalFrame(nullptr); });
351336

352-
// Step 1: Load PaimonJniWriter class through ScannerLoader (Paimon jars are
353-
// not on the default application classpath, so FindClass won't work).
354-
Jni::LocalObject local_writer_class;
355-
RETURN_IF_ERROR(
356-
Jni::Util::get_jni_scanner_class(env, PAIMON_JNI_WRITER_CLASS, &local_writer_class));
357-
auto writer_class = static_cast<jclass>(local_writer_class.get());
358-
_jni_writer_cls = static_cast<jclass>(env->NewGlobalRef(writer_class));
359-
RETURN_IF_ERROR(_check_jni_exception(env, "create global PaimonJniWriter class reference"));
360-
if (_jni_writer_cls == nullptr) {
361-
return Status::JniError("Failed to create global PaimonJniWriter class reference");
362-
}
363-
RETURN_IF_ERROR(PaimonJniMemoryManager::register_natives(env, _jni_writer_cls));
364-
RETURN_IF_ERROR(register_paimon_spill_natives(env, _jni_writer_cls));
365-
366-
// Step 2: Cache JNI method IDs for write, prepareCommit, abort, close.
367-
jmethodID open_id = env->GetMethodID(_jni_writer_cls, "open", PAIMON_JNI_WRITER_OPEN_SIGNATURE);
368-
jmethodID get_arrow_schema_id = env->GetMethodID(_jni_writer_cls, "getArrowSchema", "()[B");
369-
_write_id = env->GetMethodID(_jni_writer_cls, "writeArrow", "(JJ)V");
370-
_prepare_commit_id = env->GetMethodID(_jni_writer_cls, "prepareCommit", "()[[B");
371-
_abort_id = env->GetMethodID(_jni_writer_cls, "abort", "()V");
372-
_close_id = env->GetMethodID(_jni_writer_cls, "close", "()V");
337+
// Step 1: Create the writer through the isolated Paimon plugin. Asking the returned object for
338+
// its class is the only valid way to reach a concrete class owned by a plugin classloader.
339+
const std::map<std::string, std::string> writer_params;
340+
RETURN_IF_ERROR(Jni::PluginRegistry::create_writer(env, Jni::plugin::PAIMON_WRITER,
341+
state->batch_size(), writer_params,
342+
&_jni_writer_obj, &_writer_api));
343+
Jni::LocalClass writer_class;
344+
RETURN_IF_ERROR(Jni::Util::get_object_class(env, _jni_writer_obj, &writer_class));
345+
auto raw_writer_class = static_cast<jclass>(writer_class.get());
346+
RETURN_IF_ERROR(PaimonJniMemoryManager::register_natives(env, raw_writer_class));
347+
RETURN_IF_ERROR(register_paimon_spill_natives(env, raw_writer_class));
348+
349+
// Step 2: Cache the Paimon-specific method IDs. The common close method comes from WriterApi.
350+
jmethodID open_id =
351+
env->GetMethodID(raw_writer_class, "open", PAIMON_JNI_WRITER_OPEN_SIGNATURE);
352+
jmethodID get_arrow_schema_id = env->GetMethodID(raw_writer_class, "getArrowSchema", "()[B");
353+
_write_id = env->GetMethodID(raw_writer_class, "writeArrow", "(JJ)V");
354+
_prepare_commit_id = env->GetMethodID(raw_writer_class, "prepareCommit", "()[[B");
355+
_abort_id = env->GetMethodID(raw_writer_class, "abort", "()V");
373356
RETURN_IF_ERROR(_check_jni_exception(env, "resolve PaimonJniWriter methods"));
374357

375-
// Step 3: Create the Java PaimonJniWriter instance.
376-
jmethodID ctor_id = env->GetMethodID(_jni_writer_cls, "<init>", "()V");
377-
jobject local_obj = env->NewObject(_jni_writer_cls, ctor_id);
378-
RETURN_IF_ERROR(_check_jni_exception(env, "create PaimonJniWriter"));
379-
_jni_writer_obj = env->NewGlobalRef(local_obj);
380-
RETURN_IF_ERROR(_check_jni_exception(env, "create global PaimonJniWriter object reference"));
381-
if (_jni_writer_obj == nullptr) {
382-
return Status::JniError("Failed to create global PaimonJniWriter object reference");
383-
}
384-
385-
// Step 4: Create a lazy query-scoped spill session. Java requests its path only when Paimon
358+
// Step 3: Create a lazy query-scoped spill session. Java requests its path only when Paimon
386359
// first uses the IOManager, so a memory-only writer does not depend on spill storage.
387360
auto* spill_file_manager = state->exec_env()->spill_file_mgr();
388361
if (spill_file_manager != nullptr) {
@@ -392,7 +365,7 @@ Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* s
392365
spill_relative_path, state->get_query_ctx(), &_spill_session));
393366
}
394367

395-
// Step 5: Build Java arguments and call PaimonJniWriter.open().
368+
// Step 4: Build Java arguments and call PaimonJniWriter.open().
396369
const std::map<std::string, std::string> empty_config;
397370
jstring j_serialized_table = env->NewStringUTF(sink.serialized_table.c_str());
398371
Jni::LocalObject j_hadoop_config;
@@ -413,15 +386,16 @@ Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* s
413386

414387
PaimonJniWriterOpenMode open_mode = PaimonJniWriterOpenMode::from_write_mode(sink.write_mode);
415388
env->CallVoidMethod(
416-
_jni_writer_obj, open_id, j_serialized_table, j_hadoop_config.get(), j_cols,
389+
_jni_writer_obj.get(), open_id, j_serialized_table, j_hadoop_config.get(), j_cols,
417390
static_cast<jlong>(sink.transaction_id), j_commit_user, open_mode.overwrite,
418391
open_mode.changelog, j_time_zone, static_cast<jlong>(_memory_manager->memory_limit()),
419392
reinterpret_cast<jlong>(_memory_manager.get()),
420393
_spill_session == nullptr ? 0 : reinterpret_cast<jlong>(_spill_session.get()));
421394
Status st = _check_jni_exception(env, "open PaimonJniWriter");
422395

423396
if (st.ok()) {
424-
st = _get_paimon_arrow_schema(env, _jni_writer_obj, get_arrow_schema_id, &_arrow_schema);
397+
st = _get_paimon_arrow_schema(env, _jni_writer_obj.get(), get_arrow_schema_id,
398+
&_arrow_schema);
425399
}
426400
if (st.ok()) {
427401
_opened = true;
@@ -439,8 +413,8 @@ Status JniPaimonWriteBackend::create_writer( // NOLINT(readability-make-member-f
439413
std::unique_ptr<IPaimonWriter>* writer) {
440414
DORIS_CHECK(_opened);
441415
DORIS_CHECK(_arrow_schema != nullptr);
442-
*writer = std::make_unique<JniPaimonWriter>(_jni_writer_obj, _write_id, _prepare_commit_id,
443-
_abort_id, _arrow_schema);
416+
*writer = std::make_unique<JniPaimonWriter>(_jni_writer_obj.get(), _write_id,
417+
_prepare_commit_id, _abort_id, _arrow_schema);
444418
return Status::OK();
445419
}
446420

‎be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
#include "exec/sink/writer/paimon/paimon_write_backend.h"
2929
#include "format/parquet/arrow_memory_pool.h"
3030
#include "runtime/runtime_profile.h"
31+
#include "util/jni_plugin_registry.h"
3132

3233
namespace arrow {
3334
class Schema;
@@ -71,15 +72,15 @@ class JniPaimonWriteBackend final : public IPaimonWriteBackend {
7172
Status _check_jni_exception(JNIEnv* env, const std::string& method_name);
7273
void _refresh_memory_profile();
7374

74-
// JNI global references — live for the duration of this backend.
75-
jclass _jni_writer_cls = nullptr;
76-
jobject _jni_writer_obj = nullptr;
75+
// JNI global reference — lives for the duration of this backend. The method ids on the
76+
// shared SPI base class are owned by PluginRegistry.
77+
Jni::GlobalObject _jni_writer_obj;
78+
const Jni::WriterApi* _writer_api = nullptr;
7779

7880
// Cached JNI method IDs for the PaimonJniWriter Java methods.
7981
jmethodID _write_id = nullptr;
8082
jmethodID _prepare_commit_id = nullptr;
8183
jmethodID _abort_id = nullptr;
82-
jmethodID _close_id = nullptr;
8384

8485
std::unique_ptr<PaimonJniMemoryManager> _memory_manager;
8586
std::shared_ptr<arrow::Schema> _arrow_schema;

‎be/src/util/jni_plugin_registry.h‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -61,6 +61,7 @@ struct PluginRef {
6161
namespace plugin {
6262

6363
inline constexpr PluginRef PAIMON_SCANNER {"paimon", "reader"};
64+
inline constexpr PluginRef PAIMON_WRITER {"paimon", "writer"};
6465
inline constexpr PluginRef HUDI_SCANNER {"hudi", "reader"};
6566
// Only the Iceberg system tables come through Java; Iceberg data files are read natively - so
6667
// this one keeps its own name when it is migrated. It is not the connector's reader.

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java‎

Lines changed: 45 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,9 @@
1717

1818
package org.apache.doris.paimon;
1919

20-
import org.apache.doris.common.classloader.ThreadClassLoaderContext;
20+
import org.apache.doris.jni.spi.JniWriter;
21+
import org.apache.doris.jni.spi.ThreadContextClassLoader;
22+
import org.apache.doris.jni.spi.vec.VectorTable;
2123
import org.apache.doris.kerberos.PreExecutionAuthenticator;
2224
import org.apache.doris.kerberos.PreExecutionAuthenticatorCache;
2325

@@ -87,13 +89,12 @@
8789
* → C++ collects TPaimonCommitMessage[] → RPC to FE → PaimonTransaction
8890
* </pre>
8991
*/
90-
public class PaimonJniWriter {
92+
public class PaimonJniWriter extends JniWriter {
9193
private static final Logger LOG = LoggerFactory.getLogger(PaimonJniWriter.class);
9294
private static final int APPEND_ONLY_WRITER_MIN_PAGES = 1;
9395
private static final int MERGE_TREE_WRITER_MIN_PAGES = 3;
9496
private static final long COMPACTION_CLOSE_TIMEOUT_SECONDS = 60;
9597

96-
private final ClassLoader classLoader;
9798
private final PaimonCommitCodec commitCodec = new PaimonCommitCodec();
9899

99100
private BufferAllocator allocator;
@@ -117,11 +118,15 @@ public class PaimonJniWriter {
117118
private boolean sdkCloseFailed;
118119

119120
public PaimonJniWriter() {
121+
this(0, Collections.emptyMap());
122+
}
123+
124+
public PaimonJniWriter(int batchSize, Map<String, String> params) {
125+
super(batchSize, params);
120126
// Imported C Data vectors reference Doris-owned buffers; this allocator owns only Arrow's
121127
// Java-side views and metadata. Physical buffers remain charged to the C++ Arrow memory
122128
// pool until the synchronous writeArrow call returns.
123129
this.allocator = new RootAllocator(Long.MAX_VALUE);
124-
this.classLoader = this.getClass().getClassLoader();
125130
}
126131

127132
// ────────────────────────────────────────────────────────────
@@ -156,7 +161,8 @@ public void open(String serializedTable, Map<String, String> hadoopConfig,
156161
boolean overwrite, boolean changelogWrite, String timeZone,
157162
long nativePageMemoryLimitBytes, long nativeMemoryManager,
158163
long nativeSpillSession) throws Exception {
159-
try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) {
164+
try (ThreadContextClassLoader ignored =
165+
new ThreadContextClassLoader(getClass().getClassLoader())) {
160166
if (nativePageMemoryLimitBytes <= 0) {
161167
throw new IllegalArgumentException(
162168
"PaimonJniWriter requires a positive native page memory limit");
@@ -224,7 +230,8 @@ public byte[] getArrowSchema() {
224230
* has consumed every row; the native caller releases only callbacks left by a partial import.
225231
*/
226232
public void writeArrow(long arrayAddress, long schemaAddress) throws Exception {
227-
try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) {
233+
try (ThreadContextClassLoader ignored =
234+
new ThreadContextClassLoader(getClass().getClassLoader())) {
228235
preExecutionAuthenticator.execute(() -> {
229236
try (ArrowArray array = ArrowArray.wrap(arrayAddress);
230237
ArrowSchema schema = ArrowSchema.wrap(schemaAddress);
@@ -251,7 +258,8 @@ public void writeArrow(long arrayAddress, long schemaAddress) throws Exception {
251258
* @return byte[][] each element is a DPCM-framed serialized CommitMessage chunk
252259
*/
253260
public byte[][] prepareCommit() throws Exception {
254-
try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) {
261+
try (ThreadContextClassLoader ignored =
262+
new ThreadContextClassLoader(getClass().getClassLoader())) {
255263
return preExecutionAuthenticator.execute(() -> {
256264
try {
257265
List<CommitMessage> messages = prepareCommitMessages();
@@ -273,7 +281,8 @@ public byte[][] prepareCommit() throws Exception {
273281
* Called from C++ when write or prepareCommit fails.
274282
*/
275283
public void abort() throws Exception {
276-
try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) {
284+
try (ThreadContextClassLoader ignored =
285+
new ThreadContextClassLoader(getClass().getClassLoader())) {
277286
try {
278287
if (preExecutionAuthenticator != null) {
279288
preExecutionAuthenticator.execute(() -> {
@@ -290,24 +299,36 @@ public void abort() throws Exception {
290299
}
291300
}
292301

293-
/**
294-
* Close: release all resources.
295-
*/
296-
public void close() throws Exception {
297-
try (ThreadClassLoaderContext ignored = new ThreadClassLoaderContext(classLoader)) {
298-
try {
299-
if (preExecutionAuthenticator != null) {
300-
preExecutionAuthenticator.execute(() -> {
301-
closeResources();
302-
return null;
303-
});
304-
} else {
302+
@Override
303+
protected void openInternal() {
304+
throw new UnsupportedOperationException(
305+
"Paimon writer requires its Arrow-specific open entry point");
306+
}
307+
308+
@Override
309+
protected void writeInternal(VectorTable inputTable) {
310+
throw new UnsupportedOperationException(
311+
"Paimon writer requires its Arrow C Data write entry point");
312+
}
313+
314+
/** Close: release all resources through the shared JNI writer lifecycle. */
315+
@Override
316+
protected void closeInternal() throws IOException {
317+
try {
318+
if (preExecutionAuthenticator != null) {
319+
preExecutionAuthenticator.execute(() -> {
305320
closeResources();
306-
}
307-
} catch (Exception e) {
308-
LOG.warn("PaimonJniWriter close error", e);
309-
throw e;
321+
return null;
322+
});
323+
} else {
324+
closeResources();
325+
}
326+
} catch (Exception e) {
327+
LOG.warn("PaimonJniWriter close error", e);
328+
if (e instanceof IOException) {
329+
throw (IOException) e;
310330
}
331+
throw new IOException("PaimonJniWriter close failed", e);
311332
}
312333
}
313334

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonPlugin.java‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,19 +19,25 @@
1919

2020
import org.apache.doris.jni.spi.DorisPlugin;
2121
import org.apache.doris.jni.spi.JniScannerFactory;
22+
import org.apache.doris.jni.spi.JniWriterFactory;
2223

2324
import java.util.Collections;
2425

2526
/**
2627
* Entry point of the {@code paimon} plugin, found through
2728
* {@code META-INF/services/org.apache.doris.jni.spi.DorisPlugin}.
2829
*
29-
* <p>Reading only: Paimon writes go through Doris's own native writer, not through JNI.
30+
* <p>Publishes the isolated JNI implementations used by both Paimon reads and writes.
3031
*/
3132
public class PaimonPlugin implements DorisPlugin {
3233

3334
@Override
3435
public Iterable<JniScannerFactory> getScannerFactories() {
3536
return Collections.singletonList(new PaimonScannerFactory());
3637
}
38+
39+
@Override
40+
public Iterable<JniWriterFactory> getWriterFactories() {
41+
return Collections.singletonList(new PaimonWriterFactory());
42+
}
3743
}

0 commit comments

Comments
 (0)