Skip to content

Commit 92703b1

Browse files
committed
[refactor](paimon) Enforce writer lifecycle invariants
### What problem does this PR solve? Issue Number: close apache#65086 Related PR: apache#65381 Problem Summary: The Paimon write path treated missing writers and malformed commit payloads as normal empty work. This could silently drop writes or commit data after a lifecycle contract was violated. Enforce the successful-open writer invariant with assertions, fail invalid Java and JNI commit states explicitly, and retain nullable checks only for partial-open error cleanup. The writer constructors also move their owned sink descriptors instead of making redundant copies. ### Release note None ### Check List (For Author) - Test: Unit Test and incremental DEBUG build - PaimonJniWriterTest and PaimonWriterLifecycleTest: 4 tests passed - BUILD_TYPE=DEBUG ./build.sh --be: modified Paimon objects compiled and doris_be linked successfully - Behavior changed: Yes. Invalid writer lifecycle and malformed commit payload states now fail explicitly instead of being silently ignored. - Does this need documentation: No
1 parent 0fdefe8 commit 92703b1

6 files changed

Lines changed: 68 additions & 47 deletions

File tree

‎be/src/exec/sink/writer/paimon/jni_paimon_write_backend.cpp‎

Lines changed: 41 additions & 34 deletions
Original file line numberDiff line numberDiff line change
@@ -65,12 +65,11 @@ JniPaimonWriteBackend::~JniPaimonWriteBackend() {
6565
JNIEnv* env = nullptr;
6666
if (_get_jni_env(&env).ok()) {
6767
if (_jni_writer_obj != nullptr) {
68-
if (_close_id != nullptr) {
69-
env->CallVoidMethod(_jni_writer_obj, _close_id);
70-
Status close_status = _check_jni_exception(env, "close PaimonJniWriter");
71-
if (!close_status.ok()) {
72-
LOG(WARNING) << "Failed to close PaimonJniWriter: " << close_status.to_string();
73-
}
68+
DCHECK(_close_id != nullptr);
69+
env->CallVoidMethod(_jni_writer_obj, _close_id);
70+
Status close_status = _check_jni_exception(env, "close PaimonJniWriter");
71+
if (!close_status.ok()) {
72+
LOG(WARNING) << "Failed to close PaimonJniWriter: " << close_status.to_string();
7473
}
7574
env->DeleteGlobalRef(_jni_writer_obj);
7675
_jni_writer_obj = nullptr;
@@ -104,7 +103,7 @@ Status JniPaimonWriteBackend::_load_writer_class(JNIEnv* env, jclass* writer_cla
104103

105104
jobject loader = env->NewObject(loader_class, loader_constructor);
106105
jstring class_name = env->NewStringUTF(PAIMON_JNI_WRITER_CLASS);
107-
auto loaded_class =
106+
auto* loaded_class =
108107
static_cast<jclass>(env->CallObjectMethod(loader, get_loaded_class, class_name));
109108
RETURN_IF_ERROR(_check_jni_exception(env, "load PaimonJniWriter"));
110109

@@ -135,6 +134,8 @@ static jobject _to_java_options(JNIEnv* env, const std::map<std::string, std::st
135134

136135
Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* state) {
137136
_sink = sink;
137+
DORIS_CHECK(sink.__isset.column_names);
138+
DORIS_CHECK(sink.__isset.write_mode);
138139

139140
JNIEnv* env = nullptr;
140141
RETURN_IF_ERROR(_get_jni_env(&env));
@@ -177,7 +178,6 @@ Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* s
177178
if (sink.__isset.serialized_table && !sink.serialized_table.empty()) {
178179
jni_options["serialized_table"] = sink.serialized_table;
179180
}
180-
DORIS_CHECK(sink.__isset.write_mode);
181181
jni_options["doris.write_mode"] =
182182
sink.write_mode == TPaimonWriteMode::OVERWRITE ? "OVERWRITE" : "APPEND";
183183
if (sink.__isset.static_partition) {
@@ -191,17 +191,12 @@ Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* s
191191
jobject j_options = _to_java_options(env, jni_options);
192192

193193
jclass string_cls = env->FindClass("java/lang/String");
194-
jobjectArray j_cols = nullptr;
195-
if (sink.__isset.column_names && !sink.column_names.empty()) {
196-
j_cols = env->NewObjectArray(static_cast<jsize>(sink.column_names.size()), string_cls,
197-
nullptr);
198-
for (size_t i = 0; i < sink.column_names.size(); ++i) {
199-
jstring str = env->NewStringUTF(sink.column_names[i].c_str());
200-
env->SetObjectArrayElement(j_cols, static_cast<jsize>(i), str);
201-
env->DeleteLocalRef(str);
202-
}
203-
} else {
204-
j_cols = env->NewObjectArray(0, string_cls, nullptr);
194+
jobjectArray j_cols =
195+
env->NewObjectArray(static_cast<jsize>(sink.column_names.size()), string_cls, nullptr);
196+
for (size_t i = 0; i < sink.column_names.size(); ++i) {
197+
jstring str = env->NewStringUTF(sink.column_names[i].c_str());
198+
env->SetObjectArrayElement(j_cols, static_cast<jsize>(i), str);
199+
env->DeleteLocalRef(str);
205200
}
206201

207202
// Call Java open()
@@ -219,7 +214,10 @@ Status JniPaimonWriteBackend::open(const TPaimonTableSink& sink, RuntimeState* s
219214
return st;
220215
}
221216

222-
Status JniPaimonWriteBackend::create_writer(std::unique_ptr<IPaimonWriter>* writer) {
217+
// Writer creation stays non-const because the backend interface also supports future stateful FFI
218+
// implementations.
219+
Status JniPaimonWriteBackend::create_writer( // NOLINT(readability-make-member-function-const)
220+
std::unique_ptr<IPaimonWriter>* writer) {
223221
DORIS_CHECK(_opened);
224222
*writer = std::make_unique<JniPaimonWriter>(_jni_writer_obj, _write_id, _prepare_commit_id,
225223
_abort_id, std::make_unique<ArrowMemoryPool<>>(),
@@ -230,13 +228,13 @@ Status JniPaimonWriteBackend::create_writer(std::unique_ptr<IPaimonWriter>* writ
230228
JniPaimonWriter::JniPaimonWriter(jobject jni_writer_obj, jmethodID write_id,
231229
jmethodID prepare_commit_id, jmethodID abort_id,
232230
std::unique_ptr<ArrowMemoryPool<>> arrow_pool,
233-
const TPaimonTableSink& sink)
231+
TPaimonTableSink sink)
234232
: _jni_writer_obj(jni_writer_obj),
235233
_write_id(write_id),
236234
_prepare_commit_id(prepare_commit_id),
237235
_abort_id(abort_id),
238236
_arrow_pool(std::move(arrow_pool)),
239-
_sink(sink) {}
237+
_sink(std::move(sink)) {}
240238

241239
Status JniPaimonWriter::_write_projected_block(RuntimeState* state, Block& block) {
242240
if (block.rows() == 0) {
@@ -313,28 +311,37 @@ Status JniPaimonWriter::prepare_commit(std::vector<TPaimonCommitMessage>& messag
313311
}
314312

315313
if (j_payloads_obj == nullptr) {
316-
return Status::OK(); // no data written
314+
return Status::InternalError("PaimonJniWriter.prepareCommit returned null");
317315
}
318316

319317
auto* j_payloads = static_cast<jobjectArray>(j_payloads_obj);
320318
jsize num_payloads = env->GetArrayLength(j_payloads);
321319

322320
for (jsize i = 0; i < num_payloads; ++i) {
323-
jbyteArray j_bytes = static_cast<jbyteArray>(env->GetObjectArrayElement(j_payloads, i));
321+
auto j_bytes = static_cast<jbyteArray>(env->GetObjectArrayElement(j_payloads, i));
324322
if (j_bytes == nullptr) {
325-
continue;
323+
env->DeleteLocalRef(j_payloads);
324+
return Status::InternalError("PaimonJniWriter.prepareCommit returned a null payload");
326325
}
327326
jsize len = env->GetArrayLength(j_bytes);
328-
if (len > 0) {
329-
jbyte* bytes = env->GetByteArrayElements(j_bytes, nullptr);
330-
if (bytes != nullptr) {
331-
std::string payload(reinterpret_cast<char*>(bytes), static_cast<size_t>(len));
332-
TPaimonCommitMessage msg;
333-
msg.__set_payload(payload);
334-
messages.emplace_back(std::move(msg));
335-
env->ReleaseByteArrayElements(j_bytes, bytes, JNI_ABORT);
336-
}
327+
if (len == 0) {
328+
env->DeleteLocalRef(j_bytes);
329+
env->DeleteLocalRef(j_payloads);
330+
return Status::InternalError("PaimonJniWriter.prepareCommit returned an empty payload");
331+
}
332+
jbyte* bytes = env->GetByteArrayElements(j_bytes, nullptr);
333+
if (bytes == nullptr) {
334+
env->DeleteLocalRef(j_bytes);
335+
env->DeleteLocalRef(j_payloads);
336+
RETURN_IF_ERROR(Jni::Env::GetJniExceptionMsg(
337+
env, false, "JNI exception while reading Paimon commit payload: "));
338+
return Status::InternalError("Failed to read Paimon commit payload");
337339
}
340+
std::string payload(reinterpret_cast<char*>(bytes), static_cast<size_t>(len));
341+
TPaimonCommitMessage msg;
342+
msg.__set_payload(payload);
343+
messages.emplace_back(std::move(msg));
344+
env->ReleaseByteArrayElements(j_bytes, bytes, JNI_ABORT);
338345
env->DeleteLocalRef(j_bytes);
339346
}
340347
env->DeleteLocalRef(j_payloads);

‎be/src/exec/sink/writer/paimon/jni_paimon_write_backend.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -61,7 +61,7 @@ class JniPaimonWriter final : public IPaimonWriter {
6161
public:
6262
JniPaimonWriter(jobject jni_writer_obj, jmethodID write_id, jmethodID prepare_commit_id,
6363
jmethodID abort_id, std::unique_ptr<ArrowMemoryPool<>> arrow_pool,
64-
const TPaimonTableSink& sink);
64+
TPaimonTableSink sink);
6565

6666
Status write(RuntimeState* state, Block& block) override;
6767
Status prepare_commit(std::vector<TPaimonCommitMessage>& messages) override;

‎be/src/exec/sink/writer/paimon/vpaimon_table_writer.cpp‎

Lines changed: 13 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -17,17 +17,18 @@
1717

1818
#include "exec/sink/writer/paimon/vpaimon_table_writer.h"
1919

20+
#include "common/check.h"
2021
#include "common/logging.h"
2122
#include "core/block/block.h"
2223
#include "runtime/runtime_state.h"
2324

2425
namespace doris {
2526

26-
VPaimonTableWriter::VPaimonTableWriter(const TDataSink& t_sink,
27-
const VExprContextSPtrs& output_exprs,
27+
VPaimonTableWriter::VPaimonTableWriter(TDataSink t_sink, const VExprContextSPtrs& output_exprs,
2828
std::shared_ptr<Dependency> dep,
2929
std::shared_ptr<Dependency> fin_dep)
30-
: AsyncResultWriter(output_exprs, std::move(dep), std::move(fin_dep)), _t_sink(t_sink) {
30+
: AsyncResultWriter(output_exprs, std::move(dep), std::move(fin_dep)),
31+
_t_sink(std::move(t_sink)) {
3132
DCHECK(_t_sink.__isset.paimon_table_sink);
3233
}
3334

@@ -52,8 +53,10 @@ Status VPaimonTableWriter::open(RuntimeState* state, RuntimeProfile* profile) {
5253
SCOPED_TIMER(_open_timer);
5354

5455
RETURN_IF_ERROR(PaimonWriteBackendFactory::create(_t_sink.paimon_table_sink, &_backend));
56+
DCHECK(_backend);
5557
RETURN_IF_ERROR(_backend->open(_t_sink.paimon_table_sink, state));
5658
RETURN_IF_ERROR(_backend->create_writer(&_writer));
59+
DCHECK(_writer);
5760

5861
LOG(INFO) << "VPaimonTableWriter opened: table=" << _t_sink.paimon_table_sink.tb_name
5962
<< ", backend=" << static_cast<int>(_backend->type()) << ", writer_scope=local_state";
@@ -79,7 +82,8 @@ Status VPaimonTableWriter::write(RuntimeState* state, Block& block) {
7982
_state->update_num_rows_load_total(block.rows());
8083
_state->update_num_bytes_load_total(block.bytes());
8184

82-
if (_writer) {
85+
DCHECK(_writer);
86+
{
8387
SCOPED_TIMER(_file_store_write_timer);
8488
RETURN_IF_ERROR(_writer->write(_state, output_block));
8589
}
@@ -91,7 +95,8 @@ Status VPaimonTableWriter::close(Status status) {
9195
SCOPED_TIMER(_close_timer);
9296

9397
std::vector<TPaimonCommitMessage> messages;
94-
if (status.ok() && _writer) {
98+
if (status.ok()) {
99+
DCHECK(_writer);
95100
{
96101
SCOPED_TIMER(_prepare_commit_timer);
97102
Status prep_st = _writer->prepare_commit(messages);
@@ -103,10 +108,9 @@ Status VPaimonTableWriter::close(Status status) {
103108
if (status.ok()) {
104109
COUNTER_UPDATE(_commit_payload_count, static_cast<int64_t>(messages.size()));
105110
for (const auto& msg : messages) {
106-
if (msg.__isset.payload) {
107-
COUNTER_UPDATE(_commit_payload_bytes_counter,
108-
static_cast<int64_t>(msg.payload.size()));
109-
}
111+
DORIS_CHECK(msg.__isset.payload);
112+
COUNTER_UPDATE(_commit_payload_bytes_counter,
113+
static_cast<int64_t>(msg.payload.size()));
110114
}
111115

112116
if (!messages.empty()) {

‎be/src/exec/sink/writer/paimon/vpaimon_table_writer.h‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -63,7 +63,7 @@ class RuntimeState;
6363
/// → RPC to FE Coordinator
6464
class VPaimonTableWriter final : public AsyncResultWriter {
6565
public:
66-
VPaimonTableWriter(const TDataSink& t_sink, const VExprContextSPtrs& output_exprs,
66+
VPaimonTableWriter(TDataSink t_sink, const VExprContextSPtrs& output_exprs,
6767
std::shared_ptr<Dependency> dep, std::shared_ptr<Dependency> fin_dep);
6868

6969
~VPaimonTableWriter() override = default;

‎fe/be-java-extensions/paimon-connector/src/main/java/org/apache/doris/paimon/PaimonJniWriter.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -172,12 +172,12 @@ public void write(long address, int length) throws Exception {
172172
*/
173173
public byte[][] prepareCommit() throws Exception {
174174
if (!writerLifecycle.isOpen()) {
175-
return new byte[0][];
175+
throw new IllegalStateException("Paimon writer is not open");
176176
}
177177
return preExecutionAuthenticator.execute(() -> {
178178
try {
179179
List<CommitMessage> messages = writerLifecycle.prepareCommit(commitIdentifier);
180-
if (messages == null || messages.isEmpty()) {
180+
if (messages.isEmpty()) {
181181
LOG.info("PaimonJniWriter prepareCommit: empty, location={}", tableLocation);
182182
return new byte[0][];
183183
}

‎fe/be-java-extensions/paimon-connector/src/test/java/org/apache/doris/paimon/PaimonJniWriterTest.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,16 @@
2323

2424
public class PaimonJniWriterTest {
2525

26+
@Test
27+
public void testPrepareCommitBeforeOpenFails() throws Exception {
28+
PaimonJniWriter writer = new PaimonJniWriter();
29+
try {
30+
Assertions.assertThrows(IllegalStateException.class, writer::prepareCommit);
31+
} finally {
32+
writer.close();
33+
}
34+
}
35+
2636
@Test
2737
public void testPartitionBucketKeyCopiesReusedPartition() {
2838
BinaryRow reusedPartition = BinaryRow.singleColumn(1);

0 commit comments

Comments
 (0)