Skip to content

Commit c05c7b5

Browse files
committed
[fix](be) Avoid duplicate JNI close profiles
### What problem does this PR solve? Issue Number: None Related PR: #65503 Problem Summary: A retryable JNI scanner close published the same split profile on every cleanup attempt because the Java scanner and watcher values must remain intact after a failed close. Reserve profile publication per split so close retries only repeat resource cleanup, then reset that publication state when the split is successfully released. ### Release note Keep JNI split profile counters idempotent across close retries. ### Check List (For Author) - Test: Unit Test - FileScannerV2Test.*, JniTableReaderTest.*, and PaimonJniReaderTest.*: 30 tests passed under ASAN BE UT - FailedCloseCanBeRetried asserts the close profile counter is unchanged after a fail-then-success retry - check-format and git diff checks passed - Behavior changed: Yes. Retried JNI cleanup no longer publishes the same split profile more than once. - Does this need documentation: No
1 parent 68655ae commit c05c7b5

3 files changed

Lines changed: 61 additions & 25 deletions

File tree

‎be/src/format_v2/jni/jni_table_reader.cpp‎

Lines changed: 41 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -294,33 +294,21 @@ int64_t JniTableReader::self_split_weight() const {
294294
return _current_range.__isset.self_split_weight ? _current_range.self_split_weight : -1;
295295
}
296296

297-
Status JniTableReader::close() {
298-
if (_closed) {
299-
return Status::OK();
297+
bool JniTableReader::_reserve_split_profile_publication() {
298+
if (_split_profile_published) {
299+
return false;
300300
}
301-
auto close_status = _close_jni_scanner();
302-
auto table_status = TableReader::close();
303-
if (close_status.ok() && !table_status.ok()) {
304-
close_status = std::move(table_status);
305-
}
306-
if (close_status.ok()) {
307-
_closed = true;
308-
}
309-
return close_status;
301+
_split_profile_published = true;
302+
return true;
310303
}
311304

312-
Status JniTableReader::_close_jni_scanner() {
313-
if (!_scanner_opened) {
314-
JNIEnv* env = nullptr;
315-
if (!_jni_scanner_obj.uninitialized()) {
316-
RETURN_IF_ERROR(Jni::Env::Get(&env));
317-
}
318-
_reset_split_state(env);
319-
return Status::OK();
305+
void JniTableReader::_publish_split_profile(JNIEnv* env) {
306+
// Cleanup can fail while the Java scanner and split watchers must remain available for a
307+
// retry. Reserve profile publication separately so a retry only repeats resource cleanup.
308+
if (!_reserve_split_profile_publication()) {
309+
return;
320310
}
321311

322-
JNIEnv* env = nullptr;
323-
RETURN_IF_ERROR(Jni::Env::Get(&env));
324312
if (_scanner_profile != nullptr) {
325313
COUNTER_UPDATE(_open_scanner_time, _jni_scanner_open_watcher);
326314
COUNTER_UPDATE(_fill_block_time, _fill_block_watcher);
@@ -352,6 +340,36 @@ Status JniTableReader::_close_jni_scanner() {
352340
self_split_weight());
353341
}
354342
_collect_jni_scanner_profile(env);
343+
}
344+
345+
Status JniTableReader::close() {
346+
if (_closed) {
347+
return Status::OK();
348+
}
349+
auto close_status = _close_jni_scanner();
350+
auto table_status = TableReader::close();
351+
if (close_status.ok() && !table_status.ok()) {
352+
close_status = std::move(table_status);
353+
}
354+
if (close_status.ok()) {
355+
_closed = true;
356+
}
357+
return close_status;
358+
}
359+
360+
Status JniTableReader::_close_jni_scanner() {
361+
if (!_scanner_opened) {
362+
JNIEnv* env = nullptr;
363+
if (!_jni_scanner_obj.uninitialized()) {
364+
RETURN_IF_ERROR(Jni::Env::Get(&env));
365+
}
366+
_reset_split_state(env);
367+
return Status::OK();
368+
}
369+
370+
JNIEnv* env = nullptr;
371+
RETURN_IF_ERROR(Jni::Env::Get(&env));
372+
_publish_split_profile(env);
355373

356374
// _fill_jni_block may fail before releasing the current Java table. JniScanner::releaseTable()
357375
// is idempotent, so closing the split always releases it. Java close must still run if that
@@ -380,6 +398,7 @@ void JniTableReader::_reset_split_state(JNIEnv* env) {
380398
_jni_scanner_open_watcher = 0;
381399
_java_scan_watcher = 0;
382400
_fill_block_watcher = 0;
401+
_split_profile_published = false;
383402
}
384403

385404
Status JniTableReader::_open_jni_scanner() {

‎be/src/format_v2/jni/jni_table_reader.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,9 @@ class JniTableReader : public TableReader {
5757
void TEST_set_split_state(bool scanner_opened, bool eof) {
5858
_scanner_opened = scanner_opened;
5959
_eof = eof;
60+
if (!scanner_opened) {
61+
_split_profile_published = false;
62+
}
6063
}
6164
bool TEST_scanner_opened() const { return _scanner_opened; }
6265
bool TEST_eof() const { return _eof; }
@@ -79,6 +82,7 @@ class JniTableReader : public TableReader {
7982
virtual Status _set_open_scanner_batch_size(size_t batch_size);
8083
virtual bool supports_batch_size_update_after_open() const { return true; }
8184
virtual Status _open_jni_scanner();
85+
bool _reserve_split_profile_publication();
8286
const std::vector<JniColumn>& jni_columns() const { return _jni_columns; }
8387
TFileRangeDesc _current_range;
8488

@@ -95,6 +99,7 @@ class JniTableReader : public TableReader {
9599
Status _fill_jni_block(JniDataBridge::TableMetaAddress& table_meta, size_t num_rows);
96100
Status _get_statistics(JNIEnv* env, std::map<std::string, std::string>* result);
97101
void _collect_jni_scanner_profile(JNIEnv* env);
102+
void _publish_split_profile(JNIEnv* env);
98103

99104
std::map<std::string, std::string> _scanner_params;
100105
std::vector<JniColumn> _jni_columns;
@@ -103,6 +108,7 @@ class JniTableReader : public TableReader {
103108
bool _closed = false;
104109
bool _scanner_opened = false;
105110
bool _eof = false;
111+
bool _split_profile_published = false;
106112

107113
RuntimeProfile::Counter* _open_scanner_time = nullptr;
108114
RuntimeProfile::Counter* _java_scan_time = nullptr;

‎be/test/format_v2/jni/jni_table_reader_test.cpp‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@ class FakeJniTableReader final : public JniTableReader {
3333
public:
3434
int get_next_calls = 0;
3535
int close_calls = 0;
36+
int64_t close_profile_delta = 0;
37+
RuntimeProfile::Counter* close_profile_counter = nullptr;
3638
std::vector<size_t> open_batch_sizes;
3739
std::vector<size_t> propagated_batch_sizes;
3840
std::vector<Status> close_results;
@@ -58,6 +60,9 @@ class FakeJniTableReader final : public JniTableReader {
5860
return Status::OK();
5961
}
6062
++close_calls;
63+
if (_reserve_split_profile_publication() && close_profile_counter != nullptr) {
64+
COUNTER_UPDATE(close_profile_counter, close_profile_delta);
65+
}
6166
Status status = Status::OK();
6267
if (static_cast<size_t>(close_calls) <= close_results.size()) {
6368
status = close_results[close_calls - 1];
@@ -80,15 +85,16 @@ class FakeJniTableReader final : public JniTableReader {
8085
}
8186
};
8287

83-
Status init_reader(FakeJniTableReader* reader, const std::shared_ptr<io::IOContext>& io_ctx) {
88+
Status init_reader(FakeJniTableReader* reader, const std::shared_ptr<io::IOContext>& io_ctx,
89+
RuntimeProfile* scanner_profile = nullptr) {
8490
return reader->init({
8591
.projected_columns = {},
8692
.conjuncts = {},
8793
.format = FileFormat::JNI,
8894
.scan_params = nullptr,
8995
.io_ctx = io_ctx,
9096
.runtime_state = nullptr,
91-
.scanner_profile = nullptr,
97+
.scanner_profile = scanner_profile,
9298
});
9399
}
94100

@@ -128,19 +134,24 @@ TEST(JniTableReaderTest, EndOfSplitRemainsIdempotentAfterScannerClose) {
128134
}
129135

130136
TEST(JniTableReaderTest, FailedCloseCanBeRetried) {
137+
RuntimeProfile profile("FailedCloseCanBeRetried");
131138
FakeJniTableReader reader;
132-
ASSERT_TRUE(init_reader(&reader, nullptr).ok());
139+
reader.close_profile_counter = profile.add_counter("PublishedCloseProfile", TUnit::UNIT);
140+
reader.close_profile_delta = 17;
141+
ASSERT_TRUE(init_reader(&reader, nullptr, &profile).ok());
133142
reader.TEST_set_split_state(true, false);
134143
reader.close_results = {Status::InternalError("injected close failure"), Status::OK()};
135144

136145
EXPECT_FALSE(reader.close().ok());
137146
EXPECT_FALSE(reader.TEST_closed());
138147
EXPECT_TRUE(reader.TEST_scanner_opened());
139148
EXPECT_EQ(reader.close_calls, 1);
149+
EXPECT_EQ(reader.close_profile_counter->value(), 17);
140150

141151
EXPECT_TRUE(reader.close().ok());
142152
EXPECT_TRUE(reader.TEST_closed());
143153
EXPECT_EQ(reader.close_calls, 2);
154+
EXPECT_EQ(reader.close_profile_counter->value(), 17);
144155
}
145156

146157
TEST(JniTableReaderTest, AdaptiveBatchSizeUpdatesAnOpenJavaScanner) {

0 commit comments

Comments
 (0)