Skip to content

Commit a08d695

Browse files
committed
[improvement](paimon) Fix Paimon JNI profile snapshot aggregation
### What problem does this PR solve? Issue Number: None Related PR: None Problem Summary: Paimon JNI scanner statistics include both scanner-local cumulative values and process-wide snapshot values such as active scanner counts, async reader thread counts, and JVM memory usage. Exporting all of them through the additive JNI profile counter path made multi-split scans multiply snapshot values by the number of collected splits. This change extends the JNI statistics profile collectors to support non-additive gauge and peak metric types, then reports process-wide snapshots as set/max metrics while keeping scanner-local rows, calls, encoded lengths, and timers additive. ### Release note None ### Check List (For Author) - Test: Unit Test - JAVA_HOME=/usr/local/opt/openjdk@17/libexec/openjdk.jdk/Contents/Home mvn test -pl be-java-extensions/paimon-scanner -am -Dtest=PaimonJniScannerTest -DfailIfNoTests=false -Ddoris.thrift.executable=/usr/local/bin/thrift - python3 build-support/run_clang_format.py --clang-format-executable $(brew --prefix llvm@16)/bin/clang-format --style file --inplace true be/src/format/jni/jni_reader.cpp be/src/format_v2/jni/jni_table_reader.cpp be/src/format_v2/jni/jni_table_reader.h - git diff --check - Behavior changed: No - Does this need documentation: No
1 parent 61eb6bf commit a08d695

4 files changed

Lines changed: 89 additions & 30 deletions

File tree

‎be/src/format/jni/jni_reader.cpp‎

Lines changed: 33 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -424,29 +424,59 @@ void JniReader::_collect_profile_before_close() {
424424
return;
425425
}
426426

427+
const auto update_peak = [](int64_t previous, int64_t current) {
428+
return current > previous;
429+
};
427430
for (const auto& metric : statistics_result) {
428431
std::vector<std::string> type_and_name = split(metric.first, ":");
429432
if (type_and_name.size() != 2) {
430433
LOG(WARNING) << "Name of JNI Scanner metric should be pattern like "
431434
<< "'metricType:metricName'";
432435
continue;
433436
}
434-
long metric_value = std::stol(metric.second);
437+
int64_t metric_value = std::stoll(metric.second);
435438
RuntimeProfile::Counter* scanner_counter;
436439
if (type_and_name[0] == "timer") {
437440
scanner_counter =
438441
ADD_CHILD_TIMER(_profile, type_and_name[1], _connector_name.c_str());
442+
COUNTER_UPDATE(scanner_counter, metric_value);
439443
} else if (type_and_name[0] == "counter") {
440444
scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::UNIT,
441445
_connector_name.c_str());
446+
COUNTER_UPDATE(scanner_counter, metric_value);
442447
} else if (type_and_name[0] == "bytes") {
443448
scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::BYTES,
444449
_connector_name.c_str());
450+
COUNTER_UPDATE(scanner_counter, metric_value);
451+
} else if (type_and_name[0] == "timer_gauge") {
452+
scanner_counter =
453+
ADD_CHILD_TIMER(_profile, type_and_name[1], _connector_name.c_str());
454+
COUNTER_SET(scanner_counter, metric_value);
455+
} else if (type_and_name[0] == "gauge") {
456+
scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::UNIT,
457+
_connector_name.c_str());
458+
COUNTER_SET(scanner_counter, metric_value);
459+
} else if (type_and_name[0] == "bytes_gauge") {
460+
scanner_counter = ADD_CHILD_COUNTER(_profile, type_and_name[1], TUnit::BYTES,
461+
_connector_name.c_str());
462+
COUNTER_SET(scanner_counter, metric_value);
463+
} else if (type_and_name[0] == "timer_peak") {
464+
auto* scanner_peak_counter = _profile->add_conditition_counter(
465+
type_and_name[1], TUnit::TIME_NS, update_peak, _connector_name.c_str());
466+
scanner_peak_counter->conditional_update(metric_value, metric_value);
467+
} else if (type_and_name[0] == "peak") {
468+
auto* scanner_peak_counter = _profile->add_conditition_counter(
469+
type_and_name[1], TUnit::UNIT, update_peak, _connector_name.c_str());
470+
scanner_peak_counter->conditional_update(metric_value, metric_value);
471+
} else if (type_and_name[0] == "bytes_peak") {
472+
auto* scanner_peak_counter = _profile->add_conditition_counter(
473+
type_and_name[1], TUnit::BYTES, update_peak, _connector_name.c_str());
474+
scanner_peak_counter->conditional_update(metric_value, metric_value);
445475
} else {
446-
LOG(WARNING) << "Type of JNI Scanner metric should be timer, counter or bytes";
476+
LOG(WARNING) << "Type of JNI Scanner metric should be timer, counter, bytes, "
477+
<< "timer_gauge, gauge, bytes_gauge, timer_peak, peak or bytes_peak";
447478
continue;
448479
}
449-
COUNTER_UPDATE(scanner_counter, metric_value);
450480
}
451481
}
452482
}

‎be/src/format_v2/jni/jni_table_reader.cpp‎

Lines changed: 31 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -200,29 +200,57 @@ void JniTableReader::_collect_jni_scanner_profile(JNIEnv* env) {
200200
}
201201

202202
const auto connector_name = _connector_name();
203+
const auto update_peak = [](int64_t previous, int64_t current) { return current > previous; };
203204
for (const auto& metric : statistics_result) {
204205
std::vector<std::string> type_and_name = split(metric.first, ":");
205206
if (type_and_name.size() != 2) {
206207
LOG(WARNING) << "Name of JNI Scanner metric should be pattern like "
207208
<< "'metricType:metricName'";
208209
continue;
209210
}
210-
long metric_value = std::stol(metric.second);
211+
int64_t metric_value = std::stoll(metric.second);
211212
RuntimeProfile::Counter* scanner_counter;
212213
if (type_and_name[0] == "timer") {
213214
scanner_counter =
214215
ADD_CHILD_TIMER(_scanner_profile, type_and_name[1], connector_name.c_str());
216+
COUNTER_UPDATE(scanner_counter, metric_value);
215217
} else if (type_and_name[0] == "counter") {
216218
scanner_counter = ADD_CHILD_COUNTER(_scanner_profile, type_and_name[1], TUnit::UNIT,
217219
connector_name.c_str());
220+
COUNTER_UPDATE(scanner_counter, metric_value);
218221
} else if (type_and_name[0] == "bytes") {
219222
scanner_counter = ADD_CHILD_COUNTER(_scanner_profile, type_and_name[1], TUnit::BYTES,
220223
connector_name.c_str());
224+
COUNTER_UPDATE(scanner_counter, metric_value);
225+
} else if (type_and_name[0] == "timer_gauge") {
226+
scanner_counter =
227+
ADD_CHILD_TIMER(_scanner_profile, type_and_name[1], connector_name.c_str());
228+
COUNTER_SET(scanner_counter, metric_value);
229+
} else if (type_and_name[0] == "gauge") {
230+
scanner_counter = ADD_CHILD_COUNTER(_scanner_profile, type_and_name[1], TUnit::UNIT,
231+
connector_name.c_str());
232+
COUNTER_SET(scanner_counter, metric_value);
233+
} else if (type_and_name[0] == "bytes_gauge") {
234+
scanner_counter = ADD_CHILD_COUNTER(_scanner_profile, type_and_name[1], TUnit::BYTES,
235+
connector_name.c_str());
236+
COUNTER_SET(scanner_counter, metric_value);
237+
} else if (type_and_name[0] == "timer_peak") {
238+
auto* scanner_peak_counter = _scanner_profile->add_conditition_counter(
239+
type_and_name[1], TUnit::TIME_NS, update_peak, connector_name.c_str());
240+
scanner_peak_counter->conditional_update(metric_value, metric_value);
241+
} else if (type_and_name[0] == "peak") {
242+
auto* scanner_peak_counter = _scanner_profile->add_conditition_counter(
243+
type_and_name[1], TUnit::UNIT, update_peak, connector_name.c_str());
244+
scanner_peak_counter->conditional_update(metric_value, metric_value);
245+
} else if (type_and_name[0] == "bytes_peak") {
246+
auto* scanner_peak_counter = _scanner_profile->add_conditition_counter(
247+
type_and_name[1], TUnit::BYTES, update_peak, connector_name.c_str());
248+
scanner_peak_counter->conditional_update(metric_value, metric_value);
221249
} else {
222-
LOG(WARNING) << "Type of JNI Scanner metric should be timer, counter or bytes";
250+
LOG(WARNING) << "Type of JNI Scanner metric should be timer, counter, bytes, "
251+
<< "timer_gauge, gauge, bytes_gauge, timer_peak, peak or bytes_peak";
223252
continue;
224253
}
225-
COUNTER_UPDATE(scanner_counter, metric_value);
226254
}
227255
}
228256

‎fe/be-java-extensions/paimon-scanner/src/main/java/org/apache/doris/paimon/PaimonJniScanner.java‎

Lines changed: 16 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -405,20 +405,20 @@ protected TableSchema parseTableSchema() throws UnsupportedOperationException {
405405
@Override
406406
public Map<String, String> getStatistics() {
407407
Map<String, String> statistics = new HashMap<>();
408-
statistics.put("counter:PaimonJniIOManagerEnabled", ioManager != null ? "1" : "0");
409-
statistics.put("counter:PaimonJniActiveScannerCount", String.valueOf(ACTIVE_SCANNERS.get()));
410-
statistics.put("counter:PaimonJniActiveScannerPeakCount", String.valueOf(PEAK_ACTIVE_SCANNERS.get()));
411-
statistics.put("counter:PaimonJniAsyncReaderThreadCount",
408+
statistics.put("gauge:PaimonJniIOManagerEnabled", ioManager != null ? "1" : "0");
409+
statistics.put("gauge:PaimonJniActiveScannerCount", String.valueOf(ACTIVE_SCANNERS.get()));
410+
statistics.put("peak:PaimonJniActiveScannerPeakCount", String.valueOf(PEAK_ACTIVE_SCANNERS.get()));
411+
statistics.put("gauge:PaimonJniAsyncReaderThreadCount",
412412
String.valueOf(currentAsyncReaderThreadCount()));
413-
statistics.put("counter:PaimonJniAsyncReaderThreadPeakCount",
413+
statistics.put("peak:PaimonJniAsyncReaderThreadPeakCount",
414414
String.valueOf(PEAK_ASYNC_READER_THREADS.get()));
415-
statistics.put("counter:PaimonJniRequiredFieldCount", String.valueOf(fields.length));
415+
statistics.put("gauge:PaimonJniRequiredFieldCount", String.valueOf(fields.length));
416416
statistics.put("counter:PaimonJniSplitEncodedLength", String.valueOf(lengthOfParam("paimon_split")));
417417
statistics.put("counter:PaimonJniPredicateEncodedLength", String.valueOf(lengthOfParam("paimon_predicate")));
418-
statistics.put("counter:PaimonJniAsyncThresholdConfigured",
418+
statistics.put("gauge:PaimonJniAsyncThresholdConfigured",
419419
hasPaimonOption(FILE_READER_ASYNC_THRESHOLD) ? "1" : "0");
420-
parseDataSizeBytes(paimonOption(FILE_READER_ASYNC_THRESHOLD))
421-
.ifPresent(bytes -> statistics.put("bytes:PaimonJniAsyncThresholdBytes", String.valueOf(bytes)));
420+
parseDataSizeBytes(paimonOption(FILE_READER_ASYNC_THRESHOLD)).ifPresent(
421+
bytes -> statistics.put("bytes_gauge:PaimonJniAsyncThresholdBytes", String.valueOf(bytes)));
422422
statistics.put("counter:PaimonJniReadBatchCalls", String.valueOf(readBatchCalls));
423423
statistics.put("counter:PaimonJniEmptyReadBatchCalls", String.valueOf(emptyReadBatchCalls));
424424
statistics.put("counter:PaimonJniRowsRead", String.valueOf(rowsRead));
@@ -451,12 +451,13 @@ private static void putMemoryStatistics(Map<String, String> statistics) {
451451
MemoryMXBean memoryMXBean = ManagementFactory.getMemoryMXBean();
452452
MemoryUsage heapUsage = memoryMXBean.getHeapMemoryUsage();
453453
MemoryUsage nonHeapUsage = memoryMXBean.getNonHeapMemoryUsage();
454-
statistics.put("bytes:PaimonJniJvmHeapUsed", String.valueOf(nonNegative(heapUsage.getUsed())));
455-
statistics.put("bytes:PaimonJniJvmHeapCommitted", String.valueOf(nonNegative(heapUsage.getCommitted())));
456-
statistics.put("bytes:PaimonJniJvmHeapMax", String.valueOf(nonNegative(heapUsage.getMax())));
457-
statistics.put("bytes:PaimonJniJvmNonHeapUsed", String.valueOf(nonNegative(nonHeapUsage.getUsed())));
458-
statistics.put("bytes:PaimonJniJvmNonHeapCommitted", String.valueOf(nonNegative(nonHeapUsage.getCommitted())));
459-
statistics.put("bytes:PaimonJniJvmNonHeapMax", String.valueOf(nonNegative(nonHeapUsage.getMax())));
454+
statistics.put("bytes_gauge:PaimonJniJvmHeapUsed", String.valueOf(nonNegative(heapUsage.getUsed())));
455+
statistics.put("bytes_gauge:PaimonJniJvmHeapCommitted", String.valueOf(nonNegative(heapUsage.getCommitted())));
456+
statistics.put("bytes_gauge:PaimonJniJvmHeapMax", String.valueOf(nonNegative(heapUsage.getMax())));
457+
statistics.put("bytes_gauge:PaimonJniJvmNonHeapUsed", String.valueOf(nonNegative(nonHeapUsage.getUsed())));
458+
statistics.put("bytes_gauge:PaimonJniJvmNonHeapCommitted",
459+
String.valueOf(nonNegative(nonHeapUsage.getCommitted())));
460+
statistics.put("bytes_gauge:PaimonJniJvmNonHeapMax", String.valueOf(nonNegative(nonHeapUsage.getMax())));
460461
}
461462

462463
private static long nonNegative(long value) {

‎fe/be-java-extensions/paimon-scanner/src/test/java/org/apache/doris/paimon/PaimonJniScannerTest.java‎

Lines changed: 9 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -92,7 +92,7 @@ public void testCloseCleansIOManagerTempDirectory() throws Exception {
9292
Field ioManagerField = PaimonJniScanner.class.getDeclaredField("ioManager");
9393
ioManagerField.setAccessible(true);
9494
ioManagerField.set(scanner, ioManager);
95-
Assert.assertEquals("1", scanner.getStatistics().get("counter:PaimonJniIOManagerEnabled"));
95+
Assert.assertEquals("1", scanner.getStatistics().get("gauge:PaimonJniIOManagerEnabled"));
9696

9797
scanner.close();
9898
Assert.assertFalse(spillDir.exists());
@@ -108,20 +108,20 @@ public void testStatisticsIncludePaimonDiagnostics() throws Exception {
108108

109109
Map<String, String> statistics = scanner.getStatistics();
110110

111-
Assert.assertEquals("0", statistics.get("counter:PaimonJniIOManagerEnabled"));
112-
Assert.assertEquals("0", statistics.get("counter:PaimonJniRequiredFieldCount"));
111+
Assert.assertEquals("0", statistics.get("gauge:PaimonJniIOManagerEnabled"));
112+
Assert.assertEquals("0", statistics.get("gauge:PaimonJniRequiredFieldCount"));
113113
Assert.assertEquals("13", statistics.get("counter:PaimonJniSplitEncodedLength"));
114114
Assert.assertEquals("17", statistics.get("counter:PaimonJniPredicateEncodedLength"));
115-
Assert.assertEquals("1", statistics.get("counter:PaimonJniAsyncThresholdConfigured"));
115+
Assert.assertEquals("1", statistics.get("gauge:PaimonJniAsyncThresholdConfigured"));
116116
Assert.assertEquals(String.valueOf(10L * 1024L * 1024L),
117-
statistics.get("bytes:PaimonJniAsyncThresholdBytes"));
118-
Assert.assertTrue(statistics.containsKey("counter:PaimonJniAsyncReaderThreadCount"));
119-
Assert.assertTrue(statistics.containsKey("counter:PaimonJniActiveScannerCount"));
117+
statistics.get("bytes_gauge:PaimonJniAsyncThresholdBytes"));
118+
Assert.assertTrue(statistics.containsKey("gauge:PaimonJniAsyncReaderThreadCount"));
119+
Assert.assertTrue(statistics.containsKey("gauge:PaimonJniActiveScannerCount"));
120120
Assert.assertTrue(statistics.containsKey("counter:PaimonJniReadBatchCalls"));
121121
Assert.assertTrue(statistics.containsKey("timer:PaimonJniScannerOpenTime"));
122122
Assert.assertTrue(statistics.containsKey("timer:PaimonJniReadBatchTime"));
123-
Assert.assertTrue(Long.parseLong(statistics.get("bytes:PaimonJniJvmHeapUsed")) > 0);
124-
Assert.assertTrue(Long.parseLong(statistics.get("bytes:PaimonJniJvmHeapCommitted")) > 0);
123+
Assert.assertTrue(Long.parseLong(statistics.get("bytes_gauge:PaimonJniJvmHeapUsed")) > 0);
124+
Assert.assertTrue(Long.parseLong(statistics.get("bytes_gauge:PaimonJniJvmHeapCommitted")) > 0);
125125
}
126126

127127
@Test

0 commit comments

Comments
 (0)