Skip to content

Commit b1fe567

Browse files
authored
[feat](audit log) add queue_time_ms in audit_log (apache#60418)
Add a queue_time_ms column to table __internal_schema.audit_log to track per-query queueing behavior and elapsed time.
1 parent 170e293 commit b1fe567

7 files changed

Lines changed: 154 additions & 0 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/catalog/InternalSchema.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -163,6 +163,8 @@ public class InternalSchema {
163163
// execution info
164164
AUDIT_SCHEMA.add(new ColumnDef("query_time",
165165
ScalarType.createType(PrimitiveType.BIGINT), ColumnNullableType.NULLABLE));
166+
AUDIT_SCHEMA.add(new ColumnDef("queue_time_ms",
167+
ScalarType.createType(PrimitiveType.BIGINT), ColumnNullableType.NULLABLE));
166168
AUDIT_SCHEMA.add(new ColumnDef("cpu_time_ms",
167169
ScalarType.createType(PrimitiveType.BIGINT), ColumnNullableType.NULLABLE));
168170
AUDIT_SCHEMA.add(new ColumnDef("peak_memory_bytes",

‎fe/fe-core/src/main/java/org/apache/doris/plugin/AuditEvent.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -84,6 +84,8 @@ public enum EventType {
8484
// execution info
8585
@AuditField(value = "Time(ms)", colName = "query_time")
8686
public long queryTime = -1;
87+
@AuditField(value = "QueueTimeMs", colName = "queue_time_ms")
88+
public long queueTimeMs = -1;
8789
@AuditField(value = "CpuTimeMS", colName = "cpu_time_ms")
8890
public long cpuTimeMs = -1;
8991
@AuditField(value = "PeakMemoryBytes", colName = "peak_memory_bytes")
@@ -231,6 +233,11 @@ public AuditEventBuilder setQueryTime(long queryTime) {
231233
return this;
232234
}
233235

236+
public AuditEventBuilder setQueueTimeMs(long queueTimeMs) {
237+
auditEvent.queueTimeMs = queueTimeMs;
238+
return this;
239+
}
240+
234241
public AuditEventBuilder setScanBytes(long scanBytes) {
235242
auditEvent.scanBytes = scanBytes;
236243
return this;

‎fe/fe-core/src/main/java/org/apache/doris/plugin/audit/AuditLoader.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -168,6 +168,7 @@ private void fillLogBuffer(AuditEvent event, StringBuilder logBuffer) {
168168

169169
// execution info
170170
logBuffer.append(event.queryTime).append(AUDIT_TABLE_COL_SEPARATOR);
171+
logBuffer.append(event.queueTimeMs).append(AUDIT_TABLE_COL_SEPARATOR);
171172
logBuffer.append(event.cpuTimeMs).append(AUDIT_TABLE_COL_SEPARATOR);
172173
logBuffer.append(event.peakMemoryBytes).append(AUDIT_TABLE_COL_SEPARATOR);
173174
logBuffer.append(event.scanBytes).append(AUDIT_TABLE_COL_SEPARATOR);

‎fe/fe-core/src/main/java/org/apache/doris/qe/AuditLogHelper.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -47,6 +47,7 @@
4747
import org.apache.doris.plugin.AuditEvent.AuditEventBuilder;
4848
import org.apache.doris.plugin.AuditEvent.EventType;
4949
import org.apache.doris.qe.QueryState.MysqlStateType;
50+
import org.apache.doris.resource.workloadgroup.QueueToken;
5051
import org.apache.doris.service.FrontendOptions;
5152

5253
import com.google.common.base.Strings;
@@ -234,6 +235,8 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme
234235
}
235236
String cluster = Config.isCloudMode() ? cloudCluster : "";
236237
String stmtType = getStmtType(parsedStmt);
238+
long queueTimeMs = getQueueTimeMs(ctx);
239+
237240

238241
AuditEventBuilder auditEventBuilder = ctx.getAuditEventBuilder();
239242
// ATTN: MUST reset, otherwise, the same AuditEventBuilder instance will be used in the next query.
@@ -252,6 +255,7 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme
252255
.setErrorMessage((ctx.getState().getErrorMessage() == null ? "" :
253256
ctx.getState().getErrorMessage().replace("\n", " ").replace("\t", " ")))
254257
.setQueryTime(elapseMs)
258+
.setQueueTimeMs(queueTimeMs)
255259
.setCpuTimeMs(statistics == null ? 0 : statistics.getCpuMs())
256260
.setPeakMemoryBytes(statistics == null ? 0 : statistics.getMaxPeakMemoryBytes())
257261
.setScanBytes(statistics == null ? 0 : statistics.getScanBytes())
@@ -447,6 +451,14 @@ private static void logAuditLogImpl(ConnectContext ctx, String origStmt, Stateme
447451
}
448452
}
449453

454+
private static long getQueueTimeMs(ConnectContext ctx) {
455+
QueueToken queueToken = null;
456+
if (ctx.getExecutor() != null && ctx.getExecutor().getCoord() != null) {
457+
queueToken = ctx.getExecutor().getCoord().getQueueToken();
458+
}
459+
return queueToken == null ? -1 : queueToken.getQueueEndTime() - queueToken.getQueueStartTime();
460+
}
461+
450462
private static String getStmtType(StatementBase stmt) {
451463
if (stmt == null) {
452464
return StmtType.OTHER.name();

‎fe/fe-core/src/test/java/org/apache/doris/qe/AuditEventProcessorTest.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ public void testAuditEvent() {
5858
.setDb("db1")
5959
.setState("EOF")
6060
.setQueryTime(2000)
61+
.setQueueTimeMs(2000)
6162
.setScanBytes(100000)
6263
.setScanRows(200000)
6364
.setReturnRows(1)
@@ -68,6 +69,7 @@ public void testAuditEvent() {
6869
Assert.assertEquals("127.0.0.1", event.clientIp);
6970
Assert.assertEquals(200000, event.scanRows);
7071
Assert.assertEquals("SELECT", event.stmtType);
72+
Assert.assertEquals(2000, event.queueTimeMs);
7173
}
7274

7375
@Test

‎regression-test/data/audit/test_audit_log_behavior.out‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -11,6 +11,7 @@ state varchar(128) Yes false \N NONE
1111
error_code int Yes false \N NONE
1212
error_message text Yes false \N NONE
1313
query_time bigint Yes false \N NONE
14+
queue_time_ms bigint Yes false \N NONE
1415
cpu_time_ms bigint Yes false \N NONE
1516
peak_memory_bytes bigint Yes false \N NONE
1617
scan_bytes bigint Yes false \N NONE
Lines changed: 129 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,129 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
suite("test_audit_log_queue_time", "nonConcurrent") {
19+
// Check admin privilege
20+
try {
21+
sql "set global enable_audit_plugin = true"
22+
} catch (Exception e) {
23+
log.warn("skip this case, because " + e.getMessage())
24+
assertTrue(e.getMessage().toUpperCase().contains("ADMIN"))
25+
return
26+
}
27+
28+
def tableName = "audit_queue_time_test"
29+
def wgName = "test_queue_time_wg"
30+
def testMarker = UUID.randomUUID().toString().substring(0, 8)
31+
32+
// Cleanup environment
33+
sql "drop table if exists ${tableName}"
34+
sql "drop workload group if exists ${wgName}"
35+
36+
// Create test table
37+
sql """
38+
CREATE TABLE `${tableName}` (
39+
`id` bigint,
40+
`name` varchar(32)
41+
) ENGINE=OLAP
42+
DUPLICATE KEY(`id`)
43+
DISTRIBUTED BY HASH(`id`) BUCKETS 1
44+
PROPERTIES ("replication_allocation" = "tag.location.default: 1")
45+
"""
46+
47+
sql "insert into ${tableName} values (1, 'test')"
48+
49+
def maxConcurrency = 1
50+
// Create workload group: max_concurrency=1 ensures queries queue up
51+
sql """
52+
create workload group ${wgName}
53+
properties (
54+
'max_concurrency' = '${maxConcurrency}',
55+
'max_queue_size' = '10',
56+
'queue_timeout' = '30000'
57+
)
58+
"""
59+
60+
// Wait for workload group to take effect
61+
Thread.sleep(5000)
62+
63+
// Truncate audit_log for easier testing
64+
sql "truncate table __internal_schema.audit_log"
65+
66+
// Submit concurrent queries with marker for later lookup
67+
def sqlSleepTime = 5
68+
def queuedSqlCnt = 1
69+
def threads = []
70+
for (int i = 0; i < maxConcurrency + queuedSqlCnt; i++) {
71+
def idx = i
72+
threads << Thread.start {
73+
try {
74+
sql "set workload_group=${wgName}"
75+
// Use sleep function to simulate long query, ensuring subsequent queries need to queue
76+
sql """
77+
select sleep(${sqlSleepTime}), '${testMarker}_${idx}' as marker
78+
from ${tableName} limit 1
79+
"""
80+
} catch (Exception e) {
81+
log.warn("Query ${idx} failed: ${e.getMessage()}")
82+
}
83+
}
84+
}
85+
86+
// Wait for all queries to complete
87+
threads.each { it.join() }
88+
89+
// Wait for audit log to flush
90+
Thread.sleep(5000)
91+
sql "call flush_audit_log()"
92+
Thread.sleep(5000)
93+
94+
// Verify queue_time_ms column exists
95+
def schemaResult = sql "desc internal.__internal_schema.audit_log"
96+
def hasQueueTimeMs = schemaResult.any { it[0] == "queue_time_ms" }
97+
assertTrue(hasQueueTimeMs)
98+
99+
// check result
100+
def retry = 10
101+
def query = """
102+
select query_id, queue_time_ms, stmt
103+
from __internal_schema.audit_log
104+
where stmt like '%${testMarker}%'
105+
and queue_time_ms > 0
106+
order by time
107+
"""
108+
def auditResult = sql "${query}"
109+
110+
while (auditResult.isEmpty()) {
111+
if (retry-- < 0) {
112+
throw new RuntimeException("It has retried a few but still failed, you need to check it")
113+
}
114+
sql "call flush_audit_log()"
115+
sleep(3000)
116+
auditResult = sql "${query}"
117+
}
118+
119+
auditResult.each { row ->
120+
assertTrue(row[1] >= sqlSleepTime * 1000)
121+
}
122+
123+
assertTrue(auditResult.size() >= queuedSqlCnt)
124+
125+
// Cleanup
126+
sql "drop table if exists ${tableName}"
127+
sql "drop workload group if exists ${wgName}"
128+
sql "set global enable_audit_plugin = false"
129+
}

0 commit comments

Comments
 (0)