Skip to content

Commit d91816e

Browse files
committed
[fix](audit) Bound audit load timeouts and preserve prepared execution identity
### What problem does this PR solve? Issue Number: apache#70122 Related PR: apache#70123 Problem Summary: An oversized audit-plugin timeout caused every audit-table load to fail. Fit authenticated built-in audit loads to the master's Stream Load limits at transaction begin and load planning, where authoritative bounds are available. Also keep the timeout representable by the load sink in signed milliseconds. Forwarded plans preserve the master-selected timeout despite different follower limits. Variable replication and ordinary loads remain unchanged. Prepared executions failing before executor creation retained the previous command's statement ID and SQL hash. Clear the unallocated statement ID and restore the hash captured at PREPARE for every always-on execute. Retaining the effective preparation hash also preserves SQL conversion fallback and subsequent SQL-hash block-rule matching. ### Release note Prevent unsupported timeouts from disabling always-on audit-table loads and preserve the correct statement identity for prepared-execution audit rows. ### Check List (For Author) - Test: Regression tests cover master timeout selection, transaction-validator acceptance, planning and forwarded-plan timeouts, authentication/non-audit exclusions, actual audit events for binding/long-data failures, valid retry, and effective preparation hash reuse. - Behavior changed: Yes - Does this need documentation: No separate page; update the config description and policy comments for master-side Stream Load timeout enforcement.
1 parent 32063b0 commit d91816e

7 files changed

Lines changed: 332 additions & 5 deletions

File tree

‎fe/fe-common/src/main/java/org/apache/doris/common/Config.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -121,7 +121,8 @@ public class Config extends ConfigBase {
121121

122122
@ConfField(description = "When true, query audit cannot be disabled or hollowed out at runtime: every statement "
123123
+ "this FE receives is audited to both fe.audit.log and __internal_schema.audit_log. "
124-
+ "enable_audit_plugin and enable_prepared_stmt_audit_log stay on, the other audit_plugin_* variables "
124+
+ "enable_audit_plugin and enable_prepared_stmt_audit_log stay on, audit_plugin_load_timeout stays within "
125+
+ "the Stream Load timeout range, and the other audit_plugin_* variables "
125126
+ "and the configs audit_event_log_queue_size, query_audit_log_timeout_ms, "
126127
+ "be_report_query_statistics_timeout_ms and workload_runtime_status_thread_interval_ms cannot be set "
127128
+ "weaker than their defaults, skip_audit_user_list is ignored, and audit_log_modules always includes "

‎fe/fe-common/src/main/java/org/apache/doris/qe/QueryAuditPolicy.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,9 @@
4040
* runtime may disable query auditing or hollow it out: the audit switches stay on, and no audit limit is weaker than
4141
* its factory default. A limit whose smaller values drop events or cut statements short has its default as the
4242
* minimum, one whose larger values hold events back has it as the maximum, and a batching knob, harmful in both
43-
* directions, is fixed at it. The config itself changes only on disk, with a restart.
43+
* directions, is fixed at it. The master also fits built-in audit load timeouts to its transaction and load sink
44+
* limits before beginning the transaction and planning the load. The config itself changes only on disk,
45+
* with a restart.
4446
*
4547
* <p>The rules are defined here only, for three sides:
4648
* <ul>

‎fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -57,6 +57,7 @@
5757
import com.google.common.base.Preconditions;
5858
import com.google.common.base.Strings;
5959
import com.google.common.collect.Lists;
60+
import org.apache.commons.codec.digest.DigestUtils;
6061
import org.apache.logging.log4j.LogManager;
6162
import org.apache.logging.log4j.Logger;
6263

@@ -161,6 +162,11 @@ protected void handleExecute(PrepareCommand prepareCommand, long stmtId, Prepare
161162
// With query_audit_always_on, an execution that fails before its EXECUTE text is built below is audited with
162163
// the statement it was prepared from.
163164
String stmtStr = QueryAuditPolicy.isAlwaysOn() ? prepareCommand.getOriginalStmt().originStmt : "";
165+
if (QueryAuditPolicy.isAlwaysOn()) {
166+
String sqlHash = prepCtx.getSqlHash();
167+
// Proxy contexts can prepare without a COM_QUERY having assigned a hash.
168+
ctx.setSqlHash(sqlHash == null ? DigestUtils.md5Hex(stmtStr) : sqlHash);
169+
}
164170
try {
165171
if (prepCtx.getLongDataError() != null) {
166172
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, prepCtx.getLongDataError());
@@ -318,9 +324,9 @@ private void auditExecute(String stmtStr) {
318324
if (audit && executor != null) {
319325
auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true);
320326
} else if (audit && QueryAuditPolicy.isAlwaysOn()) {
321-
// The execution failed before its executor was created, so the connection still holds the query id of
322-
// its previous statement, which this record must not carry.
327+
// No executor assigned query/statement IDs, so do not retain the previous statement's IDs.
323328
ctx.resetQueryId();
329+
ctx.setStmtId(0);
324330
auditAfterExec(stmtStr, null, null, true);
325331
} else {
326332
// When audit log is disabled for prepared statements, still update QPS metrics.

‎fe/fe-core/src/main/java/org/apache/doris/qe/PreparedStatementContext.java‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -32,6 +32,8 @@ public class PreparedStatementContext {
3232
public ConnectContext ctx;
3333
private StatementContext statementContext;
3434
public String stmtString;
35+
// Preserve the effective PREPARE hash, including SQL conversion fallback, across connection commands.
36+
private final String sqlHash;
3537
public Optional<ShortCircuitQueryContext> shortCircuitQueryContext = Optional.empty();
3638
public Optional<GroupCommitPlanner> groupCommitPlanner = Optional.empty();
3739
// Long data belongs to a prepared statement and survives individual upload commands.
@@ -48,6 +50,11 @@ public PreparedStatementContext(PrepareCommand command,
4850
this.ctx = ctx;
4951
this.statementContext = statementContext;
5052
this.stmtString = stmtString;
53+
this.sqlHash = ctx.getSqlHash();
54+
}
55+
56+
public String getSqlHash() {
57+
return sqlHash;
5158
}
5259

5360
public boolean isEmptyCursorOpen() {

‎fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java‎

Lines changed: 34 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -123,6 +123,7 @@
123123
import org.apache.doris.persist.gson.GsonUtils;
124124
import org.apache.doris.planner.GroupCommitPlanner;
125125
import org.apache.doris.planner.OlapTableSink;
126+
import org.apache.doris.plugin.audit.AuditLoader;
126127
import org.apache.doris.qe.ConnectContext;
127128
import org.apache.doris.qe.ConnectProcessor;
128129
import org.apache.doris.qe.Coordinator;
@@ -132,6 +133,7 @@
132133
import org.apache.doris.qe.MysqlConnectProcessor;
133134
import org.apache.doris.qe.NereidsCoordinator;
134135
import org.apache.doris.qe.QeProcessorImpl;
136+
import org.apache.doris.qe.QueryAuditPolicy;
135137
import org.apache.doris.qe.QueryState;
136138
import org.apache.doris.qe.SessionVariable;
137139
import org.apache.doris.qe.StmtExecutor;
@@ -1508,6 +1510,36 @@ public TLoadTxnBeginResult loadTxnBegin(TLoadTxnBeginRequest request) throws TEx
15081510
return result;
15091511
}
15101512

1513+
static long getLoadTxnTimeout(TLoadTxnBeginRequest request) {
1514+
long timeout = request.isSetTimeout() ? request.getTimeout() : Config.stream_load_default_timeout_second;
1515+
if (!isAlwaysOnAuditLoad(request.isSetAuthCode(), request.getToken(), request.getDb(), request.getTbl())) {
1516+
return timeout;
1517+
}
1518+
// Called after token authentication on the master: a sending FE may have different timeout limits.
1519+
return boundAuditLoadTimeout(timeout);
1520+
}
1521+
1522+
private static boolean isAlwaysOnAuditLoad(boolean hasAuthCode, String token, String db, String table) {
1523+
return QueryAuditPolicy.isAlwaysOn() && !hasAuthCode && !Strings.isNullOrEmpty(token)
1524+
&& FeConstants.INTERNAL_DB_NAME.equals(db) && AuditLoader.AUDIT_LOG_TABLE.equals(table);
1525+
}
1526+
1527+
private static long boundAuditLoadTimeout(long timeout) {
1528+
// The V1 load sink stores its RPC timeout in signed milliseconds.
1529+
long maximum = Math.min(Config.max_stream_load_timeout_second, Integer.MAX_VALUE / 1000);
1530+
return Math.max(Config.min_load_timeout_second, Math.min(timeout, maximum));
1531+
}
1532+
1533+
private void normalizeAuditLoadTimeout(TStreamLoadPutRequest request) throws AuthenticationException {
1534+
if (!isAlwaysOnAuditLoad(request.isSetAuthCode(), request.getToken(), request.getDb(), request.getTbl())
1535+
|| !Env.getCurrentEnv().isMaster()) {
1536+
return;
1537+
}
1538+
checkTokenOrThrow(request.getToken());
1539+
long timeout = request.isSetTimeout() ? request.getTimeout() : Config.stream_load_default_timeout_second;
1540+
request.setTimeout((int) boundAuditLoadTimeout(timeout));
1541+
}
1542+
15111543
private TLoadTxnBeginResult loadTxnBeginImpl(TLoadTxnBeginRequest request, String clientIp) throws UserException {
15121544
if (request.isSetAuthCode()) {
15131545
// TODO: deprecated, removed in 3.1, use token instead.
@@ -1549,7 +1581,7 @@ private TLoadTxnBeginResult loadTxnBeginImpl(TLoadTxnBeginRequest request, Strin
15491581
}
15501582
}
15511583
// begin
1552-
long timeoutSecond = request.isSetTimeout() ? request.getTimeout() : Config.stream_load_default_timeout_second;
1584+
long timeoutSecond = getLoadTxnTimeout(request);
15531585
Backend backend = Env.getCurrentSystemInfo().getBackend(request.getBackendId());
15541586
long startTime = backend != null ? backend.getLastStartTime() : 0;
15551587
TxnCoordinator txnCoord = new TxnCoordinator(TxnSourceType.BE, request.getBackendId(), clientIp, startTime);
@@ -3113,6 +3145,7 @@ public TStreamLoadPutResult streamLoadPut(TStreamLoadPutRequest request) {
31133145

31143146
try {
31153147
checkGroupCommitStreamLoadAuth(request);
3148+
normalizeAuditLoadTimeout(request);
31163149
if (groupCommitMode != null && !groupCommitMode.equals("off_mode") && Env.getCurrentEnv().isMaster()
31173150
&& Config.enable_forward_group_commit_stream_load_to_follower) {
31183151
TStreamLoadPutResult forwardedResult = forwardGroupCommitStreamLoad(request);

‎fe/fe-core/src/test/java/org/apache/doris/qe/MysqlConnectProcessorLongDataTest.java‎

Lines changed: 63 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,8 +17,11 @@
1717

1818
package org.apache.doris.qe;
1919

20+
import org.apache.doris.analysis.UserIdentity;
21+
import org.apache.doris.catalog.Env;
2022
import org.apache.doris.catalog.MysqlColType;
2123
import org.apache.doris.common.Config;
24+
import org.apache.doris.datasource.CatalogMgr;
2225
import org.apache.doris.mysql.MysqlChannel;
2326
import org.apache.doris.mysql.MysqlSerializer;
2427
import org.apache.doris.mysql.protocol.MysqlProtocolAdapter;
@@ -39,11 +42,14 @@
3942
import org.apache.doris.nereids.trees.plans.logical.LogicalPlan;
4043
import org.apache.doris.nereids.types.StringType;
4144
import org.apache.doris.nereids.types.VarcharType;
45+
import org.apache.doris.plugin.AuditEvent;
46+
import org.apache.doris.resource.workloadschedpolicy.WorkloadRuntimeStatusMgr;
4247
import org.apache.doris.thrift.TUniqueId;
4348

4449
import org.junit.jupiter.api.Assertions;
4550
import org.junit.jupiter.api.Test;
4651
import org.junit.jupiter.api.function.Executable;
52+
import org.mockito.ArgumentCaptor;
4753
import org.mockito.MockedConstruction;
4854
import org.mockito.MockedStatic;
4955
import org.mockito.Mockito;
@@ -226,9 +232,11 @@ public void testAlwaysOnAuditsExecuteWithoutSessionSwitch() throws Throwable {
226232
withAlwaysOn(true, () -> {
227233
Harness h = new Harness(MysqlColType.MYSQL_TYPE_LONG);
228234
Assertions.assertFalse(h.context.getSessionVariable().isEnablePreparedStmtAuditLog());
235+
h.context.setSqlHash("previous-query-hash");
229236
h.execute(new byte[] {0, 1, 3, 0, 42, 0, 0, 0}, audit -> audit.verify(() -> AuditLogHelper.logAuditLog(
230237
Mockito.same(h.context), Mockito.eq("EXECUTE `7` USING 42 /*originalSql = select ?*/"),
231238
Mockito.isNull(), Mockito.isNull(), Mockito.eq(true))));
239+
Assertions.assertEquals("1fe1379fe2a31b8d16219655761820a2", h.context.getSqlHash());
232240
});
233241
}
234242

@@ -283,6 +291,61 @@ public void testAlwaysOnAuditsExecuteFailedOnBinding() throws Throwable {
283291
});
284292
}
285293

294+
@Test
295+
public void testAlwaysOnEarlyFailuresDoNotReusePreviousStatementIdentity() throws Throwable {
296+
withAlwaysOn(true, () -> {
297+
for (MysqlColType type : new MysqlColType[] {MysqlColType.MYSQL_TYPE_LONG,
298+
MysqlColType.MYSQL_TYPE_VARSTRING}) {
299+
Harness h = new Harness(type);
300+
h.context.getSessionVariable().maxAllowedPacket = 1;
301+
h.upload(7, 0, new byte[] {65});
302+
if (type == MysqlColType.MYSQL_TYPE_VARSTRING) {
303+
h.upload(7, 0, new byte[] {66});
304+
}
305+
// An unrelated COM_QUERY left this identity on the same connection.
306+
h.context.setStmtId(42);
307+
h.context.setSqlHash("previous-query-hash");
308+
h.context.setCurrentUserIdentity(UserIdentity.ROOT);
309+
h.context.setStartTime();
310+
h.context.getState().reset();
311+
312+
Env env = Mockito.mock(Env.class);
313+
Mockito.when(env.getCatalogMgr()).thenReturn(Mockito.mock(CatalogMgr.class));
314+
WorkloadRuntimeStatusMgr statusMgr = Mockito.mock(WorkloadRuntimeStatusMgr.class);
315+
Mockito.when(env.getWorkloadRuntimeStatusMgr()).thenReturn(statusMgr);
316+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class)) {
317+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
318+
h.processor.handleExecute(h.prepared.command, 7, h.prepared,
319+
ByteBuffer.wrap(new byte[] {0, 0}).order(ByteOrder.LITTLE_ENDIAN), null);
320+
321+
ArgumentCaptor<AuditEvent> event = ArgumentCaptor.forClass(AuditEvent.class);
322+
Mockito.verify(statusMgr).submitFinishQueryToAudit(event.capture());
323+
Assertions.assertEquals("ERR", event.getValue().state);
324+
Assertions.assertEquals("select ?", event.getValue().stmt);
325+
Assertions.assertEquals("NaN", event.getValue().queryId);
326+
Assertions.assertEquals(0, event.getValue().stmtId);
327+
Assertions.assertEquals("1fe1379fe2a31b8d16219655761820a2", event.getValue().sqlHash);
328+
}
329+
h.context.setQueryId(new TUniqueId(8L, 8L));
330+
h.execute(new byte[] {0, 1, 3, 0, 42, 0, 0, 0});
331+
Assertions.assertEquals("1fe1379fe2a31b8d16219655761820a2", h.context.getSqlHash());
332+
}
333+
});
334+
}
335+
336+
@Test
337+
public void testAlwaysOnExecutePreservesEffectivePreparationHash() throws Throwable {
338+
withAlwaysOn(true, () -> {
339+
ConnectContext context = new ConnectContext();
340+
context.setSqlHash("effective-sql-conversion-fallback-hash");
341+
Harness h = new Harness(context, List.of(
342+
new Placeholder(new PlaceholderId(0), MysqlColType.MYSQL_TYPE_LONG.getCode())));
343+
context.setSqlHash("another-command-hash");
344+
h.execute(new byte[] {0, 1, 3, 0, 42, 0, 0, 0});
345+
Assertions.assertEquals("effective-sql-conversion-fallback-hash", context.getSqlHash());
346+
});
347+
}
348+
286349
// With the policy off, neither failure is audited, as before.
287350
@Test
288351
public void testFailedExecuteIsNotAuditedWhenAlwaysOnIsOff() throws Throwable {

0 commit comments

Comments
 (0)