Skip to content

Commit 6fafa31

Browse files
authored
[fix](fe) Fix prepared statement streaming, forwarding and cursor framing (apache#69314)
### What problem does this PR solve? Problem Summary: - `COM_STMT_SEND_LONG_DATA` incorrectly returns an error packet, leaving streaming clients out of sync. Accept parameter chunks without replying, defer upload errors until EXECUTE, preserve binary bytes, and clear upload state on EXECUTE/RESET/CLOSE. - Reusing server-side prepared statements through a non-master FE can fail with `NoSuchElementException: No value present`: the driver omits unchanged parameter types, while the master prepares each forwarded execution independently. Include cached types, the NULL bitmap and complete values in forwarded EXECUTE payloads. - Forwarded PREPARE can use a stale statement context when planning placeholders. Use the parser's current prepare-stage context. - Empty results with `useCursorFetch=true` can hang or leave unread packets when client connection attributes are disabled or rewritten. Return a valid empty cursor, complete FETCH with `LAST_ROW_SENT`, retain cursor state on the client-facing FE, and preserve existing framing for older FEs. Nonempty results continue to use buffered delivery; nonempty cursor pagination is not added. - Normalized streamed values can remain attached to idle pooled connections after statement close/reset. Release the command-only execution buffer after response finalization and forwarding complete. ### Release note - Fix streamed parameters, repeated prepared-statement forwarding and empty cursor results in the MySQL protocol, and reduce retained parameter-buffer memory on idle connections. ### Check List (For Author) - Test: - [x] Unit Test: 32 focused long-data, protocol adapter and forwarding tests passed on the final source, including the new buffer-lifecycle test. - [x] Regression test: `prepared_stmt_p0/send_long_data` and `prepared_stmt_p0/cursor_fetch_empty_result` passed during implementation. - [x] Manual test: raw-protocol upload/cursor lifecycle and connection reuse; rewritten client-attribute proxy reproduction passed after the fix. - [x] Checkstyle: zero violations in the latest modified Java files. - Behavior changed: - [x] Yes. Long-data uploads no longer produce responses; forwarded parameters are self-contained; empty cursors finish through FETCH; completed commands release execution buffers. - Does this need documentation? - [x] No.
1 parent 493259c commit 6fafa31

15 files changed

Lines changed: 808 additions & 34 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/mysql/protocol/MysqlProtocolAdapter.java‎

Lines changed: 40 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,7 @@
3535
import org.apache.doris.qe.ConnectContext;
3636
import org.apache.doris.qe.ConnectContext.ConnectType;
3737
import org.apache.doris.qe.ConnectProcessor;
38+
import org.apache.doris.qe.PreparedStatementContext;
3839
import org.apache.doris.qe.QueryState.MysqlStateType;
3940
import org.apache.doris.qe.ShowResultSet;
4041
import org.apache.doris.qe.StmtExecutor;
@@ -74,6 +75,9 @@ public class MysqlProtocolAdapter implements ProtocolAdapter {
7475
private ByteBuffer prepareExecuteBuffer;
7576
// Whether the current COM_STMT_EXECUTE requested a server-side read-only cursor.
7677
private boolean cursorFetchRequested;
78+
private boolean resultHasRows;
79+
private boolean emptyCursorSupported = true;
80+
private PreparedStatementContext executingPreparedStatement;
7781

7882
public MysqlProtocolAdapter(MysqlChannel channel) {
7983
this.channel = channel;
@@ -146,7 +150,7 @@ public boolean canRetryQuery(ConnectContext ctx) {
146150
*/
147151
@Override
148152
public void beforeStatement(ConnectContext ctx) {
149-
channel.reset();
153+
resetResult();
150154
}
151155

152156
@Override
@@ -183,6 +187,7 @@ public void fillForwardRequest(ConnectContext ctx, TMasterOpRequest request) {
183187
request.setPrepareExecuteBuffer(prepareExecuteBuffer);
184188
}
185189
request.setCursorFetchRequested(cursorFetchRequested);
190+
request.setEmptyCursorSupported(true);
186191
}
187192
request.setClientDeprecatedEOF(channel.clientDeprecatedEOF());
188193
request.setMysqlCapability(capability.getFlags());
@@ -212,6 +217,7 @@ public void restoreFromForwardRequest(ConnectContext ctx, TMasterOpRequest reque
212217
channel.setClientDeprecatedEOF();
213218
}
214219
cursorFetchRequested = request.isSetCursorFetchRequested() && request.isCursorFetchRequested();
220+
emptyCursorSupported = request.isSetEmptyCursorSupported() && request.isEmptyCursorSupported();
215221
}
216222

217223
/**
@@ -277,6 +283,13 @@ public void finishCommand(ConnectContext ctx, StmtExecutor executor) throws IOEx
277283
return;
278284
}
279285

286+
if (executingPreparedStatement != null) {
287+
// A forwarded EXECUTE's final packet was encoded by the master. Retain the cursor on
288+
// the client-facing FE so its later FETCH can complete locally without another RPC.
289+
boolean cursorOpen = Byte.toUnsignedInt(packet.get(packet.position())) == 0xFE
290+
&& (packet.get(packet.position() + 3) & MysqlServerStatusFlag.SERVER_STATUS_CURSOR_EXISTS) != 0;
291+
executingPreparedStatement.setEmptyCursorOpen(cursorOpen);
292+
}
280293
LOG.debug("Send to mysql channel for query {}", DebugUtil.printId(ctx.queryId()));
281294
channel.sendAndFlush(packet);
282295
// note(wb) we should write profile after return result to mysql client
@@ -304,6 +317,11 @@ public ByteBuffer responsePacket(ConnectContext ctx) {
304317
// instead of the traditional EOF packet. This is required by the MySQL protocol
305318
// and expected by MySQL Connector/J 9.5.0+.
306319
if (ctx.getState().getStateType() == MysqlStateType.EOF && channel.clientDeprecatedEOF()) {
320+
if (emptyCursorSupported && cursorFetchRequested && !resultHasRows) {
321+
// Both old and new drivers consume a cursor's opening terminator. An empty
322+
// cursor needs no retained rows and its first FETCH sends LAST_ROW_SENT.
323+
ctx.getState().serverStatus |= MysqlServerStatusFlag.SERVER_STATUS_CURSOR_EXISTS;
324+
}
307325
packet = new MysqlResultSetEndPacket(ctx.getState());
308326
} else {
309327
packet = ctx.getState().toResponsePacket();
@@ -323,7 +341,9 @@ public ByteBuffer responsePacket(ConnectContext ctx) {
323341

324342
@Override
325343
public void afterStatement(ConnectContext ctx) {
344+
prepareExecuteBuffer = null;
326345
cursorFetchRequested = false;
346+
executingPreparedStatement = null;
327347
}
328348

329349
@Override
@@ -375,18 +395,31 @@ public void setCursorFetchRequested(boolean cursorFetchRequested) {
375395
this.cursorFetchRequested = cursorFetchRequested;
376396
}
377397

378-
/**
379-
* Whether the client will consume the first OK packet after the column definitions of the
380-
* result it asked a cursor for. Connector/J before 9.5 does, while probing whether the cursor
381-
* was created; Doris never creates one, so the result must carry one more end marker or an
382-
* empty result would leave the client waiting.
383-
*/
398+
/** Whether legacy buffered results need an extra terminator for this client. */
384399
public boolean clientConsumesCursorMetadataTerminator(ConnectContext ctx) {
385400
return cursorFetchRequested
386401
&& MysqlCursorFetchCompatibility.resolve(ctx.getConnectAttributes())
387402
!= MysqlCursorFetchCompatibility.Behavior.STANDARD;
388403
}
389404

405+
public boolean isEmptyCursorSupported() {
406+
return emptyCursorSupported;
407+
}
408+
409+
public void beginPreparedExecute(PreparedStatementContext prepared) {
410+
executingPreparedStatement = prepared;
411+
prepared.setEmptyCursorOpen(false);
412+
}
413+
414+
void recordResultRow() {
415+
resultHasRows = true;
416+
}
417+
418+
void resetResult() {
419+
resultHasRows = false;
420+
channel.reset();
421+
}
422+
390423
// The xnio read loop. A command is read and processed while reads are suspended, which is what
391424
// serializes the commands of one connection.
392425
public void startAcceptQuery(ConnectContext ctx, ConnectProcessor connectProcessor) {

‎fe/fe-core/src/main/java/org/apache/doris/mysql/protocol/MysqlResultSender.java‎

Lines changed: 6 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -135,13 +135,14 @@ public void sendFields(List<String> colNames, List<FieldInfo> fieldInfos, List<T
135135

136136
@Override
137137
public void sendRow(ByteBuffer row) throws IOException {
138+
adapter.recordResultRow();
138139
channel().sendOnePacket(row);
139140
}
140141

141142
/** Clears the send flag and whatever a failed attempt of the query left in the send buffer. */
142143
@Override
143144
public void reset() {
144-
channel().reset();
145+
adapter.resetResult();
145146
}
146147

147148
/**
@@ -270,10 +271,8 @@ private void sendMetadataTerminatorIfNeeded() throws IOException {
270271
serializer.reset();
271272
new MysqlEofPacket(ctx.getState()).writeTo(serializer);
272273
channel.sendOnePacket(serializer.toByteBuffer());
273-
} else if (adapter.clientConsumesCursorMetadataTerminator(ctx)) {
274-
// Connector/J before 9.5 consumes the first OK packet after column definitions
275-
// while probing whether a requested cursor was created. Doris does not create a
276-
// cursor, so an empty result would otherwise lose its only end marker and block.
274+
} else if (!adapter.isEmptyCursorSupported() && adapter.clientConsumesCursorMetadataTerminator(ctx)) {
275+
// Preserve framing for older client-facing FEs during rolling upgrades.
277276
serializer.reset();
278277
new MysqlResultSetEndPacket(ctx.getState()).writeTo(serializer);
279278
channel.sendOnePacket(serializer.toByteBuffer());
@@ -282,7 +281,6 @@ private void sendMetadataTerminatorIfNeeded() throws IOException {
282281

283282
private void sendTextResultRow(ResultSet resultSet) throws IOException {
284283
MysqlSerializer serializer = serializer();
285-
MysqlChannel channel = channel();
286284
for (List<String> row : resultSet.getResultRows()) {
287285
serializer.reset();
288286
for (String item : row) {
@@ -292,13 +290,12 @@ private void sendTextResultRow(ResultSet resultSet) throws IOException {
292290
serializer.writeLenEncodedString(item);
293291
}
294292
}
295-
channel.sendOnePacket(serializer.toByteBuffer());
293+
sendRow(serializer.toByteBuffer());
296294
}
297295
}
298296

299297
private void sendBinaryResultRow(ResultSet resultSet) throws IOException {
300298
MysqlSerializer serializer = serializer();
301-
MysqlChannel channel = channel();
302299
// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_binary_resultset.html#sect_protocol_binary_resultset_row_value
303300
ResultSetMetaData metaData = resultSet.getMetaData();
304301
int nullBitmapLength = (metaData.getColumnCount() + 7 + 2) / 8;
@@ -361,7 +358,7 @@ private void sendBinaryResultRow(ResultSet resultSet) throws IOException {
361358
}
362359
}
363360
}
364-
channel.sendOnePacket(serializer.toByteBuffer());
361+
sendRow(serializer.toByteBuffer());
365362
}
366363
}
367364

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/Literal.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -591,6 +591,12 @@ public static Literal getLiteralByMysqlType(MysqlColType mysqlType, boolean isUn
591591
case MYSQL_TYPE_VARCHAR:
592592
literal = handleVarcharLiteral(data);
593593
break;
594+
case MYSQL_TYPE_TINY_BLOB:
595+
case MYSQL_TYPE_MEDIUM_BLOB:
596+
case MYSQL_TYPE_LONG_BLOB:
597+
case MYSQL_TYPE_BLOB:
598+
literal = new VarBinaryLiteral(MysqlProto.readLenEncodedString(data));
599+
break;
594600
default:
595601
throw new AnalysisException("Unsupported MySQL type: " + mysqlType);
596602
}

‎fe/fe-core/src/main/java/org/apache/doris/qe/ConnectProcessor.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -156,11 +156,14 @@ protected void handleResetConnection() {
156156
}
157157

158158
protected void handleStmtResetById(int stmtId) {
159-
if (ctx.getPreparedStementContext(String.valueOf(stmtId)) == null) {
159+
PreparedStatementContext prepared = ctx.getPreparedStementContext(String.valueOf(stmtId));
160+
if (prepared == null) {
160161
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_STMT_HANDLER,
161162
String.format("Unknown prepared statement handler (%s) given to mysqld_stmt_reset", stmtId));
162163
return;
163164
}
165+
prepared.clearLongData();
166+
prepared.setEmptyCursorOpen(false);
164167
ctx.getState().setOk();
165168
}
166169

‎fe/fe-core/src/main/java/org/apache/doris/qe/MysqlConnectProcessor.java‎

Lines changed: 105 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@
2323
import org.apache.doris.catalog.Env;
2424
import org.apache.doris.catalog.MysqlColType;
2525
import org.apache.doris.catalog.TableIf;
26+
import org.apache.doris.common.AnalysisException;
2627
import org.apache.doris.common.AuthenticationException;
2728
import org.apache.doris.common.ConnectionException;
2829
import org.apache.doris.common.ErrorCode;
@@ -35,6 +36,7 @@
3536
import org.apache.doris.mysql.MysqlHandshakePacket;
3637
import org.apache.doris.mysql.MysqlProto;
3738
import org.apache.doris.mysql.MysqlSerializer;
39+
import org.apache.doris.mysql.MysqlServerStatusFlag;
3840
import org.apache.doris.mysql.authenticate.password.NativePasswordResolver;
3941
import org.apache.doris.mysql.privilege.Auth;
4042
import org.apache.doris.mysql.protocol.MysqlProtocolAdapter;
@@ -93,6 +95,27 @@ private void handleStmtReset() {
9395
handleStmtResetById(stmtId);
9496
}
9597

98+
private void handleSendLongData() {
99+
// This command never has a response, including when the upload fails. Report a known
100+
// statement's upload error on EXECUTE instead, otherwise the connection gets out of sync.
101+
ctx.getState().setNoop();
102+
packetBuf.order(ByteOrder.LITTLE_ENDIAN);
103+
if (packetBuf.remaining() < Integer.BYTES) {
104+
return;
105+
}
106+
int stmtId = packetBuf.getInt();
107+
PreparedStatementContext prepared = ctx.getPreparedStementContext(String.valueOf(stmtId));
108+
if (prepared == null) {
109+
return;
110+
}
111+
if (packetBuf.remaining() < Short.BYTES) {
112+
prepared.setLongDataError("Missing parameter index in COM_STMT_SEND_LONG_DATA");
113+
return;
114+
}
115+
int parameter = Short.toUnsignedInt(packetBuf.getShort());
116+
prepared.appendLongData(parameter, packetBuf, ctx.getSessionVariable().getMaxAllowedPacket());
117+
}
118+
96119
private String getPacket() {
97120
byte[] bytes = packetBuf.array();
98121
StringBuilder printB = new StringBuilder();
@@ -126,11 +149,16 @@ private String getHexStr(ByteBuffer packetBuf) {
126149
@Override
127150
protected void handleExecute(PrepareCommand prepareCommand, long stmtId, PreparedStatementContext prepCtx,
128151
ByteBuffer packetBuf, TUniqueId queryId) {
152+
MysqlProtocolAdapter.of(ctx).beginPreparedExecute(prepCtx);
129153
int paramCount = prepareCommand.placeholderCount();
130154
LOG.debug("execute prepared statement {}, paramCount {}", stmtId, paramCount);
131155
// null bitmap
132156
String stmtStr = "";
133157
try {
158+
if (prepCtx.getLongDataError() != null) {
159+
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, prepCtx.getLongDataError());
160+
return;
161+
}
134162
ctx.getProtocolAdapter().beforeStatement(ctx);
135163
StatementContext statementContext = prepCtx.getStatementContext();
136164
if (!ctx.isProxy()) {
@@ -144,7 +172,8 @@ protected void handleExecute(PrepareCommand prepareCommand, long stmtId, Prepare
144172
byte[] nullbitmapData = new byte[(paramCount + 7) / 8];
145173
packetBuf.get(nullbitmapData);
146174
// new_params_bind_flag
147-
if ((int) packetBuf.get() != 0) {
175+
boolean hasNewParameterTypes = packetBuf.get() != 0;
176+
if (hasNewParameterTypes) {
148177
List<Placeholder> typedPlaceholders = new ArrayList<>();
149178
// parse params's types
150179
for (int i = 0; i < paramCount; ++i) {
@@ -158,6 +187,22 @@ protected void handleExecute(PrepareCommand prepareCommand, long stmtId, Prepare
158187
prepCtx.command = prepareCommand.withPlaceholders(typedPlaceholders);
159188
prepareCommand = (PrepareCommand) prepCtx.command;
160189
}
190+
// The master prepares each forwarded EXECUTE again without connection-local state.
191+
// Include cached parameter types and inline uploaded values so it can bind independently.
192+
MysqlSerializer forwarded = prepCtx.hasLongData() || !hasNewParameterTypes
193+
? MysqlSerializer.newInstance() : null;
194+
if (forwarded != null) {
195+
for (int i = 0; i < paramCount; ++i) {
196+
if (prepCtx.hasLongData(i)) {
197+
nullbitmapData[i / 8] &= ~(1 << (i & 7));
198+
}
199+
}
200+
forwarded.writeBytes(nullbitmapData);
201+
forwarded.writeInt1(1);
202+
for (Placeholder placeholder : prepareCommand.getPlaceholders()) {
203+
forwarded.writeInt2(placeholder.getMysqlTypeCode());
204+
}
205+
}
161206
// parse param data
162207
for (int i = 0; i < paramCount; ++i) {
163208
PlaceholderId exprId = prepareCommand.getPlaceholders().get(i).getPlaceholderId();
@@ -168,9 +213,38 @@ protected void handleExecute(PrepareCommand prepareCommand, long stmtId, Prepare
168213
}
169214
MysqlColType type = prepareCommand.getPlaceholders().get(i).getMysqlColType();
170215
boolean isUnsigned = prepareCommand.getPlaceholders().get(i).isUnsigned();
171-
Literal l = Literal.getLiteralByMysqlType(type, isUnsigned, packetBuf);
216+
byte[] longData = prepCtx.getLongData(i);
217+
Literal l;
218+
if (longData != null) {
219+
switch (type) {
220+
case MYSQL_TYPE_STRING:
221+
case MYSQL_TYPE_VARSTRING:
222+
case MYSQL_TYPE_VARCHAR:
223+
case MYSQL_TYPE_TINY_BLOB:
224+
case MYSQL_TYPE_MEDIUM_BLOB:
225+
case MYSQL_TYPE_LONG_BLOB:
226+
case MYSQL_TYPE_BLOB:
227+
break;
228+
default:
229+
throw new AnalysisException("Invalid MySQL type for long data: " + type);
230+
}
231+
MysqlSerializer value = MysqlSerializer.newInstance();
232+
value.writeLenEncodedBytes(longData);
233+
l = Literal.getLiteralByMysqlType(type, isUnsigned, value.toByteBuffer());
234+
forwarded.writeBytes(value.toArray());
235+
} else {
236+
int start = packetBuf.position();
237+
l = Literal.getLiteralByMysqlType(type, isUnsigned, packetBuf);
238+
if (forwarded != null) {
239+
forwarded.writeBytes(packetBuf.array(), packetBuf.arrayOffset() + start,
240+
packetBuf.position() - start);
241+
}
242+
}
172243
statementContext.getIdToPlaceholderRealExpr().put(exprId, l);
173244
}
245+
if (forwarded != null && !ctx.isProxy()) {
246+
ctx.setPrepareExecuteBuffer(forwarded.toByteBuffer());
247+
}
174248
}
175249
ExecuteCommand executeStmt = new ExecuteCommand(String.valueOf(stmtId), prepareCommand, statementContext);
176250
// TODO set real origin statement
@@ -196,8 +270,10 @@ protected void handleExecute(PrepareCommand prepareCommand, long stmtId, Prepare
196270
LOG.warn("Process one query failed because unknown reason: ", e);
197271
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR,
198272
e.getClass().getSimpleName() + ", msg: " + e.getMessage());
273+
} finally {
274+
prepCtx.clearLongData();
199275
}
200-
if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog()) {
276+
if (ctx.getSessionVariable().isEnablePreparedStmtAuditLog() && executor != null) {
201277
auditAfterExec(stmtStr, executor.getParsedStmt(), executor.getQueryStatisticsForAuditLog(), true);
202278
} else {
203279
// When audit log is disabled for prepared statements, still update QPS metrics.
@@ -234,6 +310,23 @@ private void handleExecute() {
234310
handleExecute(preparedStatementContext.command, stmtId, preparedStatementContext, packetBuf, null);
235311
}
236312

313+
protected void handleStmtFetch(int stmtId) {
314+
PreparedStatementContext prepared = ctx.getPreparedStementContext(String.valueOf(stmtId));
315+
if (prepared == null) {
316+
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_STMT_HANDLER,
317+
String.format("Unknown prepared statement handler (%s) given to mysqld_stmt_fetch", stmtId));
318+
return;
319+
}
320+
if (!prepared.isEmptyCursorOpen()) {
321+
ctx.getState().setError(ErrorCode.ERR_STMT_HAS_NO_OPEN_CURSOR,
322+
String.format("The statement (%d) has no open cursor.", stmtId));
323+
return;
324+
}
325+
prepared.setEmptyCursorOpen(false);
326+
ctx.getState().setEof();
327+
ctx.getState().serverStatus |= MysqlServerStatusFlag.SERVER_STATUS_LAST_ROW_SENT;
328+
}
329+
237330
// Process COM_QUERY statement,
238331
private void handleQuery() throws ConnectionException {
239332
// convert statement to Java string
@@ -278,6 +371,15 @@ private void dispatch() throws IOException {
278371
case COM_STMT_EXECUTE:
279372
handleExecute();
280373
break;
374+
case COM_STMT_FETCH:
375+
packetBuf.order(ByteOrder.LITTLE_ENDIAN);
376+
int fetchStmtId = packetBuf.getInt();
377+
packetBuf.getInt(); // Requested row count; an empty cursor has no rows to return.
378+
handleStmtFetch(fetchStmtId);
379+
break;
380+
case COM_STMT_SEND_LONG_DATA:
381+
handleSendLongData();
382+
break;
281383
case COM_FIELD_LIST:
282384
handleFieldList();
283385
break;

0 commit comments

Comments
 (0)