Skip to content

Commit 06c21ce

Browse files
morningmanGabriel39claude
committed
[test](fe) Test forwarding prepared executions that reuse cached parameter types
### What problem does this PR solve? Issue Number: None Related PR: #68162, #69314 Problem Summary: A client that runs a server-side prepared statement again can leave out the parameter types (new_params_bind_flag = 0) and have the server reuse the cached ones. A follower forwards each COM_STMT_EXECUTE to the master, which prepares the statement again for every forwarded execution and therefore has no cached types. branch-4.1 fixed this in #68162 (the part not specific to Lance). On master, #69314 fixed it in MysqlConnectProcessor.handleExecute, which rewrites the packet it forwards with a complete type table, and its tests cover cached types, unsigned flags, NULL changes and type updates. Those tests use at most four parameters, so the NULL bitmap is always one byte. They also hand the master the buffer from the follower's context, not the request that FEOpExecutor builds and Thrift carries. This ports the test #68162 added, MysqlPreparedStatementForwardingTest, to master's structure. A ten-parameter statement runs five times on a follower connection. Types are sent with the first and the fourth execution and reused otherwise. One execution sets a NULL in each byte of the NULL bitmap, and an unsigned INT later becomes a BIGINT. Each execution is forwarded through FEOpExecutor.buildStmtForwardParams and a Thrift round trip and runs on a new proxy context with untyped placeholders. Every parameter must bind there as it did on the follower, including the unsigned flag, and the client's packet must be read to its end and left unchanged. ### Release note None ### Check List (For Author) - Test: Unit Test - New MysqlPreparedStatementForwardingTest passes, together with MysqlConnectProcessorLongDataTest, FEOpExecutorMysqlProtocolTest, MysqlConnectProcessorCursorFetchTest and MysqlConnectProcessorExecuteCleanupTest. - It fails when handleExecute does not rewrite the forwarded packet for cached types, and when the rewrite keeps only the first byte of the NULL bitmap; MysqlConnectProcessorLongDataTest does not catch the second case. - Behavior changed: No - Does this need documentation: No Co-authored-by: Gabriel <[email protected]> Co-Authored-By: Claude Opus 5.5 <[email protected]>
1 parent 7866f5a commit 06c21ce

1 file changed

Lines changed: 168 additions & 0 deletions

File tree

Lines changed: 168 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,168 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.qe;
19+
20+
import org.apache.doris.analysis.UserIdentity;
21+
import org.apache.doris.catalog.Env;
22+
import org.apache.doris.catalog.MysqlColType;
23+
import org.apache.doris.mysql.MysqlCapability;
24+
import org.apache.doris.mysql.MysqlCommand;
25+
import org.apache.doris.nereids.StatementContext;
26+
import org.apache.doris.nereids.trees.expressions.Expression;
27+
import org.apache.doris.nereids.trees.expressions.Placeholder;
28+
import org.apache.doris.nereids.trees.expressions.literal.Literal;
29+
import org.apache.doris.nereids.trees.plans.PlaceholderId;
30+
import org.apache.doris.nereids.trees.plans.commands.PrepareCommand;
31+
import org.apache.doris.system.SystemInfoService;
32+
import org.apache.doris.thrift.TMasterOpRequest;
33+
import org.apache.doris.thrift.TNetworkAddress;
34+
import org.apache.doris.thrift.TUniqueId;
35+
36+
import org.apache.thrift.TDeserializer;
37+
import org.apache.thrift.TSerializer;
38+
import org.junit.jupiter.api.Assertions;
39+
import org.junit.jupiter.api.Test;
40+
import org.mockito.MockedConstruction;
41+
import org.mockito.MockedStatic;
42+
import org.mockito.Mockito;
43+
44+
import java.nio.ByteBuffer;
45+
import java.nio.ByteOrder;
46+
import java.nio.charset.StandardCharsets;
47+
import java.util.ArrayList;
48+
import java.util.Arrays;
49+
import java.util.List;
50+
51+
/**
52+
* A follower forwards a COM_STMT_EXECUTE to the master in a TMasterOpRequest. The master prepares the statement
53+
* again for every forwarded execution, so it never has the parameter types a client caches after the first
54+
* execution: the request has to carry them.
55+
*/
56+
public class MysqlPreparedStatementForwardingTest {
57+
// More than eight, so that the NULL bitmap takes two bytes
58+
private static final int PARAMETER_COUNT = 10;
59+
60+
@Test
61+
public void testForwardedExecutionsRetainTypesAndNullBitmap() throws Exception {
62+
Env env = Mockito.mock(Env.class);
63+
Mockito.when(env.getSelfNode()).thenReturn(new SystemInfoService.HostInfo("127.0.0.1", 9010));
64+
try (MockedStatic<Env> mockedEnv = Mockito.mockStatic(Env.class);
65+
MockedConstruction<StmtExecutor> executors = Mockito.mockConstruction(StmtExecutor.class);
66+
MockedStatic<AuditLogHelper> audit = Mockito.mockStatic(AuditLogHelper.class)) {
67+
mockedEnv.when(Env::getCurrentEnv).thenReturn(env);
68+
ConnectContext follower = new ConnectContext();
69+
follower.setCurrentUserIdentity(UserIdentity.ROOT);
70+
follower.setRemoteIP("127.0.0.1");
71+
follower.setCapability(MysqlCapability.DEFAULT_CAPABILITY);
72+
PreparedStatementContext prepared = prepare(follower);
73+
for (int execution = 0; execution < 5; execution++) {
74+
// The client sends the parameter types with the first execution and when they change. The other
75+
// executions leave them out and have the server reuse the cached ones.
76+
boolean newTypes = execution == 0 || execution == 3;
77+
boolean wideInteger = execution >= 3;
78+
ByteBuffer packet = packet(execution, newTypes, wideInteger);
79+
byte[] original = Arrays.copyOf(packet.array(), packet.limit());
80+
execute(follower, prepared, packet);
81+
Assertions.assertEquals(packet.limit(), packet.position());
82+
Assertions.assertArrayEquals(original, Arrays.copyOf(packet.array(), packet.limit()));
83+
84+
TMasterOpRequest request = new FEOpExecutor(new TNetworkAddress("127.0.0.1", 9010),
85+
prepared.command.getOriginalStmt(), follower, true).buildStmtForwardParams();
86+
TMasterOpRequest restored = new TMasterOpRequest();
87+
new TDeserializer().deserialize(restored, new TSerializer().serialize(request));
88+
ConnectContext master = new ConnectContext(null, true);
89+
PreparedStatementContext reprepared = prepare(master);
90+
ByteBuffer forwarded = ByteBuffer.wrap(restored.getPrepareExecuteBuffer())
91+
.order(ByteOrder.LITTLE_ENDIAN);
92+
execute(master, reprepared, forwarded);
93+
Assertions.assertFalse(forwarded.hasRemaining());
94+
Assertions.assertEquals((execution + 1) * 2, executors.constructed().size());
95+
96+
for (int i = 0; i < PARAMETER_COUNT; i++) {
97+
Assertions.assertEquals(bound(prepared, i), bound(reprepared, i), "parameter " + i);
98+
}
99+
Literal integer = (Literal) bound(reprepared, 1);
100+
Assertions.assertEquals(wideInteger ? "1099511627776" : "4294967295", integer.getStringValue());
101+
Assertions.assertEquals(!wideInteger, reprepared.command.getPlaceholders().get(1).isUnsigned());
102+
}
103+
}
104+
}
105+
106+
private PreparedStatementContext prepare(ConnectContext context) throws Exception {
107+
List<Placeholder> placeholders = new ArrayList<>();
108+
for (int i = 0; i < PARAMETER_COUNT; i++) {
109+
placeholders.add(new Placeholder(new PlaceholderId(i)));
110+
}
111+
OriginStatement origin = new OriginStatement("select ?, ?, ?, ?, ?, ?, ?, ?, ?, ?", 0);
112+
PrepareCommand command = new PrepareCommand("7", null, placeholders, origin);
113+
PreparedStatementContext prepared = new PreparedStatementContext(command, context,
114+
new StatementContext(context, origin), "7");
115+
context.addPreparedStatementContext("7", prepared);
116+
// The query id the COM_STMT_PREPARE of the statement left on the connection; the mocked executors never
117+
// execute, so they do not set one.
118+
context.setQueryId(new TUniqueId(7L, 7L));
119+
return prepared;
120+
}
121+
122+
private void execute(ConnectContext context, PreparedStatementContext prepared, ByteBuffer packet) {
123+
context.setCommand(MysqlCommand.COM_STMT_EXECUTE);
124+
context.getState().reset();
125+
new MysqlConnectProcessor(context).handleExecute(prepared.command, 7, prepared, packet, null);
126+
Assertions.assertEquals(QueryState.MysqlStateType.OK, context.getState().getStateType(),
127+
context.getState().getErrorMessage());
128+
}
129+
130+
private Expression bound(PreparedStatementContext prepared, int parameter) {
131+
return prepared.getStatementContext().getIdToPlaceholderRealExpr()
132+
.get(prepared.command.getPlaceholders().get(parameter).getPlaceholderId());
133+
}
134+
135+
// The parameter part of a COM_STMT_EXECUTE: a string, an unsigned INT that later becomes a BIGINT, and eight
136+
// INTs, two of which, one in each byte of the NULL bitmap, are NULL in the second execution.
137+
private ByteBuffer packet(int execution, boolean newTypes, boolean wideInteger) {
138+
ByteBuffer packet = ByteBuffer.allocate(256).order(ByteOrder.LITTLE_ENDIAN);
139+
packet.position(9); // The MySQL dispatcher has already read the statement id, flags and iteration count.
140+
boolean withNulls = execution == 1;
141+
packet.put((byte) (withNulls ? 4 : 0));
142+
packet.put((byte) (withNulls ? 1 : 0));
143+
packet.put((byte) (newTypes ? 1 : 0));
144+
if (newTypes) {
145+
packet.putChar((char) MysqlColType.MYSQL_TYPE_VARSTRING.getCode());
146+
packet.putChar((char) (wideInteger ? MysqlColType.MYSQL_TYPE_LONGLONG.getCode()
147+
: MysqlColType.MYSQL_TYPE_LONG.getCode() | MysqlColType.UNSIGNED_MASK));
148+
for (int i = 2; i < PARAMETER_COUNT; i++) {
149+
packet.putChar((char) MysqlColType.MYSQL_TYPE_LONG.getCode());
150+
}
151+
}
152+
byte[] text = (execution == 1 ? "[3,4]" : "[1,2]").getBytes(StandardCharsets.UTF_8);
153+
packet.put((byte) text.length).put(text);
154+
if (wideInteger) {
155+
packet.putLong(1L << 40);
156+
} else {
157+
packet.putInt(-1);
158+
}
159+
for (int i = 2; i < PARAMETER_COUNT; i++) {
160+
if (!withNulls || (i != 2 && i != 8)) {
161+
packet.putInt(execution + i);
162+
}
163+
}
164+
packet.flip();
165+
packet.position(9);
166+
return packet;
167+
}
168+
}

0 commit comments

Comments
 (0)