Skip to content

Commit 18bfadc

Browse files
committed
[fix](flight-sql) Compare execution schemas using semantic column labels
1 parent a781cf9 commit 18bfadc

4 files changed

Lines changed: 289 additions & 1 deletion

File tree

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java‎

Lines changed: 9 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -24,8 +24,10 @@
2424
import org.apache.doris.common.util.DebugUtil;
2525
import org.apache.doris.common.util.Util;
2626
import org.apache.doris.mysql.MysqlCommand;
27+
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
2728
import org.apache.doris.qe.ConnectContext;
2829
import org.apache.doris.qe.QueryState.MysqlStateType;
30+
import org.apache.doris.qe.StmtExecutor;
2931
import org.apache.doris.service.arrowflight.results.FlightSqlEndpointsLocation;
3032
import org.apache.doris.service.arrowflight.results.FlightSqlResultCacheEntry;
3133
import org.apache.doris.service.arrowflight.sessions.FlightSessionsManager;
@@ -336,7 +338,13 @@ public FlightInfo getFlightInfoPreparedStatement(final CommandPreparedStatementQ
336338
.substring(context.peerIdentity().length() + 1);
337339
// Another session's DDL can change the result after reanalysis but before execution
338340
// acquires table locks. Reject the actual schema before publishing a DoGet ticket.
339-
if (!prepared.getRight().equals(info.getSchema())) {
341+
StmtExecutor executor = connection.getExecutor();
342+
boolean matches = !connection.isReturnResultFromLocal() && executor != null
343+
&& executor.getParsedStmt() instanceof LogicalPlanAdapter
344+
? FlightSqlQuerySchema.matchesExecutionSchema(prepared.getRight(), info.getSchema(),
345+
((LogicalPlanAdapter) executor.getParsedStmt()).getColLabels())
346+
: prepared.getRight().equals(info.getSchema());
347+
if (!matches) {
340348
connection.removePreparedQuery(id);
341349
try {
342350
connection.cancelQuery(Status.CANCELLED);

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java‎

Lines changed: 40 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -89,6 +89,7 @@
8989
import java.util.HashMap;
9090
import java.util.List;
9191
import java.util.Map;
92+
import java.util.Objects;
9293

9394
/** Resolves result metadata without scheduling fragments or evaluating query expressions. */
9495
final class FlightSqlQuerySchema {
@@ -234,6 +235,45 @@ static Schema analyze(ConnectContext context, String query) throws Exception {
234235
}
235236
}
236237

238+
static boolean matchesExecutionSchema(Schema prepared, Schema actual, List<String> columnLabels) {
239+
List<Field> expectedFields = prepared.getFields();
240+
List<Field> actualFields = actual.getFields();
241+
if (columnLabels == null || expectedFields.size() != actualFields.size()
242+
|| expectedFields.size() != columnLabels.size()
243+
|| !prepared.getCustomMetadata().equals(actual.getCustomMetadata())) {
244+
return false;
245+
}
246+
for (int i = 0; i < expectedFields.size(); ++i) {
247+
Field expected = expectedFields.get(i);
248+
// Compare the final planner's semantic labels: BE labels may instead contain SQL
249+
// text or type_name_index. Ignoring all names would hide a concurrent column rename.
250+
if (!expected.getName().equals(columnLabels.get(i))
251+
|| !matchesExecutionField(expected, actualFields.get(i))) {
252+
return false;
253+
}
254+
}
255+
return true;
256+
}
257+
258+
private static boolean matchesExecutionField(Field expected, Field actual) {
259+
// Rewrites can prove an expression non-null (e.g. a folded CAST). Such narrowing is
260+
// compatible with Prepare's nullable field; the reverse violates its advertised contract.
261+
if ((!expected.isNullable() && actual.isNullable()) || !expected.getType().equals(actual.getType())
262+
|| !expected.getMetadata().equals(actual.getMetadata())
263+
|| !Objects.equals(expected.getFieldType().getDictionary(), actual.getFieldType().getDictionary())
264+
|| expected.getChildren().size() != actual.getChildren().size()) {
265+
return false;
266+
}
267+
for (int i = 0; i < expected.getChildren().size(); ++i) {
268+
Field child = expected.getChildren().get(i);
269+
Field actualChild = actual.getChildren().get(i);
270+
if (!child.getName().equals(actualChild.getName()) || !matchesExecutionField(child, actualChild)) {
271+
return false;
272+
}
273+
}
274+
return true;
275+
}
276+
237277
private static ResultSetMetaData commandMetadata(ConnectContext context, Command command) throws Exception {
238278
// These getters depend on execution-time state or remote responses. Do not advertise a
239279
// guessed schema, or run the command merely to discover it.

‎fe/fe-core/src/test/java/org/apache/doris/service/arrowflight/DorisFlightSqlSchemaTest.java‎

Lines changed: 62 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.doris.datasource.test.TestExternalCatalog;
2828
import org.apache.doris.mysql.MysqlCommand;
2929
import org.apache.doris.nereids.StatementContext;
30+
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
3031
import org.apache.doris.nereids.parser.SqlDialectHelper;
3132
import org.apache.doris.qe.ConnectContext;
3233
import org.apache.doris.qe.SessionVariable;
@@ -62,6 +63,7 @@
6263

6364
import java.io.ByteArrayInputStream;
6465
import java.nio.channels.Channels;
66+
import java.util.ArrayList;
6567
import java.util.Arrays;
6668
import java.util.Collections;
6769
import java.util.List;
@@ -688,4 +690,64 @@ void executionSchemaDriftExpiresHandleAndCleansResults() throws Exception {
688690
}
689691
}
690692

693+
@Test
694+
void backendWireLabelsAndNarrowedNullabilityDoNotExpirePreparedQueries() throws Exception {
695+
Schema actual = new Schema(Arrays.asList(
696+
Field.notNullable("Int64_0", new ArrowType.Int(64, true)),
697+
Field.notNullable("String_1", new ArrowType.Utf8())));
698+
assertPreparedExecutionSchema("SELECT id, name FROM schema_input WHERE name IS NOT NULL",
699+
Arrays.asList("id", "name"), actual, true);
700+
}
701+
702+
@Test
703+
void actualSemanticRenameAndNullableWideningStillExpirePreparedQueries() throws Exception {
704+
Schema renamed = new Schema(Collections.singletonList(
705+
Field.notNullable("Int64_0", new ArrowType.Int(64, true))));
706+
assertPreparedExecutionSchema("SELECT id FROM schema_input", Collections.singletonList("renamed"),
707+
renamed, false);
708+
Schema widened = new Schema(Collections.singletonList(
709+
Field.nullable("id", new ArrowType.Int(64, true))));
710+
assertPreparedExecutionSchema("SELECT id FROM schema_input", Collections.singletonList("id"),
711+
widened, false);
712+
}
713+
714+
private void assertPreparedExecutionSchema(String query, List<String> labels, Schema actual, boolean compatible)
715+
throws Exception {
716+
StmtExecutor previous = connectContext.getExecutor();
717+
ActionCreatePreparedStatementResult result = prepare(query);
718+
CommandPreparedStatementQuery command = CommandPreparedStatementQuery.newBuilder()
719+
.setPreparedStatementHandle(result.getPreparedStatementHandle()).build();
720+
StmtExecutor executor = Mockito.mock(StmtExecutor.class);
721+
LogicalPlanAdapter statement = Mockito.mock(LogicalPlanAdapter.class);
722+
Mockito.when(statement.getColLabels()).thenReturn(new ArrayList<>(labels));
723+
Mockito.when(executor.getParsedStmt()).thenReturn(statement);
724+
try (MockedConstruction<FlightSqlConnectProcessor> processors = Mockito.mockConstruction(
725+
FlightSqlConnectProcessor.class, (processor, context) -> {
726+
Mockito.doAnswer(invocation -> {
727+
connectContext.setExecutor(executor);
728+
connectContext.setReturnResultFromLocal(false);
729+
return null;
730+
}).when(processor).handleQuery(Mockito.anyString());
731+
Mockito.when(processor.getArrowSchema()).thenReturn(actual);
732+
})) {
733+
if (compatible) {
734+
Assertions.assertEquals(actual, producer.getFlightInfoPreparedStatement(command,
735+
callContext, FlightDescriptor.command(new byte[0])).getSchema());
736+
Assertions.assertEquals(schema(query), producer.getSchemaPreparedStatement(command,
737+
callContext, FlightDescriptor.command(new byte[0])).getSchema());
738+
Mockito.verify(executor, Mockito.never()).cancel(Mockito.any(Status.class));
739+
} else {
740+
FlightRuntimeException error = Assertions.assertThrows(FlightRuntimeException.class,
741+
() -> producer.getFlightInfoPreparedStatement(command, callContext,
742+
FlightDescriptor.command(new byte[0])));
743+
Assertions.assertEquals(FlightStatusCode.NOT_FOUND, error.status().code());
744+
Mockito.verify(executor).cancel(Mockito.any(Status.class));
745+
}
746+
} finally {
747+
connectContext.setExecutor(previous);
748+
String handle = result.getPreparedStatementHandle().toStringUtf8();
749+
connectContext.removePreparedQuery(handle.substring(handle.indexOf(':') + 1));
750+
}
751+
}
752+
691753
}
Lines changed: 178 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,178 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.service.arrowflight;
19+
20+
import org.apache.doris.analysis.Expr;
21+
import org.apache.doris.catalog.ScalarType;
22+
import org.apache.doris.common.FeConstants;
23+
import org.apache.doris.nereids.NereidsPlanner;
24+
import org.apache.doris.nereids.StatementContext;
25+
import org.apache.doris.nereids.glue.LogicalPlanAdapter;
26+
import org.apache.doris.nereids.parser.NereidsParser;
27+
import org.apache.doris.qe.OriginStatement;
28+
import org.apache.doris.service.arrowflight.sessions.FlightSqlConnectContext;
29+
import org.apache.doris.thrift.TExprNode;
30+
import org.apache.doris.thrift.TExprNodeType;
31+
import org.apache.doris.thrift.TUniqueId;
32+
import org.apache.doris.utframe.TestWithFeService;
33+
34+
import org.apache.arrow.vector.types.pojo.ArrowType;
35+
import org.apache.arrow.vector.types.pojo.Field;
36+
import org.apache.arrow.vector.types.pojo.FieldType;
37+
import org.apache.arrow.vector.types.pojo.Schema;
38+
import org.junit.jupiter.api.Assertions;
39+
import org.junit.jupiter.api.Test;
40+
41+
import java.util.ArrayList;
42+
import java.util.Collections;
43+
import java.util.List;
44+
45+
class FlightSqlExecutionSchemaTest extends TestWithFeService {
46+
private FlightSqlConnectContext flightContext;
47+
private boolean previousUnitTest;
48+
49+
@Override
50+
protected void runBeforeAll() throws Exception {
51+
previousUnitTest = FeConstants.runningUnitTest;
52+
FeConstants.runningUnitTest = true;
53+
createDatabase("execution_schema_test");
54+
connectContext.setDatabase("execution_schema_test");
55+
createTable("CREATE TABLE schema_input (id INT NOT NULL, encoded STRING NULL) "
56+
+ "DISTRIBUTED BY HASH(id) BUCKETS 1 PROPERTIES ('replication_num' = '1')");
57+
flightContext = new FlightSqlConnectContext("schema-peer");
58+
flightContext.setEnv(connectContext.getEnv());
59+
flightContext.setCurrentUserIdentity(connectContext.getCurrentUserIdentity());
60+
flightContext.setSessionVariable(connectContext.getSessionVariable());
61+
flightContext.setDatabase("execution_schema_test");
62+
flightContext.setQueryId(new TUniqueId(1, 2));
63+
}
64+
65+
@Override
66+
protected void runAfterAll() throws Exception {
67+
FeConstants.runningUnitTest = previousUnitTest;
68+
flightContext.getFlightSqlChannel().close();
69+
}
70+
71+
@Test
72+
void unaliasedLiteralUsesDifferentWireLabel() throws Exception {
73+
assertPlannerSchema("SELECT 1", true, false);
74+
assertPlannerSchema("SELECT 1 AS id", false, false);
75+
}
76+
77+
@Test
78+
void unaliasedFunctionUsesDifferentWireLabel() throws Exception {
79+
assertPlannerSchema("SELECT unhex(encoded) FROM schema_input WHERE id <= 4 ORDER BY id", true, false);
80+
}
81+
82+
@Test
83+
void constantFoldingNarrowsNullableCast() throws Exception {
84+
assertPlannerSchema("SELECT CAST(CAST('2026-01-01' AS DATETIME(6)) AS STRING)", true, true);
85+
}
86+
87+
@Test
88+
void nullableColumnRemainsCompatibleWithFilter() throws Exception {
89+
assertPlannerSchema("SELECT encoded FROM schema_input WHERE encoded IS NOT NULL", false, false);
90+
}
91+
92+
@Test
93+
void tableValuedFunctionUsesQualifiedWireLabel() throws Exception {
94+
Schema actual = assertPlannerSchema("SELECT * FROM numbers('number' = '1')", true, false);
95+
Assertions.assertTrue(actual.getFields().get(0).getName().startsWith("_tvf_numbers."));
96+
}
97+
98+
private Schema assertPlannerSchema(String sql, boolean differentLabel, boolean narrowedNullability)
99+
throws Exception {
100+
flightContext.setThreadLocalInfo();
101+
Schema prepared = FlightSqlQuerySchema.analyze(flightContext, sql);
102+
try (StatementContext statementContext = new StatementContext(flightContext, new OriginStatement(sql, 0))) {
103+
flightContext.setStatementContext(statementContext);
104+
LogicalPlanAdapter statement = new LogicalPlanAdapter(
105+
new NereidsParser().parseSingle(sql), statementContext);
106+
statementContext.setParsedStatement(statement);
107+
new NereidsPlanner(statementContext).plan(statement, flightContext.getSessionVariable().toThrift());
108+
List<Field> wireFields = new ArrayList<>();
109+
for (Expr expression : statement.getResultExprs()) {
110+
TExprNode node = expression.treeToThrift().getNodes().get(0);
111+
Assertions.assertEquals(TExprNodeType.SLOT_REF, node.getNodeType());
112+
ScalarType type = (ScalarType) expression.getType();
113+
ArrowType arrowType = FlightSqlSchemaHelper.getArrowType(type.getPrimitiveType(),
114+
type.getScalarPrecision(), type.getScalarScale());
115+
// BE derives these fields from the serialized output expressions, not analyzed Slot names.
116+
wireFields.add(new Field(node.getLabel(),
117+
new FieldType(node.isIsNullable(), arrowType, null), Collections.emptyList()));
118+
}
119+
Schema actual = new Schema(wireFields);
120+
Assertions.assertEquals(differentLabel,
121+
!prepared.getFields().get(0).getName().equals(actual.getFields().get(0).getName()));
122+
Assertions.assertEquals(narrowedNullability,
123+
prepared.getFields().get(0).isNullable() && !actual.getFields().get(0).isNullable());
124+
Assertions.assertTrue(FlightSqlQuerySchema.matchesExecutionSchema(
125+
prepared, actual, statement.getColLabels()));
126+
return actual;
127+
} finally {
128+
flightContext.setStatementContext(null);
129+
connectContext.setThreadLocalInfo();
130+
}
131+
}
132+
133+
@Test
134+
void sameArityRenameIsRejectedDespiteIdenticalWireTypes() {
135+
Schema schema = schema(Field.nullable("old_name", new ArrowType.Utf8()));
136+
Assertions.assertFalse(FlightSqlQuerySchema.matchesExecutionSchema(schema, schema,
137+
Collections.singletonList("new_name")));
138+
}
139+
140+
@Test
141+
void differentTypesAreRejected() {
142+
Schema prepared = schema(Field.nullable("value", new ArrowType.Int(32, true)));
143+
Schema actual = schema(Field.nullable("value", new ArrowType.Int(64, true)));
144+
Assertions.assertFalse(FlightSqlQuerySchema.matchesExecutionSchema(prepared, actual,
145+
Collections.singletonList("value")));
146+
}
147+
148+
@Test
149+
void nestedFieldNamesAreNotTransportLabels() {
150+
Schema prepared = schema(new Field("value", FieldType.nullable(new ArrowType.Struct()),
151+
Collections.singletonList(Field.nullable("old_name", new ArrowType.Utf8()))));
152+
Schema actual = schema(new Field("value", FieldType.nullable(new ArrowType.Struct()),
153+
Collections.singletonList(Field.nullable("new_name", new ArrowType.Utf8()))));
154+
Assertions.assertFalse(FlightSqlQuerySchema.matchesExecutionSchema(prepared, actual,
155+
Collections.singletonList("value")));
156+
}
157+
158+
@Test
159+
void typeMetadataChangesAreRejected() {
160+
Schema prepared = schema(new Field("value", new FieldType(true, new ArrowType.Utf8(), null,
161+
Collections.singletonMap("doris_type", "LARGEINT")), Collections.emptyList()));
162+
Schema actual = schema(Field.nullable("value", new ArrowType.Utf8()));
163+
Assertions.assertFalse(FlightSqlQuerySchema.matchesExecutionSchema(prepared, actual,
164+
Collections.singletonList("value")));
165+
}
166+
167+
@Test
168+
void nullableWideningIsRejected() {
169+
Schema prepared = schema(Field.notNullable("value", new ArrowType.Utf8()));
170+
Schema actual = schema(Field.nullable("value", new ArrowType.Utf8()));
171+
Assertions.assertFalse(FlightSqlQuerySchema.matchesExecutionSchema(prepared, actual,
172+
Collections.singletonList("value")));
173+
}
174+
175+
private static Schema schema(Field field) {
176+
return new Schema(Collections.singletonList(field));
177+
}
178+
}

0 commit comments

Comments
 (0)