Skip to content

Commit 37f71c9

Browse files
committed
[fix](flight-sql) Validate prepared schemas and preserve command headers
1 parent a02347b commit 37f71c9

9 files changed

Lines changed: 333 additions & 15 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowCreateProcedureCommand.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -54,6 +54,12 @@ public ShowCreateProcedureCommand(FuncNameInfo procedureName) {
5454

5555
}
5656

57+
@Override
58+
public ShowResultSetMetaData getResultSetMetaData() {
59+
// These fixed headers are available before procedure lookup or execution.
60+
return getMetaData();
61+
}
62+
5763
public ShowResultSetMetaData getMetaData() {
5864
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
5965
for (String title : TITLE_NAMES) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcCommand.java‎

Lines changed: 17 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -24,6 +24,7 @@
2424
import org.apache.doris.common.AnalysisException;
2525
import org.apache.doris.common.ErrorCode;
2626
import org.apache.doris.common.ErrorReport;
27+
import org.apache.doris.common.proc.CurrentQueryStatisticsProcDir;
2728
import org.apache.doris.common.proc.ProcNodeInterface;
2829
import org.apache.doris.common.proc.ProcResult;
2930
import org.apache.doris.common.proc.ProcService;
@@ -35,6 +36,7 @@
3536
import org.apache.doris.qe.ShowResultSetMetaData;
3637
import org.apache.doris.qe.StmtExecutor;
3738

39+
import java.util.Collections;
3840
import java.util.List;
3941

4042
/**
@@ -74,13 +76,26 @@ public ShowResultSetMetaData getMetaData() {
7476
return ShowResultSetMetaData.builder().build();
7577
}
7678

77-
/** Resolve a proc node's header with the same privilege checks as SHOW PROC. */
79+
/** Resolve a static PROC header, or return null when discovery would require fetching rows. */
7880
public ShowResultSetMetaData getMetaData(ConnectContext ctx) throws AnalysisException {
7981
// PROC headers belong to the resolved node and require the same privilege as reading it.
8082
if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN_OR_NODE)) {
8183
ErrorReport.reportAnalysisException(ErrorCode.ERR_SPECIFIC_ACCESS_DENIED_ERROR, "ADMIN");
8284
}
83-
return getMetaData(ProcService.getInstance().open(path));
85+
// Even opening nested nodes can perform remote lookups. Only use known static headers.
86+
List<String> names;
87+
if ("/".equals(path)) {
88+
names = Collections.singletonList("name");
89+
} else if ("/current_queries".equals(path)) {
90+
names = CurrentQueryStatisticsProcDir.TITLE_NAMES;
91+
} else {
92+
return null;
93+
}
94+
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
95+
for (String name : names) {
96+
builder.addColumn(new Column(name, ScalarType.createVarchar(30)));
97+
}
98+
return builder.build();
8499
}
85100

86101
private ShowResultSet handleShowProc(ConnectContext ctx, StmtExecutor executor) throws Exception {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/ShowProcedureStatusCommand.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -67,6 +67,12 @@ public ShowProcedureStatusCommand(final Set<Expression> whereExpr) {
6767
this.whereExpr = Objects.requireNonNull(whereExpr, "whereExpr should not be null");
6868
}
6969

70+
@Override
71+
public ShowResultSetMetaData getResultSetMetaData() {
72+
// These fixed headers are available before procedure lookup or execution.
73+
return getMetaData();
74+
}
75+
7076
public ShowResultSetMetaData getMetaData() {
7177
ShowResultSetMetaData.Builder builder = ShowResultSetMetaData.builder();
7278
for (String title : TITLE_NAMES) {

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/WarmupSelectCommand.java‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -216,6 +216,12 @@ public void sendAggregatedBlackholeResults(Map<Long, TQueryStatisticsResult> sta
216216
executor.sendResultSet(resultSet);
217217
}
218218

219+
@Override
220+
public ShowResultSetMetaData getResultSetMetaData() {
221+
// Unlike ordinary INSERT, warmup returns per-backend statistics rather than OK.
222+
return metaData;
223+
}
224+
219225
/**
220226
* Skip auth check for warmup select command.
221227
* The blackhole table is a system table and doesn't require LOAD privilege.

‎fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java‎

Lines changed: 15 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -91,6 +91,7 @@
9191
import com.google.common.collect.Maps;
9292
import lombok.Getter;
9393
import lombok.Setter;
94+
import org.apache.arrow.vector.types.pojo.Schema;
9495
import org.apache.commons.lang3.StringUtils;
9596
import org.apache.logging.log4j.LogManager;
9697
import org.apache.logging.log4j.Logger;
@@ -160,11 +161,13 @@ private static class PreparedQuery {
160161
private final String sql;
161162
private final String catalog;
162163
private final String database;
164+
private final Schema schema;
163165

164-
private PreparedQuery(String sql, String catalog, String database) {
166+
private PreparedQuery(String sql, String catalog, String database, Schema schema) {
165167
this.sql = sql;
166168
this.catalog = catalog;
167169
this.database = database;
170+
this.schema = schema;
168171
}
169172
}
170173

@@ -922,7 +925,17 @@ public void resetLoginTime() {
922925
}
923926

924927
public synchronized void addPreparedQuery(String preparedStatementId, String preparedQuery) {
925-
preparedQuerys.put(preparedStatementId, new PreparedQuery(preparedQuery, getDefaultCatalog(), getDatabase()));
928+
addPreparedQuery(preparedStatementId, preparedQuery, null);
929+
}
930+
931+
public synchronized void addPreparedQuery(String preparedStatementId, String preparedQuery, Schema schema) {
932+
preparedQuerys.put(preparedStatementId,
933+
new PreparedQuery(preparedQuery, getDefaultCatalog(), getDatabase(), schema));
934+
}
935+
936+
public synchronized Schema getPreparedQuerySchema(String preparedStatementId) {
937+
PreparedQuery query = preparedQuerys.get(preparedStatementId);
938+
return query == null ? null : query.schema;
926939
}
927940

928941
public synchronized String getPreparedQuery(String preparedStatementId) {

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/DorisFlightSqlProducer.java‎

Lines changed: 16 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -79,6 +79,7 @@
7979
import org.apache.arrow.vector.ipc.WriteChannel;
8080
import org.apache.arrow.vector.ipc.message.MessageSerializer;
8181
import org.apache.arrow.vector.types.pojo.Schema;
82+
import org.apache.commons.lang3.tuple.Pair;
8283
import org.apache.logging.log4j.LogManager;
8384
import org.apache.logging.log4j.Logger;
8485

@@ -329,7 +330,7 @@ public FlightInfo getFlightInfoPreparedStatement(final CommandPreparedStatementQ
329330
ConnectContext connection = flightSessionsManager.getConnectContext(context.peerIdentity());
330331
synchronized (connection) {
331332
return executeQueryStatement(context.peerIdentity(), connection,
332-
preparedQuery(connection, context, command), descriptor);
333+
preparedQuery(connection, context, command).getLeft(), descriptor);
333334
}
334335
}
335336

@@ -345,24 +346,33 @@ public SchemaResult getSchemaPreparedStatement(final CommandPreparedStatementQue
345346
final CallContext context, final FlightDescriptor descriptor) {
346347
ConnectContext connection = flightSessionsManager.getConnectContext(context.peerIdentity());
347348
synchronized (connection) {
348-
return new SchemaResult(analyzeQuerySchema(connection, preparedQuery(connection, context, command)));
349+
return new SchemaResult(preparedQuery(connection, context, command).getRight());
349350
}
350351
}
351352

352-
private String preparedQuery(ConnectContext connection, CallContext context,
353+
private Pair<String, Schema> preparedQuery(ConnectContext connection, CallContext context,
353354
CommandPreparedStatementQuery command) {
354355
String prefix = context.peerIdentity() + ":";
355356
String handle = command.getPreparedStatementHandle().toStringUtf8();
356357
if (!handle.startsWith(prefix)) {
357358
throw CallStatus.INVALID_ARGUMENT.withDescription("Invalid prepared statement handle").toRuntimeException();
358359
}
359-
String query = connection.getPreparedQuery(handle.substring(prefix.length()));
360+
String id = handle.substring(prefix.length());
361+
String query = connection.getPreparedQuery(id);
360362
if (query == null) {
361363
throw CallStatus.NOT_FOUND
362364
.withDescription("Prepared statement expired; prepare again in the current namespace")
363365
.toRuntimeException();
364366
}
365-
return query;
367+
Schema schema = analyzeQuerySchema(connection, query);
368+
// Execution reparses SQL using the current session. Never silently replace the schema
369+
// advertised by Prepare when settings such as sql_mode or time_zone change its result.
370+
if (!schema.equals(connection.getPreparedQuerySchema(id))) {
371+
connection.removePreparedQuery(id);
372+
throw CallStatus.NOT_FOUND.withDescription("Prepared statement schema changed; prepare again")
373+
.toRuntimeException();
374+
}
375+
return Pair.of(query, schema);
366376
}
367377

368378
private Schema analyzeQuerySchema(ConnectContext context, String query) {
@@ -413,7 +423,7 @@ public void createPreparedStatement(final ActionCreatePreparedStatementRequest r
413423
ByteString handle = ByteString.copyFromUtf8(context.peerIdentity() + ":" + preparedStatementId);
414424
Result result = new Result(Any.pack(buildCreatePreparedStatementResult(handle,
415425
new Schema(Collections.emptyList()), schema)).toByteArray());
416-
connectContext.addPreparedQuery(preparedStatementId, query);
426+
connectContext.addPreparedQuery(preparedStatementId, query, schema);
417427
listener.onNext(result);
418428
listener.onCompleted();
419429
}

‎fe/fe-core/src/main/java/org/apache/doris/service/arrowflight/FlightSqlQuerySchema.java‎

Lines changed: 65 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -29,6 +29,7 @@
2929
import org.apache.doris.catalog.Type;
3030
import org.apache.doris.datasource.CatalogIf;
3131
import org.apache.doris.datasource.es.EsExternalCatalog;
32+
import org.apache.doris.datasource.lance.LanceExternalCatalog;
3233
import org.apache.doris.mysql.MysqlCommand;
3334
import org.apache.doris.mysql.privilege.PrivPredicate;
3435
import org.apache.doris.nereids.CascadesContext;
@@ -40,21 +41,36 @@
4041
import org.apache.doris.nereids.trees.expressions.Slot;
4142
import org.apache.doris.nereids.trees.plans.Plan;
4243
import org.apache.doris.nereids.trees.plans.PrepareCommandPlanner;
44+
import org.apache.doris.nereids.trees.plans.commands.AlterTableCommand;
4345
import org.apache.doris.nereids.trees.plans.commands.Command;
46+
import org.apache.doris.nereids.trees.plans.commands.DeleteFromCommand;
4447
import org.apache.doris.nereids.trees.plans.commands.DescribeCommand;
48+
import org.apache.doris.nereids.trees.plans.commands.ExplainCommand;
49+
import org.apache.doris.nereids.trees.plans.commands.KillCommand;
50+
import org.apache.doris.nereids.trees.plans.commands.ReplayCommand;
4551
import org.apache.doris.nereids.trees.plans.commands.ShowCreateTableCommand;
4652
import org.apache.doris.nereids.trees.plans.commands.ShowDataCommand;
4753
import org.apache.doris.nereids.trees.plans.commands.ShowPartitionsCommand;
4854
import org.apache.doris.nereids.trees.plans.commands.ShowProcCommand;
4955
import org.apache.doris.nereids.trees.plans.commands.ShowPythonPackagesCommand;
5056
import org.apache.doris.nereids.trees.plans.commands.ShowQueryStatsCommand;
5157
import org.apache.doris.nereids.trees.plans.commands.ShowTableCommand;
58+
import org.apache.doris.nereids.trees.plans.commands.TransactionCommand;
59+
import org.apache.doris.nereids.trees.plans.commands.UpdateCommand;
60+
import org.apache.doris.nereids.trees.plans.commands.info.CreateIndexOp;
61+
import org.apache.doris.nereids.trees.plans.commands.info.DropIndexOp;
62+
import org.apache.doris.nereids.trees.plans.commands.insert.BatchInsertIntoTableCommand;
63+
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTVFCommand;
64+
import org.apache.doris.nereids.trees.plans.commands.insert.InsertIntoTableCommand;
65+
import org.apache.doris.nereids.trees.plans.commands.insert.InsertOverwriteTableCommand;
66+
import org.apache.doris.nereids.trees.plans.commands.merge.MergeIntoCommand;
5267
import org.apache.doris.nereids.trees.plans.commands.use.SwitchCommand;
5368
import org.apache.doris.nereids.trees.plans.commands.use.UseCommand;
5469
import org.apache.doris.qe.ConnectContext;
5570
import org.apache.doris.qe.QueryState;
5671
import org.apache.doris.qe.ResultSetMetaData;
5772
import org.apache.doris.qe.SessionVariable;
73+
import org.apache.doris.qe.ShowResultSetMetaData;
5874
import org.apache.doris.qe.StmtExecutor;
5975
import org.apache.doris.qe.VariableMgr;
6076

@@ -97,7 +113,20 @@ static Schema analyze(ConnectContext context, String query) throws Exception {
97113
context.setStatementContext(null);
98114
// Match execution's HTTP/plugin conversion before the dialect parser sees the SQL.
99115
String converted = SqlDialectHelper.convertSqlByDialect(query, context.getSessionVariable());
100-
statements = new NereidsParser().parseSQL(converted, context.getSessionVariable());
116+
try {
117+
statements = new NereidsParser().parseSQL(converted, context.getSessionVariable());
118+
} catch (Exception convertedError) {
119+
if (!context.getSessionVariable().isRetryOriginSqlOnConvertFail() || converted.equals(query)) {
120+
throw convertedError;
121+
}
122+
// Match execution's parse fallback while discarding any failed parser context.
123+
StatementContext failed = context.getStatementContext();
124+
if (failed != null) {
125+
failed.close();
126+
context.setStatementContext(null);
127+
}
128+
statements = new NereidsParser().parseSQL(query, context.getSessionVariable());
129+
}
101130
Map<String, String> scopedDatabases = new HashMap<>();
102131
if (statements.isEmpty()) {
103132
throw CallStatus.UNIMPLEMENTED.withDescription(
@@ -219,7 +248,41 @@ private static ResultSetMetaData commandMetadata(ConnectContext context, Command
219248
} else if (command instanceof ShowProcCommand) {
220249
return ((ShowProcCommand) command).getMetaData(context);
221250
}
222-
return command.getResultSetMetaData();
251+
if (command instanceof ExplainCommand) {
252+
// PLAN PROCESS has no Flight serialization path in StmtExecutor.
253+
if (((ExplainCommand) command).showPlanProcess()) {
254+
throw CallStatus.UNIMPLEMENTED.withDescription("EXPLAIN PLAN PROCESS is not supported over Flight SQL")
255+
.toRuntimeException();
256+
}
257+
return stringMetadata("Explain String(Nereids Planner)");
258+
} else if (command instanceof ReplayCommand) {
259+
return stringMetadata("Plan Replayer dump url");
260+
} else if (command instanceof AlterTableCommand) {
261+
AlterTableCommand alter = (AlterTableCommand) command;
262+
String catalog = alter.getTbl().getCtl();
263+
// Lance index admission returns a JobId header even for an IF no-op. Do not run
264+
// validation/admission here: those paths can resolve remote tables or allocate IDs.
265+
if (context.getCatalog(catalog == null ? context.getDefaultCatalog() : catalog)
266+
instanceof LanceExternalCatalog && alter.getNereidsOps().stream().anyMatch(op ->
267+
(op instanceof CreateIndexOp && !((CreateIndexOp) op).isAlter())
268+
|| (op instanceof DropIndexOp && !((DropIndexOp) op).isAlter()))) {
269+
return stringMetadata("JobId");
270+
}
271+
}
272+
ResultSetMetaData metadata = command.getResultSetMetaData();
273+
// Concrete DML commands return OK, but subclasses such as WARM UP SELECT supply rows.
274+
if (metadata != null && metadata.getColumnCount() == 0 && (command instanceof InsertIntoTableCommand
275+
|| command instanceof InsertOverwriteTableCommand || command instanceof BatchInsertIntoTableCommand
276+
|| command instanceof InsertIntoTVFCommand || command instanceof UpdateCommand
277+
|| command instanceof DeleteFromCommand || command instanceof MergeIntoCommand
278+
|| command instanceof KillCommand || command instanceof TransactionCommand)) {
279+
return stringMetadata("StatusResult");
280+
}
281+
return metadata;
282+
}
283+
284+
private static ResultSetMetaData stringMetadata(String name) {
285+
return ShowResultSetMetaData.builder().addColumn(new Column(name, Type.STRING)).build();
223286
}
224287

225288
private static void resolveNamespace(ConnectContext context, Plan plan, Map<String, String> scopedDatabases)

0 commit comments

Comments
 (0)