Skip to content

Commit f109e13

Browse files
hutiefang76morningmanGabriel39
authored
[fix](arrow-flight) Support table type metadata and filters (#67899)
### What problem does this PR solve? Related to #67578 (partial A3). Flight SQL clients can obtain FlightInfo for GetTableTypes, but reading its ticket fails with UNIMPLEMENTED. GetTables also forwards only the first table_types entry to a service that treats only VIEW as a filter: [VIEW, BASE TABLE] loses base tables, while [BASE TABLE] or an unknown type can return excluded rows. Implement the type stream using the same Doris names as GetTables (BASE TABLE, SYSTEM VIEW and VIEW), sorted according to Flight SQL. Match the complete filter against the existing authorized listing before collecting optional schemas. Request only name/type metadata to avoid unused table statistics, and retain the session guard and Arrow buffer cleanup on failed sends. A type is the TABLE_TYPE that information_schema.tables reports for the object: table streams are VIEW, and external catalogs report their tables and views as BASE TABLE. The service lists every authorized object and the filter runs on that listing. Requested names match case-insensitively, and TABLE and SYSTEM TABLE, which Flight SQL calls commonly supported, select BASE TABLE and SYSTEM VIEW: the ADBC Flight SQL driver asks for `table` (Doris's own ADBC catalog lists a Doris source with `getObjects(..., {"table"})`), and JDBC tools ask for `TABLE`. Any other name matches only a type of that name, so UNKNOWN selects nothing. Because the filter now runs on the full listing, `FrontendServiceImpl.listTableStatus` refreshes a cloud follower's partition versions from the meta service only when UPDATE_TIME is requested, the one column that reads them. GetTables, which asks for names and types, never waits on those RPCs or is cut short when they fail, and information_schema.tables scans that do not read UPDATE_TIME skip the refresh too. Co-authored-by: morningman <[email protected]> Co-authored-by: Gabriel <[email protected]>
1 parent 173a0e2 commit f109e13

7 files changed

Lines changed: 601 additions & 11 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/arrowflight/DorisFlightSqlProducer.java‎

Lines changed: 11 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -619,7 +619,16 @@ public FlightInfo getFlightInfoTableTypes(final CommandGetTableTypes request, fi
619619

620620
@Override
621621
public void getStreamTableTypes(final CallContext context, final ServerStreamListener listener) {
622-
throw CallStatus.UNIMPLEMENTED.withDescription("getStreamTableTypes unimplemented").toRuntimeException();
622+
streamMetadata(context, listener, connectContext -> {
623+
try (VectorSchemaRoot vectorSchemaRoot = VectorSchemaRoot.create(
624+
Schemas.GET_TABLE_TYPES_SCHEMA, rootAllocator)) {
625+
listener.start(vectorSchemaRoot);
626+
vectorSchemaRoot.allocateNew();
627+
FlightSqlSchemaHelper.getTableTypes(vectorSchemaRoot);
628+
listener.putNext();
629+
listener.completed();
630+
}
631+
});
623632
}
624633

625634
@Override
@@ -761,7 +770,7 @@ private interface MetadataStream {
761770
void send(ConnectContext connectContext) throws Exception;
762771
}
763772

764-
// Answers a metadata request (catalogs, schemas, tables) from the session's context, as one
773+
// Answers a metadata request (catalogs, schemas, tables, table types) from the session's context, as one
765774
// command of that session.
766775
private void streamMetadata(final CallContext context, final ServerStreamListener listener,
767776
MetadataStream stream) {

‎fe/fe-core/src/main/java/org/apache/doris/arrowflight/FlightSqlSchemaHelper.java‎

Lines changed: 58 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.doris.catalog.Env;
2323
import org.apache.doris.catalog.MapType;
2424
import org.apache.doris.catalog.StructType;
25+
import org.apache.doris.catalog.TableIf.TableType;
2526
import org.apache.doris.catalog.Type;
2627
import org.apache.doris.datasource.CatalogIf;
2728
import org.apache.doris.qe.ConnectContext;
@@ -64,18 +65,27 @@
6465
import java.util.HashMap;
6566
import java.util.LinkedHashSet;
6667
import java.util.List;
68+
import java.util.Locale;
6769
import java.util.Map;
70+
import java.util.Objects;
6871
import java.util.Set;
72+
import java.util.stream.Collectors;
6973

7074
public class FlightSqlSchemaHelper {
7175
private static final Logger LOG = LogManager.getLogger(FlightSqlSchemaHelper.class);
76+
private static final List<String> TABLE_TYPES = Arrays.stream(TableType.values())
77+
.map(TableType::toMysqlType)
78+
.filter(Objects::nonNull)
79+
.distinct()
80+
.sorted()
81+
.collect(Collectors.toList());
7282
private final ConnectContext ctx;
7383
private final FrontendServiceImpl impl;
7484
private boolean includeSchema;
7585
private String catalogFilterPattern = null;
7686
private String dbSchemaFilterPattern = null;
7787
private String tableNameFilterPattern = null;
78-
private List<String> tableTypesList = null;
88+
private Set<String> tableTypeFilter = null;
7989

8090
public FlightSqlSchemaHelper(ConnectContext context) {
8191
ctx = context;
@@ -161,7 +171,28 @@ public void setParameterForGetTables(CommandGetTables command) {
161171
catalogFilterPattern = command.hasCatalog() ? command.getCatalog() : "internal";
162172
dbSchemaFilterPattern = command.hasDbSchemaFilterPattern() ? command.getDbSchemaFilterPattern() : null;
163173
tableNameFilterPattern = command.hasTableNameFilterPattern() ? command.getTableNameFilterPattern() : null;
164-
tableTypesList = command.getTableTypesList().isEmpty() ? null : command.getTableTypesList();
174+
tableTypeFilter = command.getTableTypesList().isEmpty() ? null
175+
: command.getTableTypesList().stream().map(FlightSqlSchemaHelper::toDorisTableType)
176+
.collect(Collectors.toSet());
177+
}
178+
179+
/**
180+
* Maps a type a GetTables request filters on to the name GetTableTypes advertises. Flight SQL leaves the
181+
* names to the server but calls TABLE, VIEW and SYSTEM TABLE the commonly supported ones, and clients send
182+
* those in either case: the ADBC Flight SQL driver asks for "table", JDBC tools for "TABLE". So a type
183+
* matches case-insensitively, and TABLE and SYSTEM TABLE select BASE TABLE and SYSTEM VIEW. Any other name
184+
* matches only a type of that name.
185+
*/
186+
private static String toDorisTableType(String requestedType) {
187+
String type = requestedType.toUpperCase(Locale.ROOT);
188+
switch (type) {
189+
case "TABLE":
190+
return "BASE TABLE";
191+
case "SYSTEM TABLE":
192+
return "SYSTEM VIEW";
193+
default:
194+
return type;
195+
}
165196
}
166197

167198
/**
@@ -199,9 +230,8 @@ private TListTableStatusResult listTableStatus(String dbName, String catalogName
199230
if (tableNameFilterPattern != null) {
200231
getTablesParams.setPattern(tableNameFilterPattern);
201232
}
202-
if (tableTypesList != null) {
203-
getTablesParams.setType(tableTypesList.get(0)); // currently only one type is supported.
204-
}
233+
// Flight SQL only consumes names and types, not the potentially expensive table statistics.
234+
getTablesParams.setRequiredColumns(Set.of("TABLE_NAME", "TABLE_TYPE"));
205235
getTablesParams.setCurrentUserIdent(ctx.getCurrentUserIdentity().toThrift());
206236
return impl.listTableStatus(getTablesParams);
207237
}
@@ -318,6 +348,17 @@ public void getSchemas(VectorSchemaRoot vectorSchemaRoot) throws TException {
318348
vectorSchemaRoot.setRowCount(getDbsResult.getDbs().size());
319349
}
320350

351+
/**
352+
* for FlightSqlProducer Schemas.GET_TABLE_TYPES_SCHEMA
353+
*/
354+
public static void getTableTypes(VectorSchemaRoot vectorSchemaRoot) {
355+
VarCharVector tableTypeVector = (VarCharVector) vectorSchemaRoot.getVector("table_type");
356+
for (int i = 0; i < TABLE_TYPES.size(); i++) {
357+
tableTypeVector.setSafe(i, new Text(TABLE_TYPES.get(i)));
358+
}
359+
vectorSchemaRoot.setRowCount(TABLE_TYPES.size());
360+
}
361+
321362
/**
322363
* for FlightSqlProducer Schemas.GET_TABLES_SCHEMA_NO_SCHEMA and Schemas.GET_TABLES_SCHEMA
323364
*/
@@ -333,12 +374,21 @@ public void getTables(VectorSchemaRoot vectorSchemaRoot) throws TException {
333374
for (int dbIndex = 0; dbIndex < getDbsResult.getDbs().size(); dbIndex++) {
334375
String dbName = getDbsResult.getDbs().get(dbIndex);
335376
String catalogName = getDbsResult.isSetCatalogs() ? getDbsResult.getCatalogs().get(dbIndex) : "";
336-
TListTableStatusResult listTableStatusResult = listTableStatus(dbName, catalogName);
377+
List<TTableStatus> tables = listTableStatus(dbName, catalogName).getTables();
378+
if (tableTypeFilter != null) {
379+
// The service's single-type filter only handles VIEW. Match all requested types here
380+
// against the authorized results, before collecting their optional schemas.
381+
tables = tables.stream().filter(table -> tableTypeFilter.contains(table.getType()))
382+
.collect(Collectors.toList());
383+
}
384+
if (tables.isEmpty()) {
385+
continue;
386+
}
337387

338388
Map<String, List<Field>> tableToFields;
339389
if (includeSchema) {
340390
List<String> tablesName = new ArrayList<>();
341-
for (TTableStatus tableStatus : listTableStatusResult.getTables()) {
391+
for (TTableStatus tableStatus : tables) {
342392
tablesName.add(tableStatus.getName());
343393
}
344394
TDescribeTablesResult describeTablesResult = describeTables(dbName, catalogName, tablesName);
@@ -347,7 +397,7 @@ public void getTables(VectorSchemaRoot vectorSchemaRoot) throws TException {
347397
tableToFields = null;
348398
}
349399

350-
for (TTableStatus tableStatus : listTableStatusResult.getTables()) {
400+
for (TTableStatus tableStatus : tables) {
351401
catalogNameVector.setSafe(tablesCount, new Text(catalogName));
352402
schemaNameVector.setSafe(tablesCount, new Text(dbName));
353403
tableNameVector.setSafe(tablesCount, new Text(tableStatus.getName()));

‎fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -756,8 +756,13 @@ public TListTableStatusResult listTableStatus(TGetTablesParams params) throws TE
756756
// the version needs to be updated.
757757
// Otherwise, the version will always be the old value
758758
// unless there is a query for the table in the follower node.
759+
// UPDATE_TIME is the only status column that reads the partition versions
760+
// (OlapTable.getUpdateTime() takes the partitions' visible version time), so a
761+
// request that does not need it, such as Flight SQL GetTables asking for names
762+
// and types, does not wait on the meta service.
759763
if (!Env.getCurrentEnv().isMaster() && Config.isCloudMode()
760-
&& table instanceof OlapTable) {
764+
&& table instanceof OlapTable
765+
&& needTableStatusColumn(requiredColumns, "UPDATE_TIME")) {
761766
OlapTable olapTable = (OlapTable) table;
762767
List<CloudPartition> partitions = olapTable.getAllPartitions().stream()
763768
.filter(p -> p instanceof CloudPartition)

0 commit comments

Comments
 (0)