Skip to content

Commit 6ce6ae7

Browse files
committed
[fix](fe) Avoid database object loading for schemata scans
### What problem does this PR solve? Issue Number: N/A Problem Summary: SCHEMATA only consumes database names, but its metadata RPC materializes every database object to compute unused IDs. Let this scanner opt out of IDs through an optional request flag while preserving existing ID-inclusive callers and privilege filtering. ### Release note Improve information_schema.schemata enumeration for large catalogs. ### Check List (For Author) - Test: 7 targeted FE tests; FE Checkstyle; clang-format 16; BE scanner syntax check; synthetic 100,000-namespace FE-handler benchmark. - Behavior changed: No SQL result-semantic changes. - Does this need documentation: No.
1 parent bdb9085 commit 6ce6ae7

4 files changed

Lines changed: 229 additions & 15 deletions

File tree

‎be/src/information_schema/schema_schemata_scanner.cpp‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -52,6 +52,8 @@ Status SchemaSchemataScanner::start(RuntimeState* state) {
5252
return Status::InternalError("used before initial.");
5353
}
5454
TGetDbsParams db_params;
55+
// SCHEMATA never reads database IDs; avoid creating one database object per namespace in FE.
56+
db_params.__set_include_db_ids(false);
5557
if (nullptr != _param->common_param->wild) {
5658
db_params.__set_pattern(*(_param->common_param->wild));
5759
}

‎fe/fe-core/src/main/java/org/apache/doris/service/FrontendServiceImpl.java‎

Lines changed: 39 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -362,7 +362,6 @@
362362
import java.util.Collections;
363363
import java.util.Comparator;
364364
import java.util.HashMap;
365-
import java.util.HashSet;
366365
import java.util.List;
367366
import java.util.Locale;
368367
import java.util.Map;
@@ -379,6 +378,7 @@
379378
import java.util.concurrent.atomic.AtomicInteger;
380379
import java.util.concurrent.atomic.AtomicLong;
381380
import java.util.concurrent.locks.ReentrantLock;
381+
import java.util.function.ObjLongConsumer;
382382
import java.util.stream.Collectors;
383383

384384
// Frontend service used to serve all request for this frontend through
@@ -512,14 +512,16 @@ public TGetDbsResult getDbNames(TGetDbsParams params) throws TException {
512512
List<Long> dbIds = Lists.newArrayList();
513513
List<Long> catalogIds = Lists.newArrayList();
514514

515-
PatternMatcher matcher = null;
515+
PatternMatcher matcher;
516516
if (params.isSetPattern()) {
517517
try {
518518
matcher = PatternMatcher.createMysqlPattern(params.getPattern(),
519519
CaseSensibility.DATABASE.getCaseSensibility());
520520
} catch (PatternMatcherException e) {
521521
throw new TException("Pattern is in bad format: " + params.getPattern());
522522
}
523+
} else {
524+
matcher = null;
523525
}
524526

525527
Env env = Env.getCurrentEnv();
@@ -533,55 +535,77 @@ public TGetDbsResult getDbNames(TGetDbsParams params) throws TException {
533535
catalog -> new TException("Unknown catalog " + catalog)));
534536
}
535537

538+
boolean includeDbIds = params.isIncludeDbIds();
536539
for (CatalogIf catalog : catalogIfs) {
537-
Collection<DatabaseIf> dbs = new HashSet<DatabaseIf>();
540+
Collection<DatabaseIf> dbs = Collections.emptyList();
541+
List<String> names = Collections.emptyList();
538542
try {
539-
dbs = catalog.getAllDbs();
543+
if (includeDbIds) {
544+
dbs = catalog.getAllDbs();
545+
} else {
546+
// SCHEMATA only needs names; loading every database churns the object cache on large catalogs.
547+
names = catalog.getDbNames();
548+
}
540549
} catch (Exception e) {
541550
LOG.warn("failed to get database names for catalog {}", catalog.getName(), e);
542551
// Some external catalog may fail to get databases due to wrong connection info.
543552
}
553+
int dbCount = includeDbIds ? dbs.size() : names.size();
544554
if (LOG.isDebugEnabled()) {
545-
LOG.debug("get db size: {}, in catalog: {}", dbs.size(), catalog.getName());
555+
LOG.debug("get db size: {}, in catalog: {}", dbCount, catalog.getName());
546556
}
547-
if (dbs.isEmpty() && params.isSetGetNullCatalog() && params.get_null_catalog) {
557+
if (dbCount == 0 && params.isSetGetNullCatalog() && params.get_null_catalog) {
548558
catalogNames.add(catalog.getName());
549559
dbNames.add("NULL");
550560
catalogIds.add(catalog.getId());
551-
dbIds.add(-1L);
561+
if (includeDbIds) {
562+
dbIds.add(-1L);
563+
}
552564
continue;
553565
}
554-
if (dbs.isEmpty()) {
566+
if (dbCount == 0) {
555567
continue;
556568
}
557-
UserIdentity currentUser = null;
569+
UserIdentity currentUser;
558570
if (params.isSetCurrentUserIdent()) {
559571
currentUser = UserIdentity.fromThrift(params.current_user_ident);
560572
} else {
561573
currentUser = UserIdentity.createAnalyzedUserIdentWithIp(params.user, params.user_ip);
562574
}
563-
for (DatabaseIf db : dbs) {
564-
String dbName = db.getFullName();
575+
ObjLongConsumer<String> addDatabase = (dbName, dbId) -> {
565576
if (!env.getAccessManager()
566577
.checkDbPriv(currentUser, catalog.getName(), dbName, PrivPredicate.SHOW)) {
567-
continue;
578+
return;
568579
}
569580

570581
if (matcher != null && !matcher.match(getMysqlTableSchema(catalog.getName(), dbName))) {
571-
continue;
582+
return;
572583
}
573584

574585
catalogNames.add(catalog.getName());
575586
dbNames.add(getMysqlTableSchema(catalog.getName(), dbName));
576587
catalogIds.add(catalog.getId());
577-
dbIds.add(db.getId());
588+
if (includeDbIds) {
589+
dbIds.add(dbId);
590+
}
591+
};
592+
if (includeDbIds) {
593+
for (DatabaseIf db : dbs) {
594+
addDatabase.accept(db.getFullName(), db.getId());
595+
}
596+
} else {
597+
for (String dbName : names) {
598+
addDatabase.accept(dbName, -1L);
599+
}
578600
}
579601
}
580602

581603
result.setDbs(dbNames);
582604
result.setCatalogs(catalogNames);
583605
result.setCatalogIds(catalogIds);
584-
result.setDbIds(dbIds);
606+
if (includeDbIds) {
607+
result.setDbIds(dbIds);
608+
}
585609
return result;
586610
}
587611

Lines changed: 186 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,186 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.service;
19+
20+
import org.apache.doris.analysis.UserIdentity;
21+
import org.apache.doris.catalog.DatabaseIf;
22+
import org.apache.doris.catalog.Env;
23+
import org.apache.doris.catalog.InfoSchemaDb;
24+
import org.apache.doris.datasource.CatalogIf;
25+
import org.apache.doris.datasource.CatalogMgr;
26+
import org.apache.doris.mysql.privilege.AccessControllerManager;
27+
import org.apache.doris.mysql.privilege.PrivPredicate;
28+
import org.apache.doris.qe.GlobalVariable;
29+
import org.apache.doris.thrift.TGetDbsParams;
30+
import org.apache.doris.thrift.TGetDbsResult;
31+
32+
import org.apache.thrift.TDeserializer;
33+
import org.apache.thrift.TSerializer;
34+
import org.apache.thrift.protocol.TBinaryProtocol;
35+
import org.junit.jupiter.api.AfterEach;
36+
import org.junit.jupiter.api.Assertions;
37+
import org.junit.jupiter.api.BeforeEach;
38+
import org.junit.jupiter.api.Test;
39+
import org.mockito.MockedStatic;
40+
import org.mockito.Mockito;
41+
42+
import java.util.ArrayList;
43+
import java.util.Arrays;
44+
import java.util.Collections;
45+
import java.util.List;
46+
47+
public class FrontendServiceGetDbNamesTest {
48+
private final Env env = Mockito.mock(Env.class);
49+
private final CatalogMgr catalogMgr = Mockito.mock(CatalogMgr.class);
50+
private final AccessControllerManager accessManager = Mockito.mock(AccessControllerManager.class);
51+
private final CatalogIf catalog = Mockito.mock(CatalogIf.class);
52+
private final FrontendServiceImpl service = new FrontendServiceImpl(Mockito.mock(ExecuteEnv.class));
53+
private MockedStatic<Env> envMock;
54+
55+
@BeforeEach
56+
public void setUp() {
57+
envMock = Mockito.mockStatic(Env.class);
58+
envMock.when(Env::getCurrentEnv).thenReturn(env);
59+
Mockito.when(env.getCatalogMgr()).thenReturn(catalogMgr);
60+
Mockito.when(env.getAccessManager()).thenReturn(accessManager);
61+
Mockito.when(catalogMgr.listCatalogs()).thenReturn(Collections.singletonList(catalog));
62+
Mockito.when(catalog.getName()).thenReturn("example_catalog");
63+
Mockito.when(catalog.getId()).thenReturn(42L);
64+
Mockito.when(accessManager.checkDbPriv(Mockito.any(UserIdentity.class), Mockito.anyString(),
65+
Mockito.anyString(), Mockito.eq(PrivPredicate.SHOW))).thenReturn(true);
66+
}
67+
68+
@AfterEach
69+
public void tearDown() {
70+
envMock.close();
71+
}
72+
73+
@Test
74+
public void testNamesOnlyDoesNotMaterializeDatabases() throws Exception {
75+
List<String> names = new ArrayList<>();
76+
for (int i = 0; i < 100_000; i++) {
77+
names.add("namespace_" + i);
78+
}
79+
names.add(InfoSchemaDb.DATABASE_NAME);
80+
names.add("mysql");
81+
Mockito.when(catalog.getDbNames()).thenReturn(names);
82+
Mockito.when(catalog.getAllDbs()).thenThrow(new AssertionError("Database objects must not be loaded"));
83+
84+
TGetDbsParams params = params();
85+
params.setIncludeDbIds(false);
86+
TGetDbsResult result = service.getDbNames(params);
87+
88+
Assertions.assertEquals(names.size(), result.getDbsSize());
89+
Assertions.assertEquals(names.size(), result.getCatalogsSize());
90+
Assertions.assertEquals(names.size(), result.getCatalogIdsSize());
91+
Assertions.assertTrue(result.getCatalogs().stream().allMatch("example_catalog"::equals));
92+
Assertions.assertTrue(result.getCatalogIds().stream().allMatch(id -> id == 42L));
93+
Assertions.assertFalse(result.isSetDbIds());
94+
Mockito.verify(catalog, Mockito.times(1)).getDbNames();
95+
Mockito.verify(catalog, Mockito.never()).getAllDbs();
96+
}
97+
98+
@Test
99+
public void testNamesOnlyPreservesPrivilegesPatternAndDisplayNames() throws Exception {
100+
Mockito.when(catalog.getDbNames()).thenReturn(Arrays.asList("visible_a", "visible_hidden", "other"));
101+
Mockito.when(accessManager.checkDbPriv(Mockito.any(UserIdentity.class), Mockito.eq("example_catalog"),
102+
Mockito.eq("visible_hidden"), Mockito.eq(PrivPredicate.SHOW))).thenReturn(false);
103+
104+
boolean original = GlobalVariable.showFullDbNameInInfoSchemaDb;
105+
try {
106+
for (boolean showFullName : new boolean[] {false, true}) {
107+
GlobalVariable.showFullDbNameInInfoSchemaDb = showFullName;
108+
TGetDbsParams params = params();
109+
params.setIncludeDbIds(false);
110+
params.setPattern(showFullName ? "example_catalog.visible%" : "visible%");
111+
TGetDbsResult result = service.getDbNames(params);
112+
Assertions.assertEquals(Collections.singletonList(
113+
showFullName ? "example_catalog.visible_a" : "visible_a"), result.getDbs());
114+
Assertions.assertFalse(result.isSetDbIds());
115+
}
116+
} finally {
117+
GlobalVariable.showFullDbNameInInfoSchemaDb = original;
118+
}
119+
}
120+
121+
@Test
122+
public void testExistingCallersKeepDatabaseIds() throws Exception {
123+
DatabaseIf database = Mockito.mock(DatabaseIf.class);
124+
Mockito.when(database.getFullName()).thenReturn("visible_a");
125+
Mockito.when(database.getId()).thenReturn(123L);
126+
Mockito.when(catalog.getAllDbs()).thenReturn(Collections.singletonList(database));
127+
128+
for (boolean explicitlySet : new boolean[] {false, true}) {
129+
TGetDbsParams params = params();
130+
if (explicitlySet) {
131+
params.setIncludeDbIds(true);
132+
}
133+
TGetDbsResult result = service.getDbNames(params);
134+
Assertions.assertEquals(Collections.singletonList(123L), result.getDbIds());
135+
Assertions.assertEquals(Collections.singletonList(42L), result.getCatalogIds());
136+
}
137+
Mockito.verify(catalog, Mockito.never()).getDbNames();
138+
}
139+
140+
@Test
141+
public void testAbsentIncludeDbIdsOnWireKeepsLegacyDefault() throws Exception {
142+
TGetDbsParams legacy = params();
143+
legacy.unsetIncludeDbIds();
144+
TSerializer serializer = new TSerializer(new TBinaryProtocol.Factory());
145+
TDeserializer deserializer = new TDeserializer(new TBinaryProtocol.Factory());
146+
TGetDbsParams decoded = new TGetDbsParams();
147+
deserializer.deserialize(decoded, serializer.serialize(legacy));
148+
Assertions.assertTrue(decoded.isIncludeDbIds());
149+
150+
legacy.setIncludeDbIds(false);
151+
deserializer.deserialize(decoded, serializer.serialize(legacy));
152+
Assertions.assertFalse(decoded.isIncludeDbIds());
153+
}
154+
155+
@Test
156+
public void testNamesOnlyPreservesEmptyCatalogPlaceholder() throws Exception {
157+
Mockito.when(catalog.getDbNames()).thenReturn(Collections.emptyList());
158+
TGetDbsParams params = params();
159+
params.setIncludeDbIds(false);
160+
params.setGetNullCatalog(true);
161+
162+
TGetDbsResult result = service.getDbNames(params);
163+
Assertions.assertEquals(Collections.singletonList("NULL"), result.getDbs());
164+
Assertions.assertEquals(Collections.singletonList("example_catalog"), result.getCatalogs());
165+
Assertions.assertEquals(Collections.singletonList(42L), result.getCatalogIds());
166+
Assertions.assertFalse(result.isSetDbIds());
167+
}
168+
169+
@Test
170+
public void testNamesOnlySkipsUnavailableCatalog() throws Exception {
171+
Mockito.when(catalog.getDbNames()).thenThrow(new IllegalStateException("Metadata service unavailable"));
172+
TGetDbsParams params = params();
173+
params.setIncludeDbIds(false);
174+
175+
TGetDbsResult result = service.getDbNames(params);
176+
Assertions.assertTrue(result.getDbs().isEmpty());
177+
Assertions.assertTrue(result.getCatalogs().isEmpty());
178+
Assertions.assertFalse(result.isSetDbIds());
179+
}
180+
181+
private TGetDbsParams params() {
182+
TGetDbsParams params = new TGetDbsParams();
183+
params.setCurrentUserIdent(UserIdentity.ROOT.toThrift());
184+
return params;
185+
}
186+
}

‎gensrc/thrift/FrontendService.thrift‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -104,6 +104,8 @@ struct TGetDbsParams {
104104
// Reserved for downstream field `current_roles` to keep thrift field ids
105105
// wire-compatible across maintained branches. Do not reuse this id.
106106
7: optional set<string> reserved_field_7
107+
// Name-only scanners can avoid materializing database objects and omit db_ids in the response.
108+
8: optional bool include_db_ids = true
107109
}
108110

109111
// getDbNames returns a list of database names , database ids and catalog names ,catalog ids

0 commit comments

Comments
 (0)