Skip to content

Commit ebb50b0

Browse files
committed
[feature](lance) Reuse WARM UP SELECT for index prewarm
1 parent a6be4b7 commit ebb50b0

13 files changed

Lines changed: 362 additions & 161 deletions

File tree

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -525,6 +525,7 @@ SESSION: 'SESSION';
525525
SESSION_USER: 'SESSION_USER';
526526
SET: 'SET';
527527
SETS: 'SETS';
528+
SETTINGS: 'SETTINGS';
528529
SHAPE: 'SHAPE';
529530
SHOW: 'SHOW';
530531
SIGNED: 'SIGNED';

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -538,15 +538,14 @@ supportedOtherStatement
538538
| RESTORE SNAPSHOT label=multipartIdentifier FROM repo=identifier
539539
((ON | EXCLUDE) LEFT_PAREN baseTableRef (COMMA baseTableRef)* RIGHT_PAREN)?
540540
properties=propertyClause? #restore
541-
| WARM UP INDEX indexName=identifier ON tableName=multipartIdentifier
542-
(WITH COMPUTE GROUP computeGroup=identifier)? #warmUpIndex
543541
| WARM UP (CLUSTER | COMPUTE GROUP) destination=identifier WITH
544542
((CLUSTER | COMPUTE GROUP) source=identifier |
545543
(warmUpItem (AND warmUpItem)*)) FORCE?
546544
onTablesClause?
547545
properties=propertyClause? #warmUpCluster
548546
| explain? WARM UP SELECT namedExpressionSeq
549-
FROM warmUpSingleTableRef whereClause? #warmUpSelect
547+
FROM warmUpSingleTableRef whereClause?
548+
(SETTINGS LEFT_PAREN settings=propertyItemList RIGHT_PAREN)? #warmUpSelect
550549
| BACKUP SNAPSHOT label=multipartIdentifier TO repo=identifier
551550
((ON | EXCLUDE) LEFT_PAREN baseTableRef (COMMA baseTableRef)* RIGHT_PAREN)?
552551
properties=propertyClause? #backup
@@ -2347,6 +2346,7 @@ nonReserved
23472346
| SERIALIZABLE
23482347
| SESSION
23492348
| SESSION_USER
2349+
| SETTINGS
23502350
| SHAPE
23512351
| SKEW
23522352
| SNAPSHOT

‎fe/fe-core/src/main/java/org/apache/doris/datasource/lance/LanceIndexPrewarm.java‎

Lines changed: 57 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -28,6 +28,7 @@
2828
import org.apache.doris.common.UserException;
2929
import org.apache.doris.datasource.CatalogIf;
3030
import org.apache.doris.datasource.lance.index.LanceIndexInspectionExecutor;
31+
import org.apache.doris.datasource.lance.index.LanceIndexSegmentGroup;
3132
import org.apache.doris.datasource.lance.metadata.LanceTableMetadata;
3233
import org.apache.doris.info.TableNameInfo;
3334
import org.apache.doris.mysql.privilege.PrivPredicate;
@@ -46,8 +47,10 @@
4647

4748
import java.util.ArrayList;
4849
import java.util.Arrays;
49-
import java.util.Collections;
50+
import java.util.HashSet;
5051
import java.util.List;
52+
import java.util.OptionalInt;
53+
import java.util.Set;
5154
import java.util.concurrent.Future;
5255
import java.util.concurrent.TimeUnit;
5356
import java.util.concurrent.TimeoutException;
@@ -71,14 +74,14 @@ public static ShowResultSetMetaData getResultSetMetaData() {
7174
}
7275

7376
public static void run(ConnectContext ctx, StmtExecutor executor, TableNameInfo tableName,
74-
String indexName, String computeGroup, BooleanSupplier cancelled) throws Exception {
77+
List<String> columns, BooleanSupplier cancelled) throws Exception {
7578
long started = System.nanoTime();
7679
long elapsedMs = ctx.getStartTime() > 0 ? Math.max(0, System.currentTimeMillis() - ctx.getStartTime()) : 0;
7780
long remainingMs = TimeUnit.SECONDS.toMillis(ctx.getQueryTimeoutS()) - elapsedMs;
7881
long deadline = started + TimeUnit.MILLISECONDS.toNanos(Math.max(0, remainingMs));
7982
tableName.analyze(ctx);
8083
checkPrivileges(ctx, tableName);
81-
List<Backend> targets = selectBackends(resolveComputeGroup(ctx, computeGroup));
84+
List<Backend> targets = selectBackends(resolveComputeGroup(ctx));
8285
// Resolve catalog/table initialization and snapshot IO in the bounded metadata pool.
8386
// The worker owns native resources until it returns, even if this caller stops waiting.
8487
String catalogName = tableName.getCtl();
@@ -87,48 +90,83 @@ public static void run(ConnectContext ctx, StmtExecutor executor, TableNameInfo
8790
LanceTableMetadata metadata = LanceIndexInspectionExecutor.execute(() -> {
8891
CatalogIf<?> catalog = Env.getCurrentEnv().getCatalogMgr().getCatalog(catalogName);
8992
if (!(catalog instanceof LanceExternalCatalog)) {
90-
throw new AnalysisException("WARM UP INDEX requires a Lance catalog table");
93+
throw new AnalysisException("Index-only WARM UP SELECT requires a Lance catalog table");
9194
}
9295
TableIf table = catalog.getDbOrAnalysisException(databaseName).getTableOrAnalysisException(name);
9396
if (!(table instanceof LanceExternalTable)) {
94-
throw new AnalysisException("WARM UP INDEX requires a Lance catalog table");
97+
throw new AnalysisException("Index-only WARM UP SELECT requires a Lance catalog table");
9598
}
9699
checkActive(deadline, cancelled);
97100
// Use the query access path, including REST-vended credentials, exactly once.
98101
return ((LanceExternalTable) table).loadMetadata();
99102
}, deadline, cancelled);
100-
PLanceIndexPrewarmRequest request = request(metadata, indexName);
101-
execute(targets, request, deadline, cancelled, (backend, rpcRequest, timeoutMs) ->
102-
BackendServiceProxy.getInstance().prewarmLanceIndexAsync(
103-
new TNetworkAddress(backend.getHost(), backend.getBrpcPort()), rpcRequest, timeoutMs));
104-
executor.sendResultSet(new ShowResultSet(RESULT_META, Collections.singletonList(Arrays.asList(
105-
tableName.toSql(), indexName, Long.toString(metadata.getVersion()), Integer.toString(targets.size()),
106-
Long.toString(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - started))))));
103+
List<String> indexes = selectIndexes(metadata, columns);
104+
List<List<String>> rows = new ArrayList<>();
105+
// All indexes share the resolved snapshot, target set and statement deadline.
106+
for (String index : indexes) {
107+
long indexStarted = System.nanoTime();
108+
PLanceIndexPrewarmRequest request = request(metadata, index);
109+
execute(targets, request, deadline, cancelled, (backend, rpcRequest, timeoutMs) ->
110+
BackendServiceProxy.getInstance().prewarmLanceIndexAsync(
111+
new TNetworkAddress(backend.getHost(), backend.getBrpcPort()), rpcRequest, timeoutMs));
112+
rows.add(Arrays.asList(tableName.toSql(), index, Long.toString(metadata.getVersion()),
113+
Integer.toString(targets.size()),
114+
Long.toString(TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - indexStarted))));
115+
}
116+
executor.sendResultSet(new ShowResultSet(RESULT_META, rows));
117+
}
118+
119+
static List<String> selectIndexes(LanceTableMetadata metadata, List<String> columns) throws AnalysisException {
120+
Set<Integer> fields = new HashSet<>();
121+
for (String column : columns) {
122+
List<String> matches = new ArrayList<>();
123+
metadata.getSchema().getFields().forEach(field -> {
124+
if (field.getName().equalsIgnoreCase(column)) {
125+
matches.add(field.getName());
126+
}
127+
});
128+
if (matches.size() != 1) {
129+
throw new AnalysisException("Unknown or ambiguous index prewarm column: " + column);
130+
}
131+
OptionalInt field = metadata.getLanceFieldId(matches.get(0));
132+
if (!field.isPresent()) {
133+
throw new AnalysisException("Lance field ID is unavailable for index prewarm column: " + column);
134+
}
135+
fields.add(field.getAsInt());
136+
}
137+
List<String> indexes = new ArrayList<>();
138+
for (LanceIndexSegmentGroup index : metadata.getIndexes()) {
139+
if (columns.isEmpty() || index.getSegments().stream()
140+
.anyMatch(segment -> segment.getFieldIds().stream().anyMatch(fields::contains))) {
141+
indexes.add(index.getName());
142+
}
143+
}
144+
if (indexes.isEmpty()) {
145+
throw new AnalysisException("No indexes match the WARM UP SELECT projection");
146+
}
147+
return indexes;
107148
}
108149

109150
static void checkPrivileges(ConnectContext ctx, TableNameInfo table) throws AnalysisException {
110151
// Authorize before catalog lookup: resolving external metadata can perform remote IO.
111152
if (!Env.getCurrentEnv().getAccessManager().checkGlobalPriv(ctx, PrivPredicate.ADMIN)) {
112-
throw new AnalysisException("ADMIN privilege is required for WARM UP INDEX");
153+
throw new AnalysisException("ADMIN privilege is required for Index-only WARM UP SELECT");
113154
}
114155
if (!Env.getCurrentEnv().getAccessManager().checkTblPriv(ctx,
115156
new TableName(table.getCtl(), table.getDb(), table.getTbl()), PrivPredicate.SELECT)) {
116-
throw new AnalysisException("SELECT denied for WARM UP INDEX on " + table.toSql());
157+
throw new AnalysisException("SELECT denied for Index-only WARM UP SELECT on " + table.toSql());
117158
}
118159
}
119160

120-
static ComputeGroup resolveComputeGroup(ConnectContext ctx, String name) throws UserException {
161+
static ComputeGroup resolveComputeGroup(ConnectContext ctx) throws UserException {
121162
if (Config.isCloudMode()) {
122-
String selected = name == null ? ctx.getCloudCluster() : name;
163+
String selected = ctx.getCloudCluster();
123164
if (!Env.getCurrentEnv().getAccessManager().checkCloudPriv(ctx.getCurrentUserIdentity(),
124165
selected, PrivPredicate.USAGE, ResourceTypeEnum.CLUSTER)) {
125166
throw new AnalysisException("USAGE denied for compute group '" + selected + "'");
126167
}
127168
return Env.getCurrentEnv().getComputeGroupMgr().getComputeGroupByName(selected);
128169
}
129-
if (name != null) {
130-
throw new AnalysisException("WITH COMPUTE GROUP requires cloud mode");
131-
}
132170
return ctx.getComputeGroup();
133171
}
134172

‎fe/fe-core/src/main/java/org/apache/doris/nereids/parser/LogicalPlanBuilder.java‎

Lines changed: 59 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -924,7 +924,6 @@
924924
import org.apache.doris.nereids.trees.plans.commands.UnsetVariableCommand;
925925
import org.apache.doris.nereids.trees.plans.commands.UpdateCommand;
926926
import org.apache.doris.nereids.trees.plans.commands.WarmUpClusterCommand;
927-
import org.apache.doris.nereids.trees.plans.commands.WarmUpIndexCommand;
928927
import org.apache.doris.nereids.trees.plans.commands.alter.AlterDatabaseRenameCommand;
929928
import org.apache.doris.nereids.trees.plans.commands.alter.AlterDatabaseSetQuotaCommand;
930929
import org.apache.doris.nereids.trees.plans.commands.alter.AlterRepositoryCommand;
@@ -9514,12 +9513,6 @@ public UserDesc visitGrantUserIdentify(DorisParser.GrantUserIdentifyContext ctx)
95149513
return new UserDesc(userIdentity, new PassVar(password, isPlain));
95159514
}
95169515

9517-
@Override
9518-
public LogicalPlan visitWarmUpIndex(DorisParser.WarmUpIndexContext ctx) {
9519-
return new WarmUpIndexCommand(new TableNameInfo(visitMultipartIdentifier(ctx.tableName)),
9520-
ctx.indexName.getText(), ctx.computeGroup == null ? null : ctx.computeGroup.getText());
9521-
}
9522-
95239516
@Override
95249517
public LogicalPlan visitWarmUpCluster(DorisParser.WarmUpClusterContext ctx) {
95259518
String dstCluster = ctx.destination.getText();
@@ -9617,6 +9610,28 @@ public LogicalPlan visitWarmUpSelect(DorisParser.WarmUpSelectContext ctx) {
96179610
}
96189611

96199612
LogicalProject project = new LogicalProject(projectList, filter);
9613+
Map<String, String> settings = visitPropertyItemList(ctx.settings);
9614+
for (String key : settings.keySet()) {
9615+
if (!"read_index_only".equals(key)) {
9616+
throw new AnalysisException("Unknown WARM UP SELECT setting: " + key);
9617+
}
9618+
}
9619+
String indexOnly = settings.getOrDefault("read_index_only", "false");
9620+
if (!"true".equalsIgnoreCase(indexOnly) && !"false".equalsIgnoreCase(indexOnly)) {
9621+
throw new AnalysisException("read_index_only must be true or false");
9622+
}
9623+
if (Boolean.parseBoolean(indexOnly)) {
9624+
if (ctx.whereClause() != null || ctx.explain() != null) {
9625+
throw new AnalysisException("Index-only WARM UP SELECT does not support WHERE or EXPLAIN");
9626+
}
9627+
// Index cache fills must not execute the data scan or require the data-file cache.
9628+
List<String> columns = indexWarmupColumns(ctx, projectList);
9629+
UnboundBlackholeSink<?> sink = new UnboundBlackholeSink<>(project,
9630+
new UnboundBlackholeSinkContext(true));
9631+
return new WarmupSelectCommand(sink,
9632+
new TableNameInfo(visitMultipartIdentifier(ctx.warmUpSingleTableRef().multipartIdentifier())),
9633+
columns);
9634+
}
96209635

96219636
if (Config.isNotCloudMode() && (!ConnectContext.get().getSessionVariable().isEnableFileCache())) {
96229637
throw new AnalysisException("WARM UP SELECT requires session variable"
@@ -9638,6 +9653,43 @@ public LogicalPlan visitWarmUpSelect(DorisParser.WarmUpSelectContext ctx) {
96389653
return withExplain(command, ctx.explain());
96399654
}
96409655

9656+
private List<String> indexWarmupColumns(DorisParser.WarmUpSelectContext ctx, List<Expression> projects) {
9657+
DorisParser.WarmUpSingleTableRefContext table = ctx.warmUpSingleTableRef();
9658+
List<String> qualifier = visitMultipartIdentifier(table.multipartIdentifier());
9659+
if (table.tableAlias() != null && table.tableAlias().strictIdentifier() != null) {
9660+
if (table.tableAlias().identifierList() != null) {
9661+
throw new AnalysisException("Index-only WARM UP SELECT does not support column aliases");
9662+
}
9663+
qualifier = ImmutableList.of(table.tableAlias().strictIdentifier().getText());
9664+
}
9665+
List<String> columns = new ArrayList<>();
9666+
for (Expression expression : projects) {
9667+
if (!(expression instanceof UnboundSlot) && !(expression instanceof UnboundStar)) {
9668+
throw new AnalysisException("Index-only WARM UP SELECT requires column names or *");
9669+
}
9670+
Slot slot = (Slot) expression;
9671+
List<String> actual = slot.getQualifier();
9672+
if (actual.size() > qualifier.size()) {
9673+
throw new AnalysisException("Unknown column qualifier in index-only WARM UP SELECT");
9674+
}
9675+
for (int i = 0; i < actual.size(); i++) {
9676+
if (!actual.get(i).equalsIgnoreCase(qualifier.get(qualifier.size() - actual.size() + i))) {
9677+
throw new AnalysisException("Unknown column qualifier in index-only WARM UP SELECT");
9678+
}
9679+
}
9680+
if (slot instanceof UnboundStar) {
9681+
UnboundStar star = (UnboundStar) slot;
9682+
if (projects.size() != 1 || !star.getExceptedSlots().isEmpty() || !star.getReplacedAlias().isEmpty()) {
9683+
throw new AnalysisException("Index-only WARM UP SELECT requires * alone without EXCEPT or REPLACE");
9684+
}
9685+
} else {
9686+
List<String> parts = ((UnboundSlot) slot).getNameParts();
9687+
columns.add(parts.get(parts.size() - 1));
9688+
}
9689+
}
9690+
return columns;
9691+
}
9692+
96419693
@Override
96429694
public LogicalPlan visitWarmUpSingleTableRef(DorisParser.WarmUpSingleTableRefContext ctx) {
96439695
List<String> nameParts = visitMultipartIdentifier(ctx.multipartIdentifier());

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/PlanType.java‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -458,7 +458,6 @@ public enum PlanType {
458458
ALTER_COLOCATE_GROUP_COMMAND,
459459
CREATE_USER_COMMAND,
460460
WARM_UP_CLUSTER_COMMAND,
461-
WARM_UP_INDEX_COMMAND,
462461
CANCEL_BACKUP_AND_RESTORE_COMMAND,
463462
CANCEL_BUILD_INDEX_COMMAND,
464463
CREATE_RESOURCE_COMMAND,

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/WarmUpIndexCommand.java‎

Lines changed: 0 additions & 57 deletions
This file was deleted.

0 commit comments

Comments
 (0)