Skip to content

Commit dbefde4

Browse files
wuwenchiwuwenchi.wwc
authored andcommitted
[opt](iceberg)support drop tag and branch (apache#52453)
Related Issue: apache#48285 Problem Summary: support drop tag and branch: ```sql alter table tb1 drop branch b1; alter table tb1 drop branch if exists b2; alter table tb1 drop tag t1; alter table tb1 drop tag if exists t2; ``` Co-authored-by: wuwenchi.wwc <[email protected]>
1 parent 42f9f32 commit dbefde4

17 files changed

Lines changed: 614 additions & 12 deletions

File tree

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -642,6 +642,8 @@ alterTableClause
642642
INTERVAL INTEGER_VALUE unit=identifier? properties=propertyClause? #alterMultiPartitionClause
643643
| createOrReplaceTagClause #createOrReplaceTagClauses
644644
| createOrReplaceBranchClause #createOrReplaceBranchClauses
645+
| dropBranchClause #dropBranchClauses
646+
| dropTagClause #dropTagClauses
645647
;
646648

647649
createOrReplaceTagClause
@@ -680,6 +682,14 @@ timeValueWithUnit
680682
: timeValue=INTEGER_VALUE timeUnit=(DAYS | HOURS | MINUTES)
681683
;
682684

685+
dropBranchClause
686+
: DROP BRANCH (IF EXISTS)? name=identifier
687+
;
688+
689+
dropTagClause
690+
: DROP TAG (IF EXISTS)? name=identifier
691+
;
692+
683693
columnPosition
684694
: FIRST
685695
| AFTER position=identifier

‎fe/fe-core/src/main/cup/sql_parser.cup‎

Lines changed: 14 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -59,11 +59,13 @@ import org.apache.doris.cloud.analysis.UseCloudClusterStmt;
5959
import org.apache.doris.cloud.proto.Cloud.StagePB;
6060
import org.apache.doris.indexpolicy.IndexPolicyTypeEnum;
6161
import org.apache.doris.mysql.MysqlPassword;
62+
import org.apache.doris.nereids.trees.plans.commands.info.BranchOptions;
6263
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo;
6364
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo;
64-
import org.apache.doris.nereids.trees.plans.commands.info.BranchOptions;
65-
import org.apache.doris.nereids.trees.plans.commands.info.TagOptions;
65+
import org.apache.doris.nereids.trees.plans.commands.info.DropBranchInfo;
66+
import org.apache.doris.nereids.trees.plans.commands.info.DropTagInfo;
6667
import org.apache.doris.nereids.trees.plans.commands.info.RetentionSnapshots;
68+
import org.apache.doris.nereids.trees.plans.commands.info.TagOptions;
6769
import org.apache.doris.load.loadv2.LoadTask;
6870
import org.apache.doris.policy.PolicyTypeEnum;
6971
import org.apache.doris.resource.workloadschedpolicy.WorkloadConditionMeta;
@@ -1812,6 +1814,16 @@ alter_table_clause ::=
18121814
CreateOrReplaceTagInfo tagInfo = new CreateOrReplaceTagInfo(tagName, true, true, false, tagOptions);
18131815
RESULT = new CreateOrReplaceTagClause(tagInfo);
18141816
:}
1817+
| KW_DROP KW_BRANCH opt_if_exists:ifExists ident:branchName
1818+
{:
1819+
DropBranchInfo branchInfo = new DropBranchInfo(branchName, ifExists);
1820+
RESULT = new DropBranchClause(branchInfo);
1821+
:}
1822+
| KW_DROP KW_TAG opt_if_exists ident:tagName
1823+
{:
1824+
DropTagInfo tagInfo = new DropTagInfo(tagName, ifExists);
1825+
RESULT = new DropTagClause(tagInfo);
1826+
:}
18151827
;
18161828

18171829
branch_options ::=

‎fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,11 @@
2828
import org.apache.doris.analysis.CreateMaterializedViewStmt;
2929
import org.apache.doris.analysis.CreateOrReplaceBranchClause;
3030
import org.apache.doris.analysis.CreateOrReplaceTagClause;
31+
import org.apache.doris.analysis.DropBranchClause;
3132
import org.apache.doris.analysis.DropMaterializedViewStmt;
3233
import org.apache.doris.analysis.DropPartitionClause;
3334
import org.apache.doris.analysis.DropPartitionFromIndexClause;
35+
import org.apache.doris.analysis.DropTagClause;
3436
import org.apache.doris.analysis.ModifyColumnCommentClause;
3537
import org.apache.doris.analysis.ModifyDistributionClause;
3638
import org.apache.doris.analysis.ModifyEngineClause;
@@ -350,6 +352,14 @@ private void processAlterTableForExternalTable(
350352
table.getCatalog().createOrReplaceTag(
351353
table.getDbName(), table.getName(),
352354
((CreateOrReplaceTagClause) alterClause).getTagInfo());
355+
} else if (alterClause instanceof DropBranchClause) {
356+
table.getCatalog().dropBranch(
357+
table.getDbName(), table.getName(),
358+
((DropBranchClause) alterClause).getDropBranchInfo());
359+
} else if (alterClause instanceof DropTagClause) {
360+
table.getCatalog().dropTag(
361+
table.getDbName(), table.getName(),
362+
((DropTagClause) alterClause).getDropTagInfo());
353363
} else {
354364
throw new UserException("Invalid alter operations for external table: " + alterClauses);
355365
}

‎fe/fe-core/src/main/java/org/apache/doris/analysis/AlterTableStmt.java‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -188,7 +188,9 @@ public void checkExternalTableOperationAllow(Table table) throws UserException {
188188
|| alterClause instanceof ReorderColumnsClause
189189
|| alterClause instanceof ModifyEngineClause
190190
|| alterClause instanceof CreateOrReplaceBranchClause
191-
|| alterClause instanceof CreateOrReplaceTagClause) {
191+
|| alterClause instanceof CreateOrReplaceTagClause
192+
|| alterClause instanceof DropBranchClause
193+
|| alterClause instanceof DropTagClause) {
192194
clauses.add(alterClause);
193195
} else {
194196
throw new AnalysisException(table.getType().toString() + " [" + table.getName() + "] "
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.analysis;
19+
20+
import org.apache.doris.alter.AlterOpType;
21+
import org.apache.doris.common.UserException;
22+
import org.apache.doris.nereids.trees.plans.commands.info.DropBranchInfo;
23+
24+
public class DropBranchClause extends AlterTableClause {
25+
private final DropBranchInfo dropBranchInfo;
26+
27+
public DropBranchClause(DropBranchInfo dropBranchInfo) {
28+
super(AlterOpType.ALTER_BRANCH);
29+
this.dropBranchInfo = dropBranchInfo;
30+
}
31+
32+
@Override
33+
public boolean allowOpMTMV() {
34+
return false;
35+
}
36+
37+
@Override
38+
public boolean needChangeMTMVState() {
39+
return false;
40+
}
41+
42+
@Override
43+
public void analyze(Analyzer analyzer) throws UserException {
44+
45+
}
46+
47+
@Override
48+
public String toSql() {
49+
return dropBranchInfo.toSql();
50+
}
51+
52+
public DropBranchInfo getDropBranchInfo() {
53+
return dropBranchInfo;
54+
}
55+
}
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.analysis;
19+
20+
import org.apache.doris.alter.AlterOpType;
21+
import org.apache.doris.common.UserException;
22+
import org.apache.doris.nereids.trees.plans.commands.info.DropTagInfo;
23+
24+
public class DropTagClause extends AlterTableClause {
25+
private final DropTagInfo dropTagInfo;
26+
27+
public DropTagClause(DropTagInfo dropTagInfo) {
28+
super(AlterOpType.ALTER_TAG);
29+
this.dropTagInfo = dropTagInfo;
30+
}
31+
32+
@Override
33+
public boolean allowOpMTMV() {
34+
return false;
35+
}
36+
37+
@Override
38+
public boolean needChangeMTMVState() {
39+
return false;
40+
}
41+
42+
@Override
43+
public void analyze(Analyzer analyzer) throws UserException {
44+
45+
}
46+
47+
@Override
48+
public String toSql() {
49+
return dropTagInfo.toSql();
50+
}
51+
52+
public DropTagInfo getDropTagInfo() {
53+
return dropTagInfo;
54+
}
55+
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
import org.apache.doris.common.UserException;
3434
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo;
3535
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo;
36+
import org.apache.doris.nereids.trees.plans.commands.info.DropBranchInfo;
37+
import org.apache.doris.nereids.trees.plans.commands.info.DropTagInfo;
3638

3739
import com.google.common.base.Strings;
3840
import com.google.common.collect.Lists;
@@ -221,7 +223,15 @@ default void createOrReplaceTag(String db, String tbl, CreateOrReplaceTagInfo ta
221223
throw new UserException("Not support create or replace tag operation");
222224
}
223225

224-
default void replayCreateOrReplaceBranchOrTag(String dbName, String tblName) {
226+
default void replayOperateOnBranchOrTag(String dbName, String tblName) {
225227

226228
}
229+
230+
default void dropBranch(String db, String tbl, DropBranchInfo branchInfo) throws UserException {
231+
throw new UserException("Not support drop branch operation");
232+
}
233+
234+
default void dropTag(String db, String tbl, DropTagInfo tagInfo) throws UserException {
235+
throw new UserException("Not support drop tag operation");
236+
}
227237
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java‎

Lines changed: 38 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,8 @@
6262
import org.apache.doris.fs.remote.dfs.DFSFileSystem;
6363
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo;
6464
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo;
65+
import org.apache.doris.nereids.trees.plans.commands.info.DropBranchInfo;
66+
import org.apache.doris.nereids.trees.plans.commands.info.DropTagInfo;
6567
import org.apache.doris.persist.CreateDbInfo;
6668
import org.apache.doris.persist.CreateTableInfo;
6769
import org.apache.doris.persist.DropDbInfo;
@@ -1333,9 +1335,43 @@ public void createOrReplaceTag(String db, String tbl, CreateOrReplaceTagInfo tag
13331335
}
13341336

13351337
@Override
1336-
public void replayCreateOrReplaceBranchOrTag(String dbName, String tblName) {
1338+
public void replayOperateOnBranchOrTag(String dbName, String tblName) {
13371339
if (metadataOps != null) {
1338-
metadataOps.afterCreateOrReplaceBranchOrTag(dbName, tblName);
1340+
metadataOps.afterOperateOnBranchOrTag(dbName, tblName);
1341+
}
1342+
}
1343+
1344+
@Override
1345+
public void dropBranch(String db, String tbl, DropBranchInfo branchInfo) throws UserException {
1346+
makeSureInitialized();
1347+
if (metadataOps == null) {
1348+
throw new DdlException("DropBranch operation is not supported for catalog: " + getName());
1349+
}
1350+
try {
1351+
metadataOps.dropBranch(db, tbl, branchInfo);
1352+
TableBranchOrTagInfo info = new TableBranchOrTagInfo(getName(), db, tbl);
1353+
Env.getCurrentEnv().getEditLog().logBranchOrTag(info);
1354+
} catch (Exception e) {
1355+
LOG.warn("Failed to drop branch for table {}.{} in catalog {}",
1356+
db, tbl, getName(), e);
1357+
throw e;
1358+
}
1359+
}
1360+
1361+
@Override
1362+
public void dropTag(String db, String tbl, DropTagInfo tagInfo) throws UserException {
1363+
makeSureInitialized();
1364+
if (metadataOps == null) {
1365+
throw new DdlException("DropTag operation is not supported for catalog: " + getName());
1366+
}
1367+
try {
1368+
metadataOps.dropTag(db, tbl, tagInfo);
1369+
TableBranchOrTagInfo info = new TableBranchOrTagInfo(getName(), db, tbl);
1370+
Env.getCurrentEnv().getEditLog().logBranchOrTag(info);
1371+
} catch (Exception e) {
1372+
LOG.warn("Failed to drop tag for table {}.{} in catalog {}",
1373+
db, tbl, getName(), e);
1374+
throw e;
13391375
}
13401376
}
13411377
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@
4242
import org.apache.doris.datasource.property.constants.HMSProperties;
4343
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo;
4444
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo;
45+
import org.apache.doris.nereids.trees.plans.commands.info.DropBranchInfo;
46+
import org.apache.doris.nereids.trees.plans.commands.info.DropTagInfo;
4547
import org.apache.doris.qe.ConnectContext;
4648

4749
import com.google.common.annotations.VisibleForTesting;
@@ -379,6 +381,16 @@ public void createOrReplaceTagImpl(String dbName, String tblName, CreateOrReplac
379381
throw new UserException("Not support create or replace tag in hive catalog.");
380382
}
381383

384+
@Override
385+
public void dropTagImpl(String dbName, String tblName, DropTagInfo tagInfo) throws UserException {
386+
throw new UserException("Not support drop tag in hive catalog.");
387+
}
388+
389+
@Override
390+
public void dropBranchImpl(String dbName, String tblName, DropBranchInfo branchInfo) throws UserException {
391+
throw new UserException("Not support drop branch in hive catalog.");
392+
}
393+
382394
@Override
383395
public List<String> listTableNames(String dbName) {
384396
return client.getAllTables(dbName);

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergMetadataOps.java‎

Lines changed: 42 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -38,12 +38,15 @@
3838
import org.apache.doris.nereids.trees.plans.commands.info.BranchOptions;
3939
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo;
4040
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo;
41+
import org.apache.doris.nereids.trees.plans.commands.info.DropBranchInfo;
42+
import org.apache.doris.nereids.trees.plans.commands.info.DropTagInfo;
4143
import org.apache.doris.nereids.trees.plans.commands.info.TagOptions;
4244

4345
import org.apache.iceberg.ManageSnapshots;
4446
import org.apache.iceberg.PartitionSpec;
4547
import org.apache.iceberg.Schema;
4648
import org.apache.iceberg.Snapshot;
49+
import org.apache.iceberg.SnapshotRef;
4750
import org.apache.iceberg.Table;
4851
import org.apache.iceberg.catalog.Catalog;
4952
import org.apache.iceberg.catalog.Namespace;
@@ -401,7 +404,7 @@ public void createOrReplaceBranchImpl(String dbName, String tblName, CreateOrRep
401404
}
402405

403406
@Override
404-
public void afterCreateOrReplaceBranchOrTag(String dbName, String tblName) {
407+
public void afterOperateOnBranchOrTag(String dbName, String tblName) {
405408
ExternalDatabase<?> db = dorisCatalog.getDbNullable(dbName);
406409
if (db != null) {
407410
ExternalTable tbl = db.getTableNullable(tblName);
@@ -455,6 +458,44 @@ public void createOrReplaceTagImpl(String dbName, String tblName, CreateOrReplac
455458
}
456459
}
457460

461+
@Override
462+
public void dropTagImpl(String dbName, String tblName, DropTagInfo tagInfo) throws UserException {
463+
String tagName = tagInfo.getTagName();
464+
boolean ifExists = tagInfo.getIfExists();
465+
Table icebergTable = IcebergUtils.getIcebergTable(dorisCatalog, dbName, tblName);
466+
SnapshotRef snapshotRef = icebergTable.refs().get(tagName);
467+
468+
if (snapshotRef != null || !ifExists) {
469+
ManageSnapshots manageSnapshots = icebergTable.manageSnapshots();
470+
try {
471+
preExecutionAuthenticator.execute(() -> manageSnapshots.removeTag(tagName).commit());
472+
} catch (Exception e) {
473+
throw new RuntimeException(
474+
"Failed to drop tag: " + tagName + " in table: " + icebergTable.name()
475+
+ ", error message is: " + e.getMessage(), e);
476+
}
477+
}
478+
}
479+
480+
@Override
481+
public void dropBranchImpl(String dbName, String tblName, DropBranchInfo branchInfo) throws UserException {
482+
String branchName = branchInfo.getBranchName();
483+
boolean ifExists = branchInfo.getIfExists();
484+
Table icebergTable = IcebergUtils.getIcebergTable(dorisCatalog, dbName, tblName);
485+
SnapshotRef snapshotRef = icebergTable.refs().get(branchName);
486+
487+
if (snapshotRef != null || !ifExists) {
488+
ManageSnapshots manageSnapshots = icebergTable.manageSnapshots();
489+
try {
490+
preExecutionAuthenticator.execute(() -> manageSnapshots.removeBranch(branchName).commit());
491+
} catch (Exception e) {
492+
throw new RuntimeException(
493+
"Failed to drop branch: " + branchName + " in table: " + icebergTable.name()
494+
+ ", error message is: " + e.getMessage(), e);
495+
}
496+
}
497+
}
498+
458499
public PreExecutionAuthenticator getPreExecutionAuthenticator() {
459500
return preExecutionAuthenticator;
460501
}

0 commit comments

Comments
 (0)