Skip to content

Commit 91d7da6

Browse files
authored
[opt](iceberg) support create branch/tag for iceberg (#51727)
### What problem does this PR solve? Related Issue: #48285 Problem Summary: You can now create Iceberg branches and tags through Doris. ```sql -- branch alter table tb1 create branch b1; alter table tb1 create branch if not exists b1; alter table tb1 create or replace branch b1; alter table tb1 create or replace branch b1 AS OF VERSION <version>; alter table tb1 create or replace branch b1 AS OF VERSION <version> RETAIN 1 HOURS; -- CREATE b1 at snapshot 1234, retain b1 for 30 days, and retain the latest 30 days. The latest 3 snapshot snapshots, and 2 days worth of snapshots. alter table tb1 CREATE BRANCH b1 AS OF VERSION 1234 RETAIN 30 DAYS WITH SNAPSHOT RETENTION 3 SNAPSHOTS 2 DAYS; -- tag alter table tb1 create tag t1; alter table tb1 create tag if not eists t1; alter table tb1 create or replace tag t1; alter table tb1 create or replace tag t1 AS OF VERSION <version>; -- CREATE t1 at snapshot 1234 and retain it for 1 year. alter table tb1 create tag t1 AS OF VERSION 1234 RETAIN 365 DAYS ``` The supported time units include: DAYS, HOURS, MINUTES
1 parent e9937a7 commit 91d7da6

30 files changed

Lines changed: 2092 additions & 4 deletions

File tree

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
use demo.test_db;
2+
CREATE TABLE tmp_schema_change_branch (id bigint, data string, col float);
3+
INSERT INTO tmp_schema_change_branch VALUES (1, 'a', 1.0), (2, 'b', 2.0), (3, 'c', 3.0);
4+
ALTER TABLE tmp_schema_change_branch CREATE BRANCH test_branch;
5+
ALTER TABLE tmp_schema_change_branch CREATE TAG test_tag;
6+
ALTER TABLE tmp_schema_change_branch DROP COLUMN col;
7+
ALTER TABLE tmp_schema_change_branch ADD COLUMN new_col date;
8+
INSERT INTO tmp_schema_change_branch VALUES (4, 'd', date('2024-04-04')), (5, 'e', date('2024-05-05'));

‎docker/thirdparties/run-thirdparties-docker.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -739,7 +739,7 @@ if [[ "${RUN_SPARK}" -eq 1 ]]; then
739739
fi
740740

741741
if [[ "${RUN_ICEBERG}" -eq 1 ]]; then
742-
start_iceberg > start_icerberg.log 2>&1 &
742+
start_iceberg > start_iceberg.log 2>&1 &
743743
pids["iceberg"]=$!
744744
fi
745745

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisLexer.g4‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -119,6 +119,7 @@ BITOR: 'BITOR';
119119
BITXOR: 'BITXOR';
120120
BLOB: 'BLOB';
121121
BOOLEAN: 'BOOLEAN';
122+
BRANCH: 'BRANCH';
122123
BRIEF: 'BRIEF';
123124
BROKER: 'BROKER';
124125
BUCKETS: 'BUCKETS';
@@ -186,6 +187,7 @@ DATEV2: 'DATEV2';
186187
DATETIMEV1: 'DATETIMEV1';
187188
DATEV1: 'DATEV1';
188189
DAY: 'DAY';
190+
DAYS: 'DAYS';
189191
DECIMAL: 'DECIMAL';
190192
DECIMALV2: 'DECIMALV2';
191193
DECIMALV3: 'DECIMALV3';
@@ -281,6 +283,7 @@ HLL_UNION: 'HLL_UNION';
281283
HOSTNAME: 'HOSTNAME';
282284
HOTSPOT: 'HOTSPOT';
283285
HOUR: 'HOUR';
286+
HOURS: 'HOURS';
284287
HUB: 'HUB';
285288
IDENTIFIED: 'IDENTIFIED';
286289
IF: 'IF';
@@ -359,6 +362,7 @@ MIGRATIONS: 'MIGRATIONS';
359362
MIN: 'MIN';
360363
MINUS: 'MINUS';
361364
MINUTE: 'MINUTE';
365+
MINUTES: 'MINUTES';
362366
MODIFY: 'MODIFY';
363367
MONTH: 'MONTH';
364368
MTMV: 'MTMV';
@@ -454,6 +458,8 @@ RESOURCES: 'RESOURCES';
454458
RESTORE: 'RESTORE';
455459
RESTRICTIVE: 'RESTRICTIVE';
456460
RESUME: 'RESUME';
461+
RETAIN: 'RETAIN';
462+
RETENTION: 'RETENTION';
457463
RETURNS: 'RETURNS';
458464
REVOKE: 'REVOKE';
459465
REWRITTEN: 'REWRITTEN';
@@ -487,6 +493,7 @@ SIGNED: 'SIGNED';
487493
SKEW: 'SKEW';
488494
SMALLINT: 'SMALLINT';
489495
SNAPSHOT: 'SNAPSHOT';
496+
SNAPSHOTS: 'SNAPSHOTS';
490497
SONAME: 'SONAME';
491498
SPLIT: 'SPLIT';
492499
SQL: 'SQL';
@@ -513,6 +520,7 @@ TABLES: 'TABLES';
513520
TABLESAMPLE: 'TABLESAMPLE';
514521
TABLET: 'TABLET';
515522
TABLETS: 'TABLETS';
523+
TAG: 'TAG';
516524
TASK: 'TASK';
517525
TASKS: 'TASKS';
518526
TEMPORARY: 'TEMPORARY';

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 46 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -722,6 +722,44 @@ alterTableClause
722722
| ADD TEMPORARY? PARTITIONS
723723
FROM from=partitionValueList TO to=partitionValueList
724724
INTERVAL INTEGER_VALUE unit=identifier? properties=propertyClause? #alterMultiPartitionClause
725+
| createOrReplaceTagClause #createOrReplaceTagClauses
726+
| createOrReplaceBranchClause #createOrReplaceBranchClauses
727+
;
728+
729+
createOrReplaceTagClause
730+
: CREATE TAG (IF NOT EXISTS)? name=identifier ops=tagOptions
731+
| (CREATE OR)? REPLACE TAG name=identifier ops=tagOptions
732+
;
733+
734+
createOrReplaceBranchClause
735+
: CREATE BRANCH (IF NOT EXISTS)? name=identifier ops=branchOptions
736+
| (CREATE OR)? REPLACE BRANCH name=identifier ops=branchOptions
737+
;
738+
739+
tagOptions
740+
: (AS OF VERSION version=INTEGER_VALUE)? (retainTime)?
741+
;
742+
743+
branchOptions
744+
: (AS OF VERSION version=INTEGER_VALUE)? (retainTime)? (retentionSnapshot)?
745+
;
746+
747+
retainTime
748+
: RETAIN timeValueWithUnit
749+
;
750+
751+
retentionSnapshot
752+
: WITH SNAPSHOT RETENTION minSnapshotsToKeep
753+
| WITH SNAPSHOT RETENTION timeValueWithUnit
754+
| WITH SNAPSHOT RETENTION minSnapshotsToKeep timeValueWithUnit
755+
;
756+
757+
minSnapshotsToKeep
758+
: value=INTEGER_VALUE SNAPSHOTS
759+
;
760+
761+
timeValueWithUnit
762+
: timeValue=INTEGER_VALUE timeUnit=(DAYS | HOURS | MINUTES)
725763
;
726764

727765
columnPosition
@@ -1785,6 +1823,7 @@ nonReserved
17851823
| BITXOR
17861824
| BLOB
17871825
| BOOLEAN
1826+
| BRANCH
17881827
| BRIEF
17891828
| BROKER
17901829
| BUCKETS
@@ -1839,6 +1878,7 @@ nonReserved
18391878
| DATEV1
18401879
| DATEV2
18411880
| DAY
1881+
| DAYS
18421882
| DECIMAL
18431883
| DECIMALV2
18441884
| DECIMALV3
@@ -1896,6 +1936,7 @@ nonReserved
18961936
| HOSTNAME
18971937
| HOTSPOT
18981938
| HOUR
1939+
| HOURS
18991940
| HUB
19001941
| IDENTIFIED
19011942
| IGNORE
@@ -1945,6 +1986,7 @@ nonReserved
19451986
| MIGRATIONS
19461987
| MIN
19471988
| MINUTE
1989+
| MINUTES
19481990
| MODIFY
19491991
| MONTH
19501992
| MTMV
@@ -2011,6 +2053,8 @@ nonReserved
20112053
| RESTORE
20122054
| RESTRICTIVE
20132055
| RESUME
2056+
| RETAIN
2057+
| RETENTION
20142058
| RETURNS
20152059
| REWRITTEN
20162060
| RIGHT_BRACE
@@ -2031,6 +2075,7 @@ nonReserved
20312075
| SHAPE
20322076
| SKEW
20332077
| SNAPSHOT
2078+
| SNAPSHOTS
20342079
| SONAME
20352080
| SPLIT
20362081
| SQL
@@ -2048,6 +2093,7 @@ nonReserved
20482093
| STRUCT
20492094
| SUM
20502095
| TABLES
2096+
| TAG
20512097
| TASK
20522098
| TASKS
20532099
| TEMPORARY

‎fe/fe-core/src/main/java/org/apache/doris/alter/Alter.java‎

Lines changed: 23 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,6 +25,8 @@
2525
import org.apache.doris.analysis.AlterViewStmt;
2626
import org.apache.doris.analysis.ColumnRenameClause;
2727
import org.apache.doris.analysis.CreateMaterializedViewStmt;
28+
import org.apache.doris.analysis.CreateOrReplaceBranchClause;
29+
import org.apache.doris.analysis.CreateOrReplaceTagClause;
2830
import org.apache.doris.analysis.DropMaterializedViewStmt;
2931
import org.apache.doris.analysis.DropPartitionClause;
3032
import org.apache.doris.analysis.DropPartitionFromIndexClause;
@@ -378,6 +380,26 @@ private void setExternalTableAutoAnalyzePolicy(ExternalTable table, List<AlterCl
378380
Env.getCurrentEnv().getEditLog().logModifyTableProperties(info);
379381
}
380382

383+
private void processAlterTableForExternalTable(
384+
ExternalTable table,
385+
List<AlterClause> alterClauses) throws UserException {
386+
for (AlterClause alterClause : alterClauses) {
387+
if (alterClause instanceof ModifyTablePropertiesClause) {
388+
setExternalTableAutoAnalyzePolicy(table, alterClauses);
389+
} else if (alterClause instanceof CreateOrReplaceBranchClause) {
390+
table.getCatalog().createOrReplaceBranch(
391+
table.getDbName(), table.getName(),
392+
((CreateOrReplaceBranchClause) alterClause).getBranchInfo());
393+
} else if (alterClause instanceof CreateOrReplaceTagClause) {
394+
table.getCatalog().createOrReplaceTag(
395+
table.getDbName(), table.getName(),
396+
((CreateOrReplaceTagClause) alterClause).getTagInfo());
397+
} else {
398+
throw new UserException("Invalid alter operations for external table: " + alterClauses);
399+
}
400+
}
401+
}
402+
381403
private boolean needChangeMTMVState(List<AlterClause> alterClauses) {
382404
for (AlterClause alterClause : alterClauses) {
383405
if (alterClause.needChangeMTMVState()) {
@@ -673,7 +695,7 @@ public void processAlterTable(AlterTableCommand command) throws UserException {
673695
case HUDI_EXTERNAL_TABLE:
674696
case TRINO_CONNECTOR_EXTERNAL_TABLE:
675697
alterClauses.addAll(command.getOps());
676-
setExternalTableAutoAnalyzePolicy((ExternalTable) tableIf, alterClauses);
698+
processAlterTableForExternalTable((ExternalTable) tableIf, alterClauses);
677699
return;
678700
default:
679701
throw new DdlException("Do not support alter "

‎fe/fe-core/src/main/java/org/apache/doris/alter/AlterOpType.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -42,6 +42,8 @@ public enum AlterOpType {
4242
MODIFY_TABLE_COMMENT,
4343
MODIFY_COLUMN_COMMENT,
4444
MODIFY_ENGINE,
45+
ALTER_BRANCH,
46+
ALTER_TAG,
4547
INVALID_OP; // INVALID_OP must be the last one
4648

4749
// true means 2 operations have no conflict.
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.analysis;
19+
20+
import org.apache.doris.alter.AlterOpType;
21+
import org.apache.doris.common.UserException;
22+
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo;
23+
24+
public class CreateOrReplaceBranchClause extends AlterTableClause {
25+
private final CreateOrReplaceBranchInfo branchInfo;
26+
27+
public CreateOrReplaceBranchClause(CreateOrReplaceBranchInfo branchInfo) {
28+
super(AlterOpType.ALTER_BRANCH);
29+
this.branchInfo = branchInfo;
30+
}
31+
32+
@Override
33+
public boolean allowOpMTMV() {
34+
return false;
35+
}
36+
37+
@Override
38+
public boolean needChangeMTMVState() {
39+
return false;
40+
}
41+
42+
@Override
43+
public void analyze(Analyzer analyzer) throws UserException {
44+
45+
}
46+
47+
@Override
48+
public String toSql() {
49+
return branchInfo.toSql();
50+
}
51+
52+
public CreateOrReplaceBranchInfo getBranchInfo() {
53+
return branchInfo;
54+
}
55+
}
Lines changed: 55 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,55 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.analysis;
19+
20+
import org.apache.doris.alter.AlterOpType;
21+
import org.apache.doris.common.UserException;
22+
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo;
23+
24+
public class CreateOrReplaceTagClause extends AlterTableClause {
25+
private final CreateOrReplaceTagInfo tagInfo;
26+
27+
public CreateOrReplaceTagClause(CreateOrReplaceTagInfo tagInfo) {
28+
super(AlterOpType.ALTER_TAG);
29+
this.tagInfo = tagInfo;
30+
}
31+
32+
@Override
33+
public boolean allowOpMTMV() {
34+
return false;
35+
}
36+
37+
@Override
38+
public boolean needChangeMTMVState() {
39+
return false;
40+
}
41+
42+
@Override
43+
public void analyze(Analyzer analyzer) throws UserException {
44+
45+
}
46+
47+
@Override
48+
public String toSql() {
49+
return tagInfo.toSql();
50+
}
51+
52+
public CreateOrReplaceTagInfo getTagInfo() {
53+
return tagInfo;
54+
}
55+
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/CatalogIf.java‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -33,6 +33,8 @@
3333
import org.apache.doris.common.UserException;
3434
import org.apache.doris.nereids.trees.plans.commands.CreateDatabaseCommand;
3535
import org.apache.doris.nereids.trees.plans.commands.TruncateTableCommand;
36+
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceBranchInfo;
37+
import org.apache.doris.nereids.trees.plans.commands.info.CreateOrReplaceTagInfo;
3638

3739
import com.google.common.base.Strings;
3840
import com.google.common.collect.Lists;
@@ -220,4 +222,19 @@ default String fromRemoteDatabaseName(String remoteDatabaseName) {
220222
default String fromRemoteTableName(String remoteDatabaseName, String remoteTableName) {
221223
return remoteTableName;
222224
}
225+
226+
// Create or replace branch operations, overridden by subclass if necessary
227+
default void createOrReplaceBranch(String db, String tbl, CreateOrReplaceBranchInfo branchInfo)
228+
throws UserException {
229+
throw new UserException("Not support create or replace branch operation");
230+
}
231+
232+
// Create or replace tag operation, overridden by subclass if necessary
233+
default void createOrReplaceTag(String db, String tbl, CreateOrReplaceTagInfo tagInfo) throws UserException {
234+
throw new UserException("Not support create or replace tag operation");
235+
}
236+
237+
default void replayCreateOrReplaceBranchOrTag(String dbName, String tblName) {
238+
239+
}
223240
}

0 commit comments

Comments
 (0)