Skip to content

Commit d375e3e

Browse files
wuwenchimorningman
authored andcommitted
[opt](iceberg)support read iceberg with tag/branch (apache#51272)
Related Issue: apache#48285 Problem Summary: if an iceberg has a branch named `branch1` and a tag named `tag1`, you can query `branch1` or `tag1` by these ways: ```sql -- query 'branch1' select * from tb FOR VERSION AS OF 'branch1'; select * from tb@branch(branch1); select * from tb@branch('name'='branch1') -- query 'tag1' select * from tb FOR VERSION AS OF 'tag1'; select * from tb@tag(tag1); select * from tb@tag('name'='tag1') ```
1 parent 16e7b1f commit d375e3e

31 files changed

Lines changed: 1453 additions & 135 deletions

File tree

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,23 @@
1+
create database if not exists demo.test_db;
2+
use demo.test_db;
3+
4+
drop table if exists tag_branch_table;
5+
create table tag_branch_table (c1 int);
6+
insert into tag_branch_table values (1);
7+
8+
alter table demo.test_db.tag_branch_table create branch b1;
9+
alter table demo.test_db.tag_branch_table create tag t1;
10+
11+
insert into tag_branch_table values (2);
12+
13+
alter table demo.test_db.tag_branch_table create branch b2;
14+
alter table demo.test_db.tag_branch_table create tag t2;
15+
16+
alter table demo.test_db.tag_branch_table add column c2 int;
17+
18+
insert into tag_branch_table values (3, 4);
19+
20+
alter table demo.test_db.tag_branch_table create branch b3;
21+
alter table demo.test_db.tag_branch_table create tag t3;
22+
23+
alter table demo.test_db.tag_branch_table add column c3 int;

‎fe/fe-core/src/main/antlr4/org/apache/doris/nereids/DorisParser.g4‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1282,7 +1282,7 @@ identifierSeq
12821282
;
12831283

12841284
optScanParams
1285-
: ATSIGN funcName=identifier LEFT_PAREN (properties=propertyItemList)? RIGHT_PAREN
1285+
: ATSIGN funcName=identifier LEFT_PAREN (mapParams=propertyItemList | listParams=identifierSeq)? RIGHT_PAREN
12861286
;
12871287

12881288
relationPrimary
@@ -1701,7 +1701,7 @@ sampleMethod
17011701
;
17021702

17031703
tableSnapshot
1704-
: FOR VERSION AS OF version=INTEGER_VALUE
1704+
: FOR VERSION AS OF version=(INTEGER_VALUE | STRING_LITERAL)
17051705
| FOR TIME AS OF time=STRING_LITERAL
17061706
;
17071707

‎fe/fe-core/src/main/cup/sql_parser.cup‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -6194,7 +6194,7 @@ opt_scan_params_ref ::=
61946194
:}
61956195
| AT ident:func_name LPAREN opt_key_value_map_in_paren:properties RPAREN
61966196
{:
6197-
RESULT = new TableScanParams(func_name, properties);
6197+
RESULT = new TableScanParams(func_name, properties, new ArrayList<String>());
61986198
:}
61996199
;
62006200

@@ -6340,11 +6340,11 @@ opt_table_snapshot ::=
63406340
table_snapshot ::=
63416341
KW_FOR KW_VERSION KW_AS KW_OF INTEGER_LITERAL:version
63426342
{:
6343-
RESULT = new TableSnapshot(version);
6343+
RESULT = TableSnapshot.versionOf(String.valueOf(version));
63446344
:}
63456345
| KW_FOR KW_TIME KW_AS KW_OF STRING_LITERAL:time
63466346
{:
6347-
RESULT = new TableSnapshot(time);
6347+
RESULT = TableSnapshot.timeOf(time);
63486348
:}
63496349
;
63506350

‎fe/fe-core/src/main/java/org/apache/doris/analysis/TableRef.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -562,7 +562,7 @@ protected void analyzeTableSnapshot(Analyzer analyzer) throws AnalysisException
562562
switch (extTable.getDlaType()) {
563563
case ICEBERG:
564564
if (tableSnapshot.getType() == TableSnapshot.VersionType.TIME) {
565-
String asOfTime = tableSnapshot.getTime();
565+
String asOfTime = tableSnapshot.getValue();
566566
Matcher matcher = TimeUtils.DATETIME_FORMAT_REG.matcher(asOfTime);
567567
if (!matcher.matches()) {
568568
throw new AnalysisException("Invalid datetime string: " + asOfTime);
@@ -574,7 +574,7 @@ protected void analyzeTableSnapshot(Analyzer analyzer) throws AnalysisException
574574
throw new AnalysisException("Hudi table only supports timestamp as snapshot ID");
575575
}
576576
try {
577-
tableSnapshot.setTime(HudiUtils.formatQueryInstant(tableSnapshot.getTime()));
577+
tableSnapshot.setValue(HudiUtils.formatQueryInstant(tableSnapshot.getValue()));
578578
} catch (Exception e) {
579579
throw new AnalysisException("Failed to parse hudi timestamp: " + e.getMessage(), e);
580580
}
@@ -584,7 +584,7 @@ protected void analyzeTableSnapshot(Analyzer analyzer) throws AnalysisException
584584
}
585585
} else if (tableType == TableIf.TableType.ICEBERG_EXTERNAL_TABLE) {
586586
if (tableSnapshot.getType() == TableSnapshot.VersionType.TIME) {
587-
String asOfTime = tableSnapshot.getTime();
587+
String asOfTime = tableSnapshot.getValue();
588588
Matcher matcher = TimeUtils.DATETIME_FORMAT_REG.matcher(asOfTime);
589589
if (!matcher.matches()) {
590590
throw new AnalysisException("Invalid datetime string: " + asOfTime);

‎fe/fe-core/src/main/java/org/apache/doris/analysis/TableScanParams.java‎

Lines changed: 29 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -18,29 +18,52 @@
1818
package org.apache.doris.analysis;
1919

2020
import com.google.common.collect.ImmutableMap;
21+
import org.bouncycastle.util.Strings;
2122

23+
import java.util.List;
2224
import java.util.Map;
2325

2426
public class TableScanParams {
2527
public static String INCREMENTAL_READ = "incr";
28+
public static String BRANCH = "branch";
29+
public static String TAG = "tag";
2630

2731
private final String paramType;
28-
private final Map<String, String> params;
32+
// There are two ways to pass parameters to a function.
33+
// - One is in map form, where the data is stored in `mapParams`.
34+
// such as: @func_name('param1'='value1', 'param2'='value2', 'param3'='value3')
35+
// - The other is in list form, where the data is stored in `listParams`.
36+
// such as: `listParams` is used for @func_name('value1', 'value2', 'value3')
37+
private final Map<String, String> mapParams;
38+
private final List<String> listParams;
2939

30-
public TableScanParams(String paramType, Map<String, String> params) {
31-
this.paramType = paramType;
32-
this.params = params == null ? ImmutableMap.of() : ImmutableMap.copyOf(params);
40+
public TableScanParams(String paramType, Map<String, String> mapParams, List<String> listParams) {
41+
this.paramType = Strings.toLowerCase(paramType);
42+
this.mapParams = mapParams == null ? ImmutableMap.of() : ImmutableMap.copyOf(mapParams);
43+
this.listParams = listParams;
44+
}
45+
46+
public List<String> getListParams() {
47+
return listParams;
3348
}
3449

3550
public String getParamType() {
3651
return paramType;
3752
}
3853

39-
public Map<String, String> getParams() {
40-
return params;
54+
public Map<String, String> getMapParams() {
55+
return mapParams;
4156
}
4257

4358
public boolean incrementalRead() {
4459
return INCREMENTAL_READ.equals(paramType);
4560
}
61+
62+
public boolean isBranch() {
63+
return BRANCH.equals(paramType);
64+
}
65+
66+
public boolean isTag() {
67+
return TAG.equals(paramType);
68+
}
4669
}

‎fe/fe-core/src/main/java/org/apache/doris/analysis/TableSnapshot.java‎

Lines changed: 19 additions & 22 deletions
Original file line numberDiff line numberDiff line change
@@ -28,47 +28,44 @@ public enum VersionType {
2828
}
2929

3030
private final VersionType type;
31-
private String time;
32-
private long version;
31+
private String value;
3332

34-
public TableSnapshot(long version) {
35-
this.version = version;
36-
this.type = VersionType.VERSION;
33+
public TableSnapshot(TableSnapshot other) {
34+
this.type = other.type;
35+
this.value = other.value;
3736
}
3837

39-
public TableSnapshot(String time) {
40-
this.time = time;
41-
this.type = VersionType.TIME;
38+
public TableSnapshot(String value, VersionType type) {
39+
this.value = value;
40+
this.type = type;
4241
}
4342

44-
public TableSnapshot(TableSnapshot other) {
45-
this.type = other.type;
46-
this.time = other.time;
47-
this.version = other.version;
43+
public static TableSnapshot timeOf(String time) {
44+
return new TableSnapshot(time, VersionType.TIME);
4845
}
4946

50-
public VersionType getType() {
51-
return type;
47+
public static TableSnapshot versionOf(String time) {
48+
return new TableSnapshot(time, VersionType.VERSION);
5249
}
5350

54-
public String getTime() {
55-
return time;
51+
public String getValue() {
52+
return value;
5653
}
5754

58-
public void setTime(String time) {
59-
this.time = time;
55+
public VersionType getType() {
56+
return type;
6057
}
6158

62-
public long getVersion() {
63-
return version;
59+
public void setValue(String value) {
60+
this.value = value;
6461
}
6562

6663
@Override
6764
public String toString() {
6865
if (this.type == VersionType.VERSION) {
69-
return " FOR VERSION AS OF " + version;
66+
return " FOR VERSION AS OF " + value;
7067
} else {
71-
return " FOR TIME AS OF '" + time + "'";
68+
return " FOR TIME AS OF '" + value + "'";
7269
}
7370
}
7471
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/FileQueryScanNode.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,6 +20,7 @@
2020
import org.apache.doris.analysis.Analyzer;
2121
import org.apache.doris.analysis.SlotDescriptor;
2222
import org.apache.doris.analysis.TableSample;
23+
import org.apache.doris.analysis.TableScanParams;
2324
import org.apache.doris.analysis.TableSnapshot;
2425
import org.apache.doris.analysis.TupleDescriptor;
2526
import org.apache.doris.catalog.Column;
@@ -99,6 +100,8 @@ public abstract class FileQueryScanNode extends FileScanNode {
99100
// connection context is a thread local variable, it is not available is running in other thread.
100101
protected SessionVariable sessionVariable;
101102

103+
protected TableScanParams scanParams;
104+
102105
/**
103106
* External file scan node for Query hms table
104107
* needCheckColumnPriv: Some of ExternalFileScanNode do not need to check column priv
@@ -628,6 +631,18 @@ public TableSnapshot getQueryTableSnapshot() {
628631
return this.tableSnapshot;
629632
}
630633

634+
public void setScanParams(TableScanParams scanParams) {
635+
this.scanParams = scanParams;
636+
}
637+
638+
public TableScanParams getScanParams() {
639+
TableScanParams scan = desc.getRef().getScanParams();
640+
if (scan != null) {
641+
return scan;
642+
}
643+
return this.scanParams;
644+
}
645+
631646
/**
632647
* The real file split size is determined by:
633648
* 1. If user specify the split size in session variable `file_split_size`, use user specified value.

‎fe/fe-core/src/main/java/org/apache/doris/datasource/FileScanNode.java‎

Lines changed: 8 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -105,15 +105,21 @@ public String getNodeExplainString(String prefix, TExplainLevel detailLevel) {
105105
}
106106

107107
output.append(prefix);
108-
if (isBatchMode()) {
108+
boolean isBatch;
109+
try {
110+
isBatch = isBatchMode();
111+
} catch (UserException e) {
112+
throw new RuntimeException(e);
113+
}
114+
if (isBatch) {
109115
output.append("(approximate)");
110116
}
111117
output.append("inputSplitNum=").append(selectedSplitNum).append(", totalFileSize=")
112118
.append(totalFileSize).append(", scanRanges=").append(scanRangeLocations.size()).append("\n");
113119
output.append(prefix).append("partition=").append(selectedPartitionNum).append("/").append(totalPartitionNum)
114120
.append("\n");
115121

116-
if (detailLevel == TExplainLevel.VERBOSE && !isBatchMode()) {
122+
if (detailLevel == TExplainLevel.VERBOSE && !isBatch) {
117123
output.append(prefix).append("backends:").append("\n");
118124
Multimap<Long, TFileRangeDesc> scanRangeLocationsMap = ArrayListMultimap.create();
119125
// 1. group by backend id

‎fe/fe-core/src/main/java/org/apache/doris/datasource/SplitGenerator.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -40,7 +40,7 @@ default List<Split> getSplits(int numBackends) throws UserException {
4040
/**
4141
* Whether the producer(e.g. ScanNode) support batch mode.
4242
*/
43-
default boolean isBatchMode() {
43+
default boolean isBatchMode() throws UserException {
4444
return false;
4545
}
4646

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalTable.java‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,7 @@
1717

1818
package org.apache.doris.datasource.hive;
1919

20+
import org.apache.doris.analysis.TableScanParams;
2021
import org.apache.doris.analysis.TableSnapshot;
2122
import org.apache.doris.catalog.Column;
2223
import org.apache.doris.catalog.Env;
@@ -1046,12 +1047,13 @@ public void beforeMTMVRefresh(MTMV mtmv) throws DdlException {
10461047
}
10471048

10481049
@Override
1049-
public MvccSnapshot loadSnapshot(Optional<TableSnapshot> tableSnapshot) {
1050+
public MvccSnapshot loadSnapshot(Optional<TableSnapshot> tableSnapshot, Optional<TableScanParams> scanParams) {
10501051
if (getDlaType() == DLAType.HUDI) {
10511052
return HudiUtils.getHudiMvccSnapshot(tableSnapshot, this);
10521053
} else if (getDlaType() == DLAType.ICEBERG) {
10531054
return new IcebergMvccSnapshot(
1054-
IcebergUtils.getIcebergSnapshotCacheValue(tableSnapshot, getCatalog(), getDbName(), getName()));
1055+
IcebergUtils.getIcebergSnapshotCacheValue(
1056+
tableSnapshot, getCatalog(), getDbName(), getName(), scanParams));
10551057
} else {
10561058
return new EmptyMvccSnapshot();
10571059
}

0 commit comments

Comments
 (0)