Skip to content

Commit 09cb8b9

Browse files
authored
[feature](iceberg)support iceberg insert grammar (apache#34257)
1 parent c0865cc commit 09cb8b9

41 files changed

Lines changed: 1544 additions & 323 deletions

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalCatalog.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import org.apache.doris.datasource.SessionContext;
2323
import org.apache.doris.datasource.operations.ExternalMetadataOperations;
2424
import org.apache.doris.datasource.property.PropertyConverter;
25+
import org.apache.doris.transaction.TransactionManagerFactory;
2526

2627
import org.apache.hadoop.conf.Configuration;
2728
import org.apache.hadoop.fs.s3a.Constants;
@@ -51,7 +52,9 @@ public IcebergExternalCatalog(long catalogId, String name, String comment) {
5152
@Override
5253
protected void initLocalObjectsImpl() {
5354
initCatalog();
54-
metadataOps = ExternalMetadataOperations.newIcebergMetadataOps(this, catalog);
55+
IcebergMetadataOps ops = ExternalMetadataOperations.newIcebergMetadataOps(this, catalog);
56+
transactionManager = TransactionManagerFactory.createIcebergTransactionManager(ops);
57+
metadataOps = ops;
5558
}
5659

5760
public Catalog getCatalog() {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergExternalTable.java‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -28,9 +28,14 @@
2828
import org.apache.doris.thrift.TTableDescriptor;
2929
import org.apache.doris.thrift.TTableType;
3030

31+
import org.apache.iceberg.PartitionField;
32+
import org.apache.iceberg.Table;
33+
3134
import java.util.HashMap;
3235
import java.util.List;
3336
import java.util.Optional;
37+
import java.util.Set;
38+
import java.util.stream.Collectors;
3439

3540
public class IcebergExternalTable extends ExternalTable {
3641

@@ -83,4 +88,15 @@ public long fetchRowCount() {
8388
makeSureInitialized();
8489
return IcebergUtils.getIcebergRowCount(getCatalog(), getDbName(), getName());
8590
}
91+
92+
public Table getIcebergTable() {
93+
return IcebergUtils.getIcebergTable(getCatalog(), getDbName(), getName());
94+
}
95+
96+
@Override
97+
public Set<String> getPartitionNames() {
98+
getIcebergTable();
99+
return IcebergUtils.getIcebergTable(getCatalog(), getDbName(), getName())
100+
.spec().fields().stream().map(PartitionField::name).collect(Collectors.toSet());
101+
}
86102
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergTransaction.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -133,6 +133,10 @@ public void rollback() {
133133

134134
}
135135

136+
public long getUpdateCnt() {
137+
return commitDataList.stream().mapToLong(TIcebergCommitData::getRowCount).sum();
138+
}
139+
136140
public static class CommitTaskData {
137141
private final String path;
138142
private final long fileSizeInBytes;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 52 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -46,6 +46,7 @@
4646
import org.apache.doris.common.util.TimeUtils;
4747
import org.apache.doris.datasource.ExternalCatalog;
4848
import org.apache.doris.datasource.hive.HiveMetaStoreClientHelper;
49+
import org.apache.doris.nereids.exceptions.NotSupportedException;
4950
import org.apache.doris.thrift.TExprOpcode;
5051

5152
import com.google.common.collect.Lists;
@@ -62,6 +63,7 @@
6263
import org.apache.iceberg.expressions.Unbound;
6364
import org.apache.iceberg.types.Type.TypeID;
6465
import org.apache.iceberg.types.Types;
66+
import org.apache.iceberg.util.LocationUtil;
6567
import org.apache.logging.log4j.LogManager;
6668
import org.apache.logging.log4j.Logger;
6769

@@ -91,6 +93,13 @@ public Integer initialValue() {
9193
public static final String TOTAL_POSITION_DELETES = "total-position-deletes";
9294
public static final String TOTAL_EQUALITY_DELETES = "total-equality-deletes";
9395

96+
// nickname in flink and spark
97+
public static final String WRITE_FORMAT = "write-format";
98+
public static final String COMPRESSION_CODEC = "compression-codec";
99+
100+
// nickname in spark
101+
public static final String SPARK_SQL_COMPRESSION_CODEC = "spark.sql.iceberg.compression-codec";
102+
94103
public static Expression convertToIcebergExpr(Expr expr, Schema schema) {
95104
if (expr == null) {
96105
return null;
@@ -576,12 +585,49 @@ public static long getIcebergRowCount(ExternalCatalog catalog, String dbName, St
576585
}
577586

578587
public static String getFileFormat(Table table) {
579-
Snapshot snapshot = table.currentSnapshot();
580-
if (snapshot == null) {
581-
return TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
582-
} else {
583-
return snapshot.summary().getOrDefault(
584-
TableProperties.DEFAULT_FILE_FORMAT, TableProperties.DEFAULT_FILE_FORMAT_DEFAULT);
588+
Map<String, String> properties = table.properties();
589+
if (properties.containsKey(WRITE_FORMAT)) {
590+
return properties.get(WRITE_FORMAT);
591+
}
592+
if (properties.containsKey(TableProperties.DEFAULT_FILE_FORMAT)) {
593+
return properties.get(TableProperties.DEFAULT_FILE_FORMAT);
594+
}
595+
return TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
596+
}
597+
598+
public static String getFileCompress(Table table) {
599+
Map<String, String> properties = table.properties();
600+
if (properties.containsKey(COMPRESSION_CODEC)) {
601+
return properties.get(COMPRESSION_CODEC);
602+
} else if (properties.containsKey(SPARK_SQL_COMPRESSION_CODEC)) {
603+
return properties.get(SPARK_SQL_COMPRESSION_CODEC);
604+
}
605+
String fileFormat = getFileFormat(table);
606+
if (fileFormat.equalsIgnoreCase("parquet")) {
607+
return properties.getOrDefault(
608+
TableProperties.PARQUET_COMPRESSION, TableProperties.PARQUET_COMPRESSION_DEFAULT_SINCE_1_4_0);
609+
} else if (fileFormat.equalsIgnoreCase("orc")) {
610+
return properties.getOrDefault(
611+
TableProperties.ORC_COMPRESSION, TableProperties.ORC_COMPRESSION_DEFAULT);
612+
}
613+
throw new NotSupportedException("Unsupported file format: " + fileFormat);
614+
}
615+
616+
public static String dataLocation(Table table) {
617+
Map<String, String> properties = table.properties();
618+
if (properties.containsKey(TableProperties.WRITE_LOCATION_PROVIDER_IMPL)) {
619+
throw new NotSupportedException(
620+
"Table " + table.name() + " specifies " + properties.get(TableProperties.WRITE_LOCATION_PROVIDER_IMPL)
621+
+ " as a location provider. "
622+
+ "Writing to Iceberg tables with custom location provider is not supported.");
623+
}
624+
String dataLocation = properties.get(TableProperties.WRITE_DATA_LOCATION);
625+
if (dataLocation == null) {
626+
dataLocation = properties.get(TableProperties.WRITE_FOLDER_STORAGE_LOCATION);
627+
if (dataLocation == null) {
628+
dataLocation = String.format("%s/data", LocationUtil.stripTrailingSlash(table.location()));
629+
}
585630
}
631+
return dataLocation;
586632
}
587633
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergApiSource.java‎

Lines changed: 2 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -24,11 +24,11 @@
2424
import org.apache.doris.common.UserException;
2525
import org.apache.doris.datasource.ExternalCatalog;
2626
import org.apache.doris.datasource.iceberg.IcebergExternalTable;
27+
import org.apache.doris.datasource.iceberg.IcebergUtils;
2728
import org.apache.doris.planner.ColumnRange;
2829
import org.apache.doris.thrift.TFileAttributes;
2930

3031
import org.apache.iceberg.Table;
31-
import org.apache.iceberg.TableProperties;
3232

3333
import java.util.Map;
3434

@@ -61,14 +61,7 @@ public TupleDescriptor getDesc() {
6161

6262
@Override
6363
public String getFileFormat() {
64-
Map<String, String> properties = originTable.properties();
65-
if (properties.containsKey(TableProperties.DEFAULT_FILE_FORMAT)) {
66-
return properties.get(TableProperties.DEFAULT_FILE_FORMAT);
67-
}
68-
if (properties.containsKey(FLINK_WRITE_FORMAT)) {
69-
return properties.get(FLINK_WRITE_FORMAT);
70-
}
71-
return TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
64+
return IcebergUtils.getFileFormat(originTable);
7265
}
7366

7467
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergHMSSource.java‎

Lines changed: 2 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -26,12 +26,11 @@
2626
import org.apache.doris.datasource.ExternalCatalog;
2727
import org.apache.doris.datasource.hive.HMSExternalTable;
2828
import org.apache.doris.datasource.hive.source.HiveScanNode;
29+
import org.apache.doris.datasource.iceberg.IcebergUtils;
2930
import org.apache.doris.planner.ColumnRange;
3031
import org.apache.doris.thrift.TFileAttributes;
3132
import org.apache.doris.thrift.TFileTextScanRangeParams;
3233

33-
import org.apache.iceberg.TableProperties;
34-
3534
import java.util.Map;
3635

3736
public class IcebergHMSSource implements IcebergSource {
@@ -59,14 +58,7 @@ public TupleDescriptor getDesc() {
5958

6059
@Override
6160
public String getFileFormat() throws DdlException, MetaNotFoundException {
62-
Map<String, String> properties = hmsTable.getRemoteTable().getParameters();
63-
if (properties.containsKey(TableProperties.DEFAULT_FILE_FORMAT)) {
64-
return properties.get(TableProperties.DEFAULT_FILE_FORMAT);
65-
}
66-
if (properties.containsKey(FLINK_WRITE_FORMAT)) {
67-
return properties.get(FLINK_WRITE_FORMAT);
68-
}
69-
return TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
61+
return IcebergUtils.getFileFormat(icebergTable);
7062
}
7163

7264
public org.apache.iceberg.Table getIcebergTable() throws MetaNotFoundException {

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/source/IcebergSource.java‎

Lines changed: 0 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -27,9 +27,6 @@
2727

2828
public interface IcebergSource {
2929

30-
// compatible with flink, which is "write.format.default" in spark
31-
String FLINK_WRITE_FORMAT = "write-format";
32-
3330
TupleDescriptor getDesc();
3431

3532
org.apache.iceberg.Table getIcebergTable() throws MetaNotFoundException;
Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.nereids.analyzer;
19+
20+
import org.apache.doris.nereids.exceptions.UnboundException;
21+
import org.apache.doris.nereids.memo.GroupExpression;
22+
import org.apache.doris.nereids.properties.LogicalProperties;
23+
import org.apache.doris.nereids.properties.UnboundLogicalProperties;
24+
import org.apache.doris.nereids.trees.expressions.Expression;
25+
import org.apache.doris.nereids.trees.expressions.NamedExpression;
26+
import org.apache.doris.nereids.trees.expressions.Slot;
27+
import org.apache.doris.nereids.trees.plans.BlockFuncDepsPropagation;
28+
import org.apache.doris.nereids.trees.plans.Plan;
29+
import org.apache.doris.nereids.trees.plans.PlanType;
30+
import org.apache.doris.nereids.trees.plans.algebra.Sink;
31+
import org.apache.doris.nereids.trees.plans.commands.info.DMLCommandType;
32+
import org.apache.doris.nereids.trees.plans.logical.UnboundLogicalSink;
33+
import org.apache.doris.nereids.util.Utils;
34+
35+
import java.util.List;
36+
import java.util.Objects;
37+
import java.util.Optional;
38+
39+
/**
40+
* Represent an external table sink plan node that has not been bound.
41+
*/
42+
public abstract class UnboundBaseExternalTableSink<CHILD_TYPE extends Plan> extends UnboundLogicalSink<CHILD_TYPE>
43+
implements Unbound, Sink, BlockFuncDepsPropagation {
44+
List<String> hints;
45+
List<String> partitions;
46+
47+
/**
48+
* constructor
49+
*/
50+
public UnboundBaseExternalTableSink(List<String> nameParts,
51+
PlanType type,
52+
List<NamedExpression> outputExprs,
53+
Optional<GroupExpression> groupExpression,
54+
Optional<LogicalProperties> logicalProperties,
55+
List<String> colNames,
56+
DMLCommandType dmlCommandType,
57+
CHILD_TYPE child,
58+
List<String> hints,
59+
List<String> partitions) {
60+
super(nameParts, type, outputExprs, groupExpression,
61+
logicalProperties, colNames, dmlCommandType, child);
62+
this.hints = Utils.copyRequiredList(hints);
63+
this.partitions = Utils.copyRequiredList(partitions);
64+
}
65+
66+
public List<String> getColNames() {
67+
return colNames;
68+
}
69+
70+
public List<String> getPartitions() {
71+
return partitions;
72+
}
73+
74+
public List<String> getHints() {
75+
return hints;
76+
}
77+
78+
@Override
79+
public UnboundBaseExternalTableSink<CHILD_TYPE> withOutputExprs(List<NamedExpression> outputExprs) {
80+
throw new UnboundException("could not call withOutputExprs on " + this.getClass().getSimpleName());
81+
}
82+
83+
@Override
84+
public List<? extends Expression> getExpressions() {
85+
throw new UnsupportedOperationException(this.getClass().getSimpleName() + " don't support getExpression()");
86+
}
87+
88+
@Override
89+
public boolean equals(Object o) {
90+
if (this == o) {
91+
return true;
92+
}
93+
if (o == null || getClass() != o.getClass()) {
94+
return false;
95+
}
96+
UnboundBaseExternalTableSink<?> that = (UnboundBaseExternalTableSink<?>) o;
97+
return Objects.equals(nameParts, that.nameParts)
98+
&& Objects.equals(colNames, that.colNames)
99+
&& Objects.equals(hints, that.hints)
100+
&& Objects.equals(partitions, that.partitions);
101+
}
102+
103+
@Override
104+
public int hashCode() {
105+
return Objects.hash(nameParts, colNames, hints, partitions);
106+
}
107+
108+
@Override
109+
public LogicalProperties computeLogicalProperties() {
110+
return UnboundLogicalProperties.INSTANCE;
111+
}
112+
113+
@Override
114+
public List<Slot> computeOutput() {
115+
throw new UnboundException("output");
116+
}
117+
}

0 commit comments

Comments
 (0)