Skip to content

Commit 68b33c8

Browse files
authored
[feature](iceberg) add iceberg transaction implement (apache#33629)
Issue apache#31442 add iceberg transaction
1 parent e8a03ad commit 68b33c8

5 files changed

Lines changed: 539 additions & 0 deletions

File tree

Lines changed: 182 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,182 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
// This file is copied from
18+
// https://github.com/trinodb/trino/blob/438/plugin/trino-iceberg/src/main/java/io/trino/plugin/iceberg/IcebergMetadata.java
19+
// and modified by Doris
20+
21+
package org.apache.doris.datasource.iceberg;
22+
23+
import org.apache.doris.common.UserException;
24+
import org.apache.doris.thrift.TFileContent;
25+
import org.apache.doris.thrift.TIcebergCommitData;
26+
import org.apache.doris.transaction.Transaction;
27+
28+
import avro.shaded.com.google.common.collect.Lists;
29+
import com.google.common.base.VerifyException;
30+
import org.apache.iceberg.AppendFiles;
31+
import org.apache.iceberg.DataFiles;
32+
import org.apache.iceberg.FileContent;
33+
import org.apache.iceberg.Metrics;
34+
import org.apache.iceberg.Table;
35+
import org.apache.iceberg.catalog.TableIdentifier;
36+
import org.apache.logging.log4j.LogManager;
37+
import org.apache.logging.log4j.Logger;
38+
39+
import java.util.ArrayList;
40+
import java.util.Collections;
41+
import java.util.List;
42+
import java.util.Optional;
43+
44+
public class IcebergTransaction implements Transaction {
45+
46+
private static final Logger LOG = LogManager.getLogger(IcebergTransaction.class);
47+
private final IcebergMetadataOps ops;
48+
private org.apache.iceberg.Transaction transaction;
49+
private final List<TIcebergCommitData> commitDataList = Lists.newArrayList();
50+
51+
public IcebergTransaction(IcebergMetadataOps ops) {
52+
this.ops = ops;
53+
}
54+
55+
public void updateIcebergCommitData(List<TIcebergCommitData> commitDataList) {
56+
synchronized (this) {
57+
this.commitDataList.addAll(commitDataList);
58+
}
59+
}
60+
61+
public void beginInsert(String dbName, String tbName) {
62+
Table icebergTable = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbName));
63+
transaction = icebergTable.newTransaction();
64+
}
65+
66+
public void finishInsert() {
67+
Table icebergTable = transaction.table();
68+
AppendFiles appendFiles = transaction.newAppend();
69+
70+
for (CommitTaskData task : convertToCommitTaskData()) {
71+
DataFiles.Builder builder = DataFiles.builder(icebergTable.spec())
72+
.withPath(task.getPath())
73+
.withFileSizeInBytes(task.getFileSizeInBytes())
74+
.withFormat(IcebergUtils.getFileFormat(icebergTable))
75+
.withMetrics(task.getMetrics());
76+
77+
if (icebergTable.spec().isPartitioned()) {
78+
List<String> partitionValues = task.getPartitionValues()
79+
.orElseThrow(() -> new VerifyException("No partition data for partitioned table"));
80+
builder.withPartitionValues(partitionValues);
81+
}
82+
appendFiles.appendFile(builder.build());
83+
}
84+
85+
// in appendFiles.commit, it will generate metadata(manifest and snapshot)
86+
// after appendFiles.commit, in current transaction, you can already see the new snapshot
87+
appendFiles.commit();
88+
}
89+
90+
public List<CommitTaskData> convertToCommitTaskData() {
91+
List<CommitTaskData> commitTaskData = new ArrayList<>();
92+
for (TIcebergCommitData data : this.commitDataList) {
93+
commitTaskData.add(new CommitTaskData(
94+
data.getFilePath(),
95+
data.getFileSize(),
96+
new Metrics(
97+
data.getRowCount(),
98+
Collections.EMPTY_MAP,
99+
Collections.EMPTY_MAP,
100+
Collections.EMPTY_MAP,
101+
Collections.EMPTY_MAP
102+
),
103+
data.isSetPartitionValues() ? Optional.of(data.getPartitionValues()) : Optional.empty(),
104+
convertToFileContent(data.getFileContent()),
105+
data.isSetReferencedDataFiles() ? Optional.of(data.getReferencedDataFiles()) : Optional.empty()
106+
));
107+
}
108+
return commitTaskData;
109+
}
110+
111+
private FileContent convertToFileContent(TFileContent content) {
112+
if (content.equals(TFileContent.DATA)) {
113+
return FileContent.DATA;
114+
} else if (content.equals(TFileContent.POSITION_DELETES)) {
115+
return FileContent.POSITION_DELETES;
116+
} else {
117+
return FileContent.EQUALITY_DELETES;
118+
}
119+
}
120+
121+
@Override
122+
public void commit() throws UserException {
123+
// Externally readable
124+
// Manipulate the relevant data so that others can also see the latest table, such as:
125+
// 1. hadoop: it will change the version number information in 'version-hint.text'
126+
// 2. hive: it will change the table properties, the most important thing is to revise 'metadata_location'
127+
// 3. and so on ...
128+
transaction.commitTransaction();
129+
}
130+
131+
@Override
132+
public void rollback() {
133+
134+
}
135+
136+
public static class CommitTaskData {
137+
private final String path;
138+
private final long fileSizeInBytes;
139+
private final Metrics metrics;
140+
private final Optional<List<String>> partitionValues;
141+
private final FileContent content;
142+
private final Optional<List<String>> referencedDataFiles;
143+
144+
public CommitTaskData(String path,
145+
long fileSizeInBytes,
146+
Metrics metrics,
147+
Optional<List<String>> partitionValues,
148+
FileContent content,
149+
Optional<List<String>> referencedDataFiles) {
150+
this.path = path;
151+
this.fileSizeInBytes = fileSizeInBytes;
152+
this.metrics = metrics;
153+
this.partitionValues = partitionValues;
154+
this.content = content;
155+
this.referencedDataFiles = referencedDataFiles;
156+
}
157+
158+
public String getPath() {
159+
return path;
160+
}
161+
162+
public long getFileSizeInBytes() {
163+
return fileSizeInBytes;
164+
}
165+
166+
public Metrics getMetrics() {
167+
return metrics;
168+
}
169+
170+
public Optional<List<String>> getPartitionValues() {
171+
return partitionValues;
172+
}
173+
174+
public FileContent getContent() {
175+
return content;
176+
}
177+
178+
public Optional<List<String>> getReferencedDataFiles() {
179+
return referencedDataFiles;
180+
}
181+
}
182+
}

‎fe/fe-core/src/main/java/org/apache/doris/datasource/iceberg/IcebergUtils.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -50,6 +50,7 @@
5050
import org.apache.iceberg.Schema;
5151
import org.apache.iceberg.Snapshot;
5252
import org.apache.iceberg.Table;
53+
import org.apache.iceberg.TableProperties;
5354
import org.apache.iceberg.expressions.And;
5455
import org.apache.iceberg.expressions.Expression;
5556
import org.apache.iceberg.expressions.Expressions;
@@ -560,4 +561,14 @@ public static long getIcebergRowCount(ExternalCatalog catalog, String dbName, St
560561
}
561562
return -1;
562563
}
564+
565+
public static String getFileFormat(Table table) {
566+
Snapshot snapshot = table.currentSnapshot();
567+
if (snapshot == null) {
568+
return TableProperties.DEFAULT_FILE_FORMAT_DEFAULT;
569+
} else {
570+
return snapshot.summary().getOrDefault(
571+
TableProperties.DEFAULT_FILE_FORMAT, TableProperties.DEFAULT_FILE_FORMAT_DEFAULT);
572+
}
573+
}
563574
}

0 commit comments

Comments
 (0)