Skip to content

Commit 52da35f

Browse files
committed
[feature](iceberg) add iceberg transaction implement (apache#33629)
Issue apache#31442 add iceberg transaction (cherry picked from commit 68b33c8)
1 parent 817306f commit 52da35f

3 files changed

Lines changed: 397 additions & 5 deletions

File tree

Lines changed: 328 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,328 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.datasource.iceberg;
19+
20+
import org.apache.doris.common.UserException;
21+
import org.apache.doris.thrift.TFileContent;
22+
import org.apache.doris.thrift.TIcebergCommitData;
23+
24+
import org.apache.hadoop.conf.Configuration;
25+
import org.apache.iceberg.CatalogProperties;
26+
import org.apache.iceberg.FileScanTask;
27+
import org.apache.iceberg.PartitionSpec;
28+
import org.apache.iceberg.Schema;
29+
import org.apache.iceberg.Table;
30+
import org.apache.iceberg.catalog.Namespace;
31+
import org.apache.iceberg.catalog.TableIdentifier;
32+
import org.apache.iceberg.expressions.Expression;
33+
import org.apache.iceberg.expressions.Expressions;
34+
import org.apache.iceberg.expressions.UnboundPredicate;
35+
import org.apache.iceberg.hadoop.HadoopCatalog;
36+
import org.apache.iceberg.io.CloseableIterable;
37+
import org.apache.iceberg.transforms.Transform;
38+
import org.apache.iceberg.transforms.Transforms;
39+
import org.apache.iceberg.types.Types;
40+
import org.apache.iceberg.util.DateTimeUtil;
41+
import org.junit.Assert;
42+
import org.junit.BeforeClass;
43+
import org.junit.Test;
44+
45+
import java.io.IOException;
46+
import java.nio.file.Files;
47+
import java.nio.file.Path;
48+
import java.time.Instant;
49+
import java.util.ArrayList;
50+
import java.util.HashMap;
51+
import java.util.List;
52+
import java.util.Map;
53+
import java.util.concurrent.atomic.AtomicReference;
54+
55+
public class IcebergTransactionTest {
56+
57+
public static String dbName = "db3";
58+
public static String tbWithPartition = "tbWithPartition";
59+
public static String tbWithoutPartition = "tbWithoutPartition";
60+
public static IcebergMetadataOps ops;
61+
public static Schema schema;
62+
63+
@BeforeClass
64+
public static void beforeClass() throws IOException {
65+
createCatalog();
66+
createTable();
67+
}
68+
69+
public static void createCatalog() throws IOException {
70+
Path warehousePath = Files.createTempDirectory("test_warehouse_");
71+
String warehouse = "file://" + warehousePath.toAbsolutePath() + "/";
72+
HadoopCatalog hadoopCatalog = new HadoopCatalog();
73+
Map<String, String> props = new HashMap<>();
74+
props.put(CatalogProperties.WAREHOUSE_LOCATION, warehouse);
75+
hadoopCatalog.setConf(new Configuration());
76+
hadoopCatalog.initialize("df", props);
77+
ops = new IcebergMetadataOps(null, hadoopCatalog);
78+
}
79+
80+
public static void createTable() throws IOException {
81+
HadoopCatalog icebergCatalog = (HadoopCatalog) ops.getCatalog();
82+
icebergCatalog.createNamespace(Namespace.of(dbName));
83+
schema = new Schema(
84+
Types.NestedField.required(11, "ts1", Types.TimestampType.withoutZone()),
85+
Types.NestedField.required(12, "ts2", Types.TimestampType.withoutZone()),
86+
Types.NestedField.required(13, "ts3", Types.TimestampType.withoutZone()),
87+
Types.NestedField.required(14, "ts4", Types.TimestampType.withoutZone()),
88+
Types.NestedField.required(15, "dt1", Types.DateType.get()),
89+
Types.NestedField.required(16, "dt2", Types.DateType.get()),
90+
Types.NestedField.required(17, "dt3", Types.DateType.get()),
91+
Types.NestedField.required(18, "dt4", Types.DateType.get()),
92+
Types.NestedField.required(19, "str1", Types.StringType.get()),
93+
Types.NestedField.required(20, "str2", Types.StringType.get()),
94+
Types.NestedField.required(21, "int1", Types.IntegerType.get()),
95+
Types.NestedField.required(22, "int2", Types.IntegerType.get())
96+
);
97+
98+
PartitionSpec partitionSpec = PartitionSpec.builderFor(schema)
99+
.year("ts1")
100+
.month("ts2")
101+
.day("ts3")
102+
.hour("ts4")
103+
.year("dt1")
104+
.month("dt2")
105+
.day("dt3")
106+
.identity("dt4")
107+
.identity("str1")
108+
.truncate("str2", 10)
109+
.bucket("int1", 2)
110+
.build();
111+
icebergCatalog.createTable(TableIdentifier.of(dbName, tbWithPartition), schema, partitionSpec);
112+
icebergCatalog.createTable(TableIdentifier.of(dbName, tbWithoutPartition), schema);
113+
}
114+
115+
public List<String> createPartitionValues() {
116+
117+
Instant instant = Instant.parse("2024-12-11T12:34:56.123456Z");
118+
long ts = DateTimeUtil.microsFromInstant(instant);
119+
int dt = DateTimeUtil.daysFromInstant(instant);
120+
121+
List<String> partitionValues = new ArrayList<>();
122+
123+
// reference: org.apache.iceberg.transforms.Timestamps
124+
partitionValues.add(Integer.valueOf(DateTimeUtil.microsToYears(ts)).toString());
125+
partitionValues.add(Integer.valueOf(DateTimeUtil.microsToMonths(ts)).toString());
126+
partitionValues.add("2024-12-11");
127+
partitionValues.add(Integer.valueOf(DateTimeUtil.microsToHours(ts)).toString());
128+
129+
// reference: org.apache.iceberg.transforms.Dates
130+
partitionValues.add(Integer.valueOf(DateTimeUtil.daysToYears(dt)).toString());
131+
partitionValues.add(Integer.valueOf(DateTimeUtil.daysToMonths(dt)).toString());
132+
partitionValues.add("2024-12-11");
133+
134+
// identity dt4
135+
partitionValues.add("2024-12-11");
136+
// identity str1
137+
partitionValues.add("2024-12-11");
138+
// truncate str2
139+
partitionValues.add("2024-12-11");
140+
// bucket int1
141+
partitionValues.add("1");
142+
143+
return partitionValues;
144+
}
145+
146+
@Test
147+
public void testPartitionedTable() throws UserException {
148+
List<String> partitionValues = createPartitionValues();
149+
150+
List<TIcebergCommitData> ctdList = new ArrayList<>();
151+
TIcebergCommitData ctd1 = new TIcebergCommitData();
152+
ctd1.setFilePath("f1.parquet");
153+
ctd1.setPartitionValues(partitionValues);
154+
ctd1.setFileContent(TFileContent.DATA);
155+
ctd1.setRowCount(2);
156+
ctd1.setFileSize(2);
157+
158+
TIcebergCommitData ctd2 = new TIcebergCommitData();
159+
ctd2.setFilePath("f2.parquet");
160+
ctd2.setPartitionValues(partitionValues);
161+
ctd2.setFileContent(TFileContent.DATA);
162+
ctd2.setRowCount(4);
163+
ctd2.setFileSize(4);
164+
165+
ctdList.add(ctd1);
166+
ctdList.add(ctd2);
167+
168+
IcebergTransaction txn = getTxn();
169+
txn.updateIcebergCommitData(ctdList);
170+
txn.beginInsert(dbName, tbWithPartition);
171+
txn.finishInsert();
172+
txn.commit();
173+
Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithPartition));
174+
checkSnapshotProperties(table.currentSnapshot().summary(), "6", "2", "6");
175+
176+
checkPushDownByPartitionForTs(table, "ts1");
177+
checkPushDownByPartitionForTs(table, "ts2");
178+
checkPushDownByPartitionForTs(table, "ts3");
179+
checkPushDownByPartitionForTs(table, "ts4");
180+
181+
checkPushDownByPartitionForDt(table, "dt1");
182+
checkPushDownByPartitionForDt(table, "dt2");
183+
checkPushDownByPartitionForDt(table, "dt3");
184+
checkPushDownByPartitionForDt(table, "dt4");
185+
186+
checkPushDownByPartitionForString(table, "str1");
187+
checkPushDownByPartitionForString(table, "str2");
188+
189+
checkPushDownByPartitionForBucketInt(table, "int1");
190+
}
191+
192+
public void checkPushDownByPartitionForBucketInt(Table table, String column) {
193+
// (BucketUtil.hash(15) & Integer.MAX_VALUE) % 2 = 0
194+
Integer i1 = 15;
195+
196+
UnboundPredicate<Integer> lessThan = Expressions.lessThan(column, i1);
197+
checkPushDownByPartition(table, lessThan, 2);
198+
// can only filter this case
199+
UnboundPredicate<Integer> equal = Expressions.equal(column, i1);
200+
checkPushDownByPartition(table, equal, 0);
201+
UnboundPredicate<Integer> greaterThan = Expressions.greaterThan(column, i1);
202+
checkPushDownByPartition(table, greaterThan, 2);
203+
204+
// (BucketUtil.hash(25) & Integer.MAX_VALUE) % 2 = 1
205+
Integer i2 = 25;
206+
207+
UnboundPredicate<Integer> lessThan2 = Expressions.lessThan(column, i2);
208+
checkPushDownByPartition(table, lessThan2, 2);
209+
UnboundPredicate<Integer> equal2 = Expressions.equal(column, i2);
210+
checkPushDownByPartition(table, equal2, 2);
211+
UnboundPredicate<Integer> greaterThan2 = Expressions.greaterThan(column, i2);
212+
checkPushDownByPartition(table, greaterThan2, 2);
213+
}
214+
215+
public void checkPushDownByPartitionForString(Table table, String column) {
216+
// Since the string used to create the partition is in date format, the date check can be reused directly
217+
checkPushDownByPartitionForDt(table, column);
218+
}
219+
220+
public void checkPushDownByPartitionForTs(Table table, String column) {
221+
String lessTs = "2023-12-11T12:34:56.123456";
222+
String eqTs = "2024-12-11T12:34:56.123456";
223+
String greaterTs = "2025-12-11T12:34:56.123456";
224+
225+
UnboundPredicate<String> lessThan = Expressions.lessThan(column, lessTs);
226+
checkPushDownByPartition(table, lessThan, 0);
227+
UnboundPredicate<String> equal = Expressions.equal(column, eqTs);
228+
checkPushDownByPartition(table, equal, 2);
229+
UnboundPredicate<String> greaterThan = Expressions.greaterThan(column, greaterTs);
230+
checkPushDownByPartition(table, greaterThan, 0);
231+
}
232+
233+
public void checkPushDownByPartitionForDt(Table table, String column) {
234+
String less = "2023-12-11";
235+
String eq = "2024-12-11";
236+
String greater = "2025-12-11";
237+
238+
UnboundPredicate<String> lessThan = Expressions.lessThan(column, less);
239+
checkPushDownByPartition(table, lessThan, 0);
240+
UnboundPredicate<String> equal = Expressions.equal(column, eq);
241+
checkPushDownByPartition(table, equal, 2);
242+
UnboundPredicate<String> greaterThan = Expressions.greaterThan(column, greater);
243+
checkPushDownByPartition(table, greaterThan, 0);
244+
}
245+
246+
public void checkPushDownByPartition(Table table, Expression expr, Integer expectFiles) {
247+
CloseableIterable<FileScanTask> fileScanTasks = table.newScan().filter(expr).planFiles();
248+
AtomicReference<Integer> cnt = new AtomicReference<>(0);
249+
fileScanTasks.forEach(notUse -> cnt.updateAndGet(v -> v + 1));
250+
Assert.assertEquals(expectFiles, cnt.get());
251+
}
252+
253+
@Test
254+
public void testUnPartitionedTable() throws UserException {
255+
ArrayList<TIcebergCommitData> ctdList = new ArrayList<>();
256+
TIcebergCommitData ctd1 = new TIcebergCommitData();
257+
ctd1.setFilePath("f1.parquet");
258+
ctd1.setFileContent(TFileContent.DATA);
259+
ctd1.setRowCount(2);
260+
ctd1.setFileSize(2);
261+
262+
TIcebergCommitData ctd2 = new TIcebergCommitData();
263+
ctd2.setFilePath("f1.parquet");
264+
ctd2.setFileContent(TFileContent.DATA);
265+
ctd2.setRowCount(4);
266+
ctd2.setFileSize(4);
267+
268+
ctdList.add(ctd1);
269+
ctdList.add(ctd2);
270+
271+
IcebergTransaction txn = getTxn();
272+
txn.updateIcebergCommitData(ctdList);
273+
txn.beginInsert(dbName, tbWithoutPartition);
274+
txn.finishInsert();
275+
txn.commit();
276+
277+
Table table = ops.getCatalog().loadTable(TableIdentifier.of(dbName, tbWithoutPartition));
278+
checkSnapshotProperties(table.currentSnapshot().summary(), "6", "2", "6");
279+
}
280+
281+
public void checkSnapshotProperties(Map<String, String> props,
282+
String addRecords,
283+
String addFileCnt,
284+
String addFileSize) {
285+
Assert.assertEquals(addRecords, props.get("added-records"));
286+
Assert.assertEquals(addFileCnt, props.get("added-data-files"));
287+
Assert.assertEquals(addFileSize, props.get("added-files-size"));
288+
}
289+
290+
public String numToYear(Integer num) {
291+
Transform<Object, Integer> year = Transforms.year();
292+
return year.toHumanString(Types.IntegerType.get(), num);
293+
}
294+
295+
public String numToMonth(Integer num) {
296+
Transform<Object, Integer> month = Transforms.month();
297+
return month.toHumanString(Types.IntegerType.get(), num);
298+
}
299+
300+
public String numToDay(Integer num) {
301+
Transform<Object, Integer> day = Transforms.day();
302+
return day.toHumanString(Types.IntegerType.get(), num);
303+
}
304+
305+
public String numToHour(Integer num) {
306+
Transform<Object, Integer> hour = Transforms.hour();
307+
return hour.toHumanString(Types.IntegerType.get(), num);
308+
}
309+
310+
@Test
311+
public void testTransform() {
312+
Instant instant = Instant.parse("2024-12-11T12:34:56.123456Z");
313+
long ts = DateTimeUtil.microsFromInstant(instant);
314+
Assert.assertEquals("2024", numToYear(DateTimeUtil.microsToYears(ts)));
315+
Assert.assertEquals("2024-12", numToMonth(DateTimeUtil.microsToMonths(ts)));
316+
Assert.assertEquals("2024-12-11", numToDay(DateTimeUtil.microsToDays(ts)));
317+
Assert.assertEquals("2024-12-11-12", numToHour(DateTimeUtil.microsToHours(ts)));
318+
319+
int dt = DateTimeUtil.daysFromInstant(instant);
320+
Assert.assertEquals("2024", numToYear(DateTimeUtil.daysToYears(dt)));
321+
Assert.assertEquals("2024-12", numToMonth(DateTimeUtil.daysToMonths(dt)));
322+
Assert.assertEquals("2024-12-11", numToDay(dt));
323+
}
324+
325+
public IcebergTransaction getTxn() {
326+
return new IcebergTransaction(ops);
327+
}
328+
}

‎gensrc/thrift/DataSinks.thrift‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -395,6 +395,21 @@ struct TIcebergTableSink {
395395
13: optional PlanNodes.TFileCompressType compression_type
396396
}
397397

398+
enum TFileContent {
399+
DATA = 0,
400+
POSITION_DELETES = 1,
401+
EQUALITY_DELETES = 2
402+
}
403+
404+
struct TIcebergCommitData {
405+
1: optional string file_path
406+
2: optional i64 row_count
407+
3: optional i64 file_size
408+
4: optional TFileContent file_content
409+
5: optional list<string> partition_values
410+
6: optional list<string> referenced_data_files
411+
}
412+
398413
struct TDataSink {
399414
1: required TDataSinkType type
400415
2: optional TDataStreamSink stream_sink

0 commit comments

Comments
 (0)