Skip to content

Commit 3bc8399

Browse files
wuwenchiHappenLee
authored andcommitted
[feature](hive)support ExternalTransaction for writing exteral table (apache#32726)
Issue apache#31442 Add `TransactionManager` and `Transaction`. ``` public interface Transaction { void commit() throws UserException; void rollback(); } public interface TransactionManager { long begin(); void commit(long id) throws UserException; void rollback(long id); Transaction getTransaction(long id); } ``` `TransactionManager` is used to manage all external transactions: The application layer should manage the entire transaction through this `TransactionManager`, like: ``` transactionManager.commit(); transactionManager.rollback(); ``` `Transaction` is an interface. You can implement this interface according to the specific content, such as `HMSTransaction` currently implemented, iceberg that may be implemented in the future, etc.
1 parent b8041d9 commit 3bc8399

14 files changed

Lines changed: 1597 additions & 822 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/ExternalCatalog.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -53,6 +53,7 @@
5353
import org.apache.doris.persist.gson.GsonUtils;
5454
import org.apache.doris.qe.ConnectContext;
5555
import org.apache.doris.qe.MasterCatalogExecutor;
56+
import org.apache.doris.transaction.TransactionManager;
5657

5758
import com.google.common.base.Strings;
5859
import com.google.common.collect.Lists;
@@ -114,6 +115,7 @@ public abstract class ExternalCatalog
114115
private boolean objectCreated = false;
115116
protected boolean invalidCacheInInit = true;
116117
protected ExternalMetadataOps metadataOps;
118+
protected TransactionManager transactionManager;
117119

118120
private ExternalSchemaCache schemaCache;
119121
private String comment;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSCommitter.java‎

Lines changed: 0 additions & 754 deletions
This file was deleted.

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSExternalCatalog.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -34,6 +34,7 @@
3434
import org.apache.doris.datasource.operations.ExternalMetadataOperations;
3535
import org.apache.doris.datasource.property.PropertyConverter;
3636
import org.apache.doris.datasource.property.constants.HMSProperties;
37+
import org.apache.doris.transaction.TransactionManagerFactory;
3738

3839
import com.google.common.base.Strings;
3940
import com.google.common.collect.Lists;
@@ -145,7 +146,10 @@ protected void initLocalObjectsImpl() {
145146
AuthenticationConfig.HADOOP_KERBEROS_PRINCIPAL,
146147
AuthenticationConfig.HADOOP_KERBEROS_KEYTAB));
147148
}
148-
metadataOps = ExternalMetadataOperations.newHiveMetadataOps(hiveConf, jdbcClientConfig, this);
149+
HiveMetadataOps hiveOps = ExternalMetadataOperations.newHiveMetadataOps(hiveConf, jdbcClientConfig, this);
150+
transactionManager = TransactionManagerFactory.createHiveTransactionManager(hiveOps);
151+
transactionManager.setEditLog(Env.getCurrentEnv().getEditLog());
152+
metadataOps = hiveOps;
149153
}
150154

151155
@Override

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java‎

Lines changed: 1322 additions & 0 deletions
Large diffs are not rendered by default.

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java‎

Lines changed: 5 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -36,13 +36,11 @@
3636
import org.apache.doris.datasource.operations.ExternalMetadataOps;
3737
import org.apache.doris.fs.remote.RemoteFileSystem;
3838
import org.apache.doris.fs.remote.dfs.DFSFileSystem;
39-
import org.apache.doris.thrift.THivePartitionUpdate;
4039

4140
import com.google.common.annotations.VisibleForTesting;
4241
import com.google.common.base.Preconditions;
4342
import com.google.common.collect.ImmutableSet;
4443
import org.apache.hadoop.hive.conf.HiveConf;
45-
import org.apache.hadoop.hive.metastore.api.Table;
4644
import org.apache.logging.log4j.LogManager;
4745
import org.apache.logging.log4j.Logger;
4846

@@ -73,13 +71,18 @@ public HiveMetadataOps(HiveConf hiveConf, JdbcClientConfig jdbcClientConfig, HMS
7371
public HiveMetadataOps(HMSExternalCatalog catalog, HMSCachedClient client) {
7472
this.catalog = catalog;
7573
this.client = client;
74+
// TODO Currently only supports DFSFileSystem, more types will be supported in the future
7675
this.fs = new DFSFileSystem(catalog.getProperties());
7776
}
7877

7978
public HMSCachedClient getClient() {
8079
return client;
8180
}
8281

82+
public RemoteFileSystem getFs() {
83+
return fs;
84+
}
85+
8386
public static HMSCachedClient createCachedClient(HiveConf hiveConf, int thriftClientPoolSize,
8487
JdbcClientConfig jdbcClientConfig) {
8588
if (hiveConf != null) {
@@ -253,23 +256,6 @@ public List<String> listDatabaseNames() {
253256
return client.getAllDatabases();
254257
}
255258

256-
public void commit(String dbName,
257-
String tableName,
258-
List<THivePartitionUpdate> hivePUs) {
259-
Table table = client.getTable(dbName, tableName);
260-
HMSCommitter hmsCommitter = new HMSCommitter(this, fs, table);
261-
hmsCommitter.commit(hivePUs);
262-
try {
263-
Env.getCurrentEnv().getCatalogMgr().refreshExternalTable(
264-
dbName,
265-
tableName,
266-
catalog.getName(),
267-
true);
268-
} catch (DdlException e) {
269-
LOG.warn("Failed to refresh table {}.{} : {}", dbName, tableName, e.getMessage());
270-
}
271-
}
272-
273259
public void updateTableStatistics(
274260
String dbName,
275261
String tableName,

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HivePartitionStatistics.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,7 @@
2525
import java.util.Map;
2626

2727
public class HivePartitionStatistics {
28-
private static final HivePartitionStatistics EMPTY =
28+
public static final HivePartitionStatistics EMPTY =
2929
new HivePartitionStatistics(HiveCommonStatistics.EMPTY, ImmutableMap.of());
3030

3131
private final HiveCommonStatistics commonStatistics;

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HivePartitionWithStatistics.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -18,9 +18,9 @@
1818
package org.apache.doris.datasource.hive;
1919

2020
public class HivePartitionWithStatistics {
21-
private String name;
22-
private HivePartition partition;
23-
private HivePartitionStatistics statistics;
21+
private final String name;
22+
private final HivePartition partition;
23+
private final HivePartitionStatistics statistics;
2424

2525
public HivePartitionWithStatistics(String name, HivePartition partition, HivePartitionStatistics statistics) {
2626
this.name = name;

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java‎

Lines changed: 22 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -17,13 +17,12 @@
1717

1818
package org.apache.doris.nereids.trees.plans.commands.insert;
1919

20+
import org.apache.doris.catalog.Env;
2021
import org.apache.doris.common.ErrorCode;
2122
import org.apache.doris.common.UserException;
2223
import org.apache.doris.common.util.DebugUtil;
23-
import org.apache.doris.datasource.ExternalCatalog;
2424
import org.apache.doris.datasource.hive.HMSExternalTable;
25-
import org.apache.doris.datasource.hive.HiveMetadataOps;
26-
import org.apache.doris.datasource.operations.ExternalMetadataOps;
25+
import org.apache.doris.datasource.hive.HMSTransaction;
2726
import org.apache.doris.nereids.NereidsPlanner;
2827
import org.apache.doris.nereids.exceptions.AnalysisException;
2928
import org.apache.doris.nereids.trees.plans.Plan;
@@ -35,14 +34,13 @@
3534
import org.apache.doris.qe.ConnectContext;
3635
import org.apache.doris.qe.QueryState;
3736
import org.apache.doris.qe.StmtExecutor;
38-
import org.apache.doris.thrift.THivePartitionUpdate;
37+
import org.apache.doris.transaction.TransactionManager;
3938
import org.apache.doris.transaction.TransactionStatus;
4039

4140
import com.google.common.base.Strings;
4241
import org.apache.logging.log4j.LogManager;
4342
import org.apache.logging.log4j.Logger;
4443

45-
import java.util.List;
4644
import java.util.Optional;
4745

4846
/**
@@ -53,6 +51,8 @@ public class HiveInsertExecutor extends AbstractInsertExecutor {
5351
private static final long INVALID_TXN_ID = -1L;
5452
private long txnId = INVALID_TXN_ID;
5553
private TransactionStatus txnStatus = TransactionStatus.ABORTED;
54+
private final TransactionManager transactionManager;
55+
private final String catalogName;
5656

5757
/**
5858
* constructor
@@ -61,6 +61,8 @@ public HiveInsertExecutor(ConnectContext ctx, HMSExternalTable table,
6161
String labelName, NereidsPlanner planner,
6262
Optional<InsertCommandContext> insertCtx) {
6363
super(ctx, table, labelName, planner, insertCtx);
64+
catalogName = table.getCatalog().getName();
65+
transactionManager = table.getCatalog().getTransactionManager();
6466
}
6567

6668
public long getTxnId() {
@@ -69,7 +71,9 @@ public long getTxnId() {
6971

7072
@Override
7173
public void beginTransaction() {
72-
// TODO: use hive txn rather than internal txn
74+
txnId = transactionManager.begin();
75+
HMSTransaction transaction = (HMSTransaction) transactionManager.getTransaction(txnId);
76+
coordinator.setHivePartitionUpdateFunc(transaction::updateHivePartitionUpdates);
7377
}
7478

7579
@Override
@@ -93,13 +97,18 @@ protected void onComplete() throws UserException {
9397
if (ctx.getState().getStateType() == QueryState.MysqlStateType.ERR) {
9498
LOG.warn("errors when abort txn. {}", ctx.getQueryIdentifier());
9599
} else {
96-
// TODO use transaction
97-
List<THivePartitionUpdate> ups = coordinator.getHivePartitionUpdates();
98-
loadedRows = ups.stream().mapToLong(THivePartitionUpdate::getRowCount).sum();
99-
ExternalCatalog catalog = ((HMSExternalTable) table).getCatalog();
100-
ExternalMetadataOps metadataOps = catalog.getMetadataOps();
101-
((HiveMetadataOps) metadataOps).commit(((HMSExternalTable) table).getDbName(), table.getName(), ups);
100+
HMSTransaction transaction = (HMSTransaction) transactionManager.getTransaction(txnId);
101+
loadedRows = transaction.getUpdateCnt();
102+
String dbName = ((HMSExternalTable) table).getDbName();
103+
String tbName = table.getName();
104+
transaction.finishInsertTable(dbName, tbName);
105+
transactionManager.commit(txnId);
102106
txnStatus = TransactionStatus.COMMITTED;
107+
Env.getCurrentEnv().getCatalogMgr().refreshExternalTable(
108+
dbName,
109+
tbName,
110+
catalogName,
111+
true);
103112
}
104113
}
105114

@@ -117,6 +126,7 @@ protected void onFail(Throwable t) {
117126
}
118127
}
119128
ctx.getState().setError(ErrorCode.ERR_UNKNOWN_ERROR, t.getMessage());
129+
transactionManager.rollback(txnId);
120130
}
121131

122132
@Override

‎fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java‎

Lines changed: 11 additions & 19 deletions
Original file line numberDiff line numberDiff line change
@@ -152,6 +152,7 @@
152152
import java.util.concurrent.TimeoutException;
153153
import java.util.concurrent.locks.Lock;
154154
import java.util.concurrent.locks.ReentrantLock;
155+
import java.util.function.Consumer;
155156
import java.util.stream.Collectors;
156157

157158
public class Coordinator implements CoordInterface {
@@ -235,8 +236,8 @@ public class Coordinator implements CoordInterface {
235236
private final List<TTabletCommitInfo> commitInfos = Lists.newArrayList();
236237
private final List<TErrorTabletInfo> errorTabletInfos = Lists.newArrayList();
237238

238-
// TODO moved to ExternalTransactionManager
239-
private final List<THivePartitionUpdate> hivePartitionUpdates = Lists.newArrayList();
239+
// Collect all hivePartitionUpdates obtained from be
240+
Consumer<List<THivePartitionUpdate>> hivePartitionUpdateFunc;
240241

241242
// Input parameter
242243
private long jobId = -1; // job which this task belongs to
@@ -503,10 +504,6 @@ public List<TErrorTabletInfo> getErrorTabletInfos() {
503504
return errorTabletInfos;
504505
}
505506

506-
public List<THivePartitionUpdate> getHivePartitionUpdates() {
507-
return hivePartitionUpdates;
508-
}
509-
510507
public Map<String, Integer> getBeToInstancesNum() {
511508
Map<String, Integer> result = Maps.newTreeMap();
512509
if (enablePipelineEngine) {
@@ -2442,13 +2439,8 @@ private void updateScanRangeNumByScanRange(TScanRangeParams param) {
24422439
// TODO: more ranges?
24432440
}
24442441

2445-
private void updateHivePartitionUpdates(List<THivePartitionUpdate> hivePartitionUpdates) {
2446-
lock.lock();
2447-
try {
2448-
this.hivePartitionUpdates.addAll(hivePartitionUpdates);
2449-
} finally {
2450-
lock.unlock();
2451-
}
2442+
public void setHivePartitionUpdateFunc(Consumer<List<THivePartitionUpdate>> hivePartitionUpdateFunc) {
2443+
this.hivePartitionUpdateFunc = hivePartitionUpdateFunc;
24522444
}
24532445

24542446
// update job progress from BE
@@ -2498,8 +2490,8 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) {
24982490
if (params.isSetErrorTabletInfos()) {
24992491
updateErrorTabletInfos(params.getErrorTabletInfos());
25002492
}
2501-
if (params.isSetHivePartitionUpdates()) {
2502-
updateHivePartitionUpdates(params.getHivePartitionUpdates());
2493+
if (params.isSetHivePartitionUpdates() && hivePartitionUpdateFunc != null) {
2494+
hivePartitionUpdateFunc.accept(params.getHivePartitionUpdates());
25032495
}
25042496

25052497
Preconditions.checkArgument(params.isSetDetailedReport());
@@ -2563,8 +2555,8 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) {
25632555
if (params.isSetErrorTabletInfos()) {
25642556
updateErrorTabletInfos(params.getErrorTabletInfos());
25652557
}
2566-
if (params.isSetHivePartitionUpdates()) {
2567-
updateHivePartitionUpdates(params.getHivePartitionUpdates());
2558+
if (params.isSetHivePartitionUpdates() && hivePartitionUpdateFunc != null) {
2559+
hivePartitionUpdateFunc.accept(params.getHivePartitionUpdates());
25682560
}
25692561
if (LOG.isDebugEnabled()) {
25702562
LOG.debug("Query {} instance {} is marked done",
@@ -2635,8 +2627,8 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) {
26352627
if (params.isSetErrorTabletInfos()) {
26362628
updateErrorTabletInfos(params.getErrorTabletInfos());
26372629
}
2638-
if (params.isSetHivePartitionUpdates()) {
2639-
updateHivePartitionUpdates(params.getHivePartitionUpdates());
2630+
if (params.isSetHivePartitionUpdates() && hivePartitionUpdateFunc != null) {
2631+
hivePartitionUpdateFunc.accept(params.getHivePartitionUpdates());
26402632
}
26412633
instancesDoneLatch.markedCountDown(params.getFragmentInstanceId(), -1L);
26422634
}
Lines changed: 79 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,79 @@
1+
// Licensed to the Apache Software Foundation (ASF) under one
2+
// or more contributor license agreements. See the NOTICE file
3+
// distributed with this work for additional information
4+
// regarding copyright ownership. The ASF licenses this file
5+
// to you under the Apache License, Version 2.0 (the
6+
// "License"); you may not use this file except in compliance
7+
// with the License. You may obtain a copy of the License at
8+
//
9+
// http://www.apache.org/licenses/LICENSE-2.0
10+
//
11+
// Unless required by applicable law or agreed to in writing,
12+
// software distributed under the License is distributed on an
13+
// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
14+
// KIND, either express or implied. See the License for the
15+
// specific language governing permissions and limitations
16+
// under the License.
17+
18+
package org.apache.doris.transaction;
19+
20+
import org.apache.doris.common.UserException;
21+
import org.apache.doris.datasource.hive.HMSTransaction;
22+
import org.apache.doris.datasource.hive.HiveMetadataOps;
23+
import org.apache.doris.persist.EditLog;
24+
25+
import java.util.Map;
26+
import java.util.concurrent.ConcurrentHashMap;
27+
28+
public class HiveTransactionManager implements TransactionManager {
29+
30+
private final Map<Long, HMSTransaction> transactions = new ConcurrentHashMap<>();
31+
private final TransactionIdGenerator idGenerator = new TransactionIdGenerator();
32+
private final HiveMetadataOps ops;
33+
34+
public HiveTransactionManager(HiveMetadataOps ops) {
35+
this.ops = ops;
36+
}
37+
38+
public Long getNextTransactionId() {
39+
return idGenerator.getNextTransactionId();
40+
}
41+
42+
@Override
43+
public void setEditLog(EditLog editLog) {
44+
this.idGenerator.setEditLog(editLog);
45+
}
46+
47+
@Override
48+
public long begin() {
49+
long id = idGenerator.getNextTransactionId();
50+
HMSTransaction hiveTransaction = new HMSTransaction(ops);
51+
transactions.put(id, hiveTransaction);
52+
return id;
53+
}
54+
55+
@Override
56+
public void commit(long id) throws UserException {
57+
getTransactionWithException(id).commit();
58+
transactions.remove(id);
59+
}
60+
61+
@Override
62+
public void rollback(long id) {
63+
getTransactionWithException(id).rollback();
64+
transactions.remove(id);
65+
}
66+
67+
@Override
68+
public HMSTransaction getTransaction(long id) {
69+
return getTransactionWithException(id);
70+
}
71+
72+
public HMSTransaction getTransactionWithException(long id) {
73+
HMSTransaction hiveTransaction = transactions.get(id);
74+
if (hiveTransaction == null) {
75+
throw new RuntimeException("Can't find transaction for " + id);
76+
}
77+
return hiveTransaction;
78+
}
79+
}

0 commit comments

Comments
 (0)