Skip to content

Commit 5dc4ff7

Browse files
wuwenchiseawinde
authored andcommitted
[feature](hive)Get updated information from coordinate and commit (apache#32441)
issue: apache#31442 1. Get updated information from coordinate and commit 2. refresh table after commit
1 parent 004e69b commit 5dc4ff7

3 files changed

Lines changed: 44 additions & 1 deletion

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HiveMetadataOps.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -186,6 +186,15 @@ public void commit(String dbName,
186186
Table table = client.getTable(dbName, tableName);
187187
HMSCommitter hmsCommitter = new HMSCommitter(this, fs, table);
188188
hmsCommitter.commit(hivePUs);
189+
try {
190+
Env.getCurrentEnv().getCatalogMgr().refreshExternalTable(
191+
dbName,
192+
tableName,
193+
catalog.getName(),
194+
true);
195+
} catch (DdlException e) {
196+
LOG.warn("Failed to refresh table {}.{} : {}", dbName, tableName, e.getMessage());
197+
}
189198
}
190199

191200
public void updateTableStatistics(

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,10 @@
2020
import org.apache.doris.common.ErrorCode;
2121
import org.apache.doris.common.UserException;
2222
import org.apache.doris.common.util.DebugUtil;
23+
import org.apache.doris.datasource.ExternalCatalog;
2324
import org.apache.doris.datasource.hive.HMSExternalTable;
25+
import org.apache.doris.datasource.hive.HiveMetadataOps;
26+
import org.apache.doris.datasource.operations.ExternalMetadataOps;
2427
import org.apache.doris.nereids.NereidsPlanner;
2528
import org.apache.doris.nereids.exceptions.AnalysisException;
2629
import org.apache.doris.nereids.trees.plans.Plan;
@@ -32,12 +35,14 @@
3235
import org.apache.doris.qe.ConnectContext;
3336
import org.apache.doris.qe.QueryState;
3437
import org.apache.doris.qe.StmtExecutor;
38+
import org.apache.doris.thrift.THivePartitionUpdate;
3539
import org.apache.doris.transaction.TransactionStatus;
3640

3741
import com.google.common.base.Strings;
3842
import org.apache.logging.log4j.LogManager;
3943
import org.apache.logging.log4j.Logger;
4044

45+
import java.util.List;
4146
import java.util.Optional;
4247

4348
/**
@@ -88,6 +93,11 @@ protected void onComplete() throws UserException {
8893
if (ctx.getState().getStateType() == QueryState.MysqlStateType.ERR) {
8994
LOG.warn("errors when abort txn. {}", ctx.getQueryIdentifier());
9095
} else {
96+
// TODO use transaction
97+
List<THivePartitionUpdate> ups = coordinator.getHivePartitionUpdates();
98+
ExternalCatalog catalog = ((HMSExternalTable) table).getCatalog();
99+
ExternalMetadataOps metadataOps = catalog.getMetadataOps();
100+
((HiveMetadataOps) metadataOps).commit(((HMSExternalTable) table).getDbName(), table.getName(), ups);
91101
txnStatus = TransactionStatus.COMMITTED;
92102
}
93103
}

‎fe/fe-core/src/main/java/org/apache/doris/qe/Coordinator.java‎

Lines changed: 25 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,7 @@
8888
import org.apache.doris.thrift.TExternalScanRange;
8989
import org.apache.doris.thrift.TFileScanRange;
9090
import org.apache.doris.thrift.TFileScanRangeParams;
91+
import org.apache.doris.thrift.THivePartitionUpdate;
9192
import org.apache.doris.thrift.TNetworkAddress;
9293
import org.apache.doris.thrift.TPaloScanRange;
9394
import org.apache.doris.thrift.TPipelineFragmentParams;
@@ -229,6 +230,9 @@ public class Coordinator implements CoordInterface {
229230
private final List<TTabletCommitInfo> commitInfos = Lists.newArrayList();
230231
private final List<TErrorTabletInfo> errorTabletInfos = Lists.newArrayList();
231232

233+
// TODO moved to ExternalTransactionManager
234+
private final List<THivePartitionUpdate> hivePartitionUpdates = Lists.newArrayList();
235+
232236
// Input parameter
233237
private long jobId = -1; // job which this task belongs to
234238
private TUniqueId queryId;
@@ -477,6 +481,10 @@ public List<TErrorTabletInfo> getErrorTabletInfos() {
477481
return errorTabletInfos;
478482
}
479483

484+
public List<THivePartitionUpdate> getHivePartitionUpdates() {
485+
return hivePartitionUpdates;
486+
}
487+
480488
public Map<String, Integer> getBeToInstancesNum() {
481489
Map<String, Integer> result = Maps.newTreeMap();
482490
if (enablePipelineEngine) {
@@ -2400,7 +2408,14 @@ private void updateScanRangeNumByScanRange(TScanRangeParams param) {
24002408
// TODO: more ranges?
24012409
}
24022410

2403-
2411+
private void updateHivePartitionUpdates(List<THivePartitionUpdate> hivePartitionUpdates) {
2412+
lock.lock();
2413+
try {
2414+
this.hivePartitionUpdates.addAll(hivePartitionUpdates);
2415+
} finally {
2416+
lock.unlock();
2417+
}
2418+
}
24042419

24052420
// update job progress from BE
24062421
public void updateFragmentExecStatus(TReportExecStatusParams params) {
@@ -2450,6 +2465,9 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) {
24502465
if (params.isSetErrorTabletInfos()) {
24512466
updateErrorTabletInfos(params.getErrorTabletInfos());
24522467
}
2468+
if (params.isSetHivePartitionUpdates()) {
2469+
updateHivePartitionUpdates(params.getHivePartitionUpdates());
2470+
}
24532471

24542472
Preconditions.checkArgument(params.isSetDetailedReport());
24552473
if (ctx.done) {
@@ -2515,6 +2533,9 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) {
25152533
if (params.isSetErrorTabletInfos()) {
25162534
updateErrorTabletInfos(params.getErrorTabletInfos());
25172535
}
2536+
if (params.isSetHivePartitionUpdates()) {
2537+
updateHivePartitionUpdates(params.getHivePartitionUpdates());
2538+
}
25182539
if (LOG.isDebugEnabled()) {
25192540
LOG.debug("Query {} instance {} is marked done",
25202541
DebugUtil.printId(queryId), DebugUtil.printId(params.getFragmentInstanceId()));
@@ -2592,6 +2613,9 @@ public void updateFragmentExecStatus(TReportExecStatusParams params) {
25922613
if (params.isSetErrorTabletInfos()) {
25932614
updateErrorTabletInfos(params.getErrorTabletInfos());
25942615
}
2616+
if (params.isSetHivePartitionUpdates()) {
2617+
updateHivePartitionUpdates(params.getHivePartitionUpdates());
2618+
}
25952619
executionProfile.markOneInstanceDone(params.getFragmentInstanceId());
25962620
}
25972621
}

0 commit comments

Comments
 (0)