Skip to content

Commit 02927fb

Browse files
wuwenchiHappenLee
authored andcommitted
[bugfix](hive)delete write path after hive insert (apache#33798)
Issue apache#31442 1. delete file according query id 2. delete write path after insert
1 parent d2b74bf commit 02927fb

6 files changed

Lines changed: 215 additions & 48 deletions

File tree

‎fe/fe-core/src/main/java/org/apache/doris/datasource/hive/HMSTransaction.java‎

Lines changed: 41 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -27,6 +27,7 @@
2727
import org.apache.doris.fs.FileSystem;
2828
import org.apache.doris.fs.FileSystemUtil;
2929
import org.apache.doris.fs.remote.RemoteFile;
30+
import org.apache.doris.nereids.trees.plans.commands.insert.HiveInsertCommandContext;
3031
import org.apache.doris.qe.ConnectContext;
3132
import org.apache.doris.thrift.THivePartitionUpdate;
3233
import org.apache.doris.thrift.TUpdateMode;
@@ -72,13 +73,15 @@ public class HMSTransaction implements Transaction {
7273
private String dbName;
7374
private String tbName;
7475
private Optional<SummaryProfile> summaryProfile = Optional.empty();
76+
private String queryId;
7577

7678
private final Map<DatabaseTableName, Action<TableAndMore>> tableActions = new HashMap<>();
7779
private final Map<DatabaseTableName, Map<List<String>, Action<PartitionAndMore>>>
7880
partitionActions = new HashMap<>();
7981

8082
private HmsCommitter hmsCommitter;
8183
private List<THivePartitionUpdate> hivePartitionUpdates = Lists.newArrayList();
84+
private String declaredIntentionsToWrite;
8285

8386
public HMSTransaction(HiveMetadataOps hiveOps) {
8487
this.hiveOps = hiveOps;
@@ -124,6 +127,11 @@ public void rollback() {
124127
}
125128
}
126129

130+
public void beginInsertTable(HiveInsertCommandContext ctx) {
131+
declaredIntentionsToWrite = ctx.getWritePath();
132+
queryId = ctx.getQueryId();
133+
}
134+
127135
public void finishInsertTable(String dbName, String tbName) {
128136
this.dbName = dbName;
129137
this.tbName = tbName;
@@ -239,8 +247,12 @@ public void doCommit() {
239247
hmsCommitter.doCommit();
240248
} catch (Throwable t) {
241249
LOG.warn("Failed to commit for {}.{}, abort it.", dbName, tbName);
242-
hmsCommitter.abort();
243-
hmsCommitter.rollback();
250+
try {
251+
hmsCommitter.abort();
252+
hmsCommitter.rollback();
253+
} catch (RuntimeException e) {
254+
t.addSuppressed(new Exception("Failed to roll back after commit failure", e));
255+
}
244256
throw t;
245257
} finally {
246258
hmsCommitter.runClearPathsForFinish();
@@ -560,10 +572,10 @@ private DeleteRecursivelyResult recursiveDeleteFiles(Path directory, boolean del
560572
return new DeleteRecursivelyResult(false, notDeletedEligibleItems.build());
561573
}
562574

563-
return doRecursiveDeleteFiles(directory, deleteEmptyDir);
575+
return doRecursiveDeleteFiles(directory, deleteEmptyDir, queryId);
564576
}
565577

566-
private DeleteRecursivelyResult doRecursiveDeleteFiles(Path directory, boolean deleteEmptyDir) {
578+
private DeleteRecursivelyResult doRecursiveDeleteFiles(Path directory, boolean deleteEmptyDir, String queryId) {
567579
List<RemoteFile> allFiles = new ArrayList<>();
568580
Set<String> allDirs = new HashSet<>();
569581
Status statusFile = fs.listFiles(directory.toString(), true, allFiles);
@@ -577,15 +589,18 @@ private DeleteRecursivelyResult doRecursiveDeleteFiles(Path directory, boolean d
577589
boolean allDescendentsDeleted = true;
578590
ImmutableList.Builder<String> notDeletedEligibleItems = ImmutableList.builder();
579591
for (RemoteFile file : allFiles) {
580-
String fileName = file.getName();
581-
if (!deleteIfExists(new Path(fileName))) {
592+
if (file.getName().startsWith(queryId)) {
593+
if (!deleteIfExists(file.getPath())) {
594+
allDescendentsDeleted = false;
595+
notDeletedEligibleItems.add(file.getPath().toString());
596+
}
597+
} else {
582598
allDescendentsDeleted = false;
583-
notDeletedEligibleItems.add(fileName);
584599
}
585600
}
586601

587602
for (String dir : allDirs) {
588-
DeleteRecursivelyResult subResult = doRecursiveDeleteFiles(new Path(dir), deleteEmptyDir);
603+
DeleteRecursivelyResult subResult = doRecursiveDeleteFiles(new Path(dir), deleteEmptyDir, queryId);
589604
if (!subResult.dirNotExists()) {
590605
allDescendentsDeleted = false;
591606
}
@@ -1038,6 +1053,9 @@ private void undoUpdateStatisticsTasks() {
10381053
}
10391054

10401055
private void undoAddPartitionsTask() {
1056+
if (addPartitionsTask.isEmpty()) {
1057+
return;
1058+
}
10411059

10421060
HivePartition firstPartition = addPartitionsTask.getPartitions().get(0).getPartition();
10431061
String dbName = firstPartition.getDbName();
@@ -1319,6 +1337,10 @@ private void doUpdateStatisticsTasks() {
13191337
summaryProfile.ifPresent(SummaryProfile::setHmsUpdatePartitionTime);
13201338
}
13211339

1340+
public void pruneAndDeleteStagingDirectories() {
1341+
recursiveDeleteItems(new Path(declaredIntentionsToWrite), true);
1342+
}
1343+
13221344
public void doNothing() {
13231345
// do nothing
13241346
// only for regression test and unit test to throw exception
@@ -1342,6 +1364,7 @@ public void abort() {
13421364

13431365
public void rollback() {
13441366
//delete write path
1367+
pruneAndDeleteStagingDirectories();
13451368
}
13461369
}
13471370

@@ -1372,22 +1395,22 @@ public Status wrapperDeleteWithProfileSummary(String remotePath) {
13721395
}
13731396

13741397
public void wrapperAsyncRenameWithProfileSummary(Executor executor,
1375-
List<CompletableFuture<?>> renameFileFutures,
1376-
AtomicBoolean cancelled,
1377-
String origFilePath,
1378-
String destFilePath,
1379-
List<String> fileNames) {
1398+
List<CompletableFuture<?>> renameFileFutures,
1399+
AtomicBoolean cancelled,
1400+
String origFilePath,
1401+
String destFilePath,
1402+
List<String> fileNames) {
13801403
FileSystemUtil.asyncRenameFiles(
13811404
fs, executor, renameFileFutures, cancelled, origFilePath, destFilePath, fileNames);
13821405
summaryProfile.ifPresent(profile -> profile.addRenameFileCnt(fileNames.size()));
13831406
}
13841407

13851408
public void wrapperAsyncRenameDirWithProfileSummary(Executor executor,
1386-
List<CompletableFuture<?>> renameFileFutures,
1387-
AtomicBoolean cancelled,
1388-
String origFilePath,
1389-
String destFilePath,
1390-
Runnable runWhenPathNotExist) {
1409+
List<CompletableFuture<?>> renameFileFutures,
1410+
AtomicBoolean cancelled,
1411+
String origFilePath,
1412+
String destFilePath,
1413+
Runnable runWhenPathNotExist) {
13911414
FileSystemUtil.asyncRenameDir(
13921415
fs, executor, renameFileFutures, cancelled, origFilePath, destFilePath, runWhenPathNotExist);
13931416
summaryProfile.ifPresent(SummaryProfile::incRenameDirCnt);

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertCommandContext.java‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,8 @@
2222
*/
2323
public class HiveInsertCommandContext extends InsertCommandContext {
2424
private boolean overwrite = false;
25+
private String writePath;
26+
private String queryId;
2527

2628
public boolean isOverwrite() {
2729
return overwrite;
@@ -30,4 +32,20 @@ public boolean isOverwrite() {
3032
public void setOverwrite(boolean overwrite) {
3133
this.overwrite = overwrite;
3234
}
35+
36+
public String getWritePath() {
37+
return writePath;
38+
}
39+
40+
public void setWritePath(String writePath) {
41+
this.writePath = writePath;
42+
}
43+
44+
public String getQueryId() {
45+
return queryId;
46+
}
47+
48+
public void setQueryId(String queryId) {
49+
this.queryId = queryId;
50+
}
3351
}

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/HiveInsertExecutor.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,10 +35,12 @@
3535
import org.apache.doris.qe.ConnectContext;
3636
import org.apache.doris.qe.QueryState;
3737
import org.apache.doris.qe.StmtExecutor;
38+
import org.apache.doris.thrift.TUniqueId;
3839
import org.apache.doris.transaction.TransactionManager;
3940
import org.apache.doris.transaction.TransactionStatus;
4041
import org.apache.doris.transaction.TransactionType;
4142

43+
import com.google.common.base.Preconditions;
4244
import com.google.common.base.Strings;
4345
import org.apache.logging.log4j.LogManager;
4446
import org.apache.logging.log4j.Logger;
@@ -97,6 +99,13 @@ protected void finalizeSink(PlanFragment fragment, DataSink sink, PhysicalSink p
9799
@Override
98100
protected void beforeExec() {
99101
// check params
102+
HMSTransaction transaction = (HMSTransaction) transactionManager.getTransaction(txnId);
103+
Preconditions.checkArgument(insertCtx.isPresent(), "insert context must be present");
104+
HiveInsertCommandContext ctx = (HiveInsertCommandContext) insertCtx.get();
105+
TUniqueId tUniqueId = ConnectContext.get().queryId();
106+
Preconditions.checkArgument(tUniqueId != null, "query id shouldn't be null");
107+
ctx.setQueryId(DebugUtil.printId(tUniqueId));
108+
transaction.beginInsertTable(ctx);
100109
}
101110

102111
@Override

‎fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/commands/insert/InsertIntoTableCommand.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -173,7 +173,8 @@ public AbstractInsertExecutor initPlan(ConnectContext ctx, StmtExecutor executor
173173
.setEnableMemtableOnSinkNode(isEnableMemtableOnSinkNode);
174174
} else if (physicalSink instanceof PhysicalHiveTableSink) {
175175
HMSExternalTable hiveExternalTable = (HMSExternalTable) targetTableIf;
176-
insertExecutor = new HiveInsertExecutor(ctx, hiveExternalTable, label, planner, insertCtx);
176+
insertExecutor = new HiveInsertExecutor(ctx, hiveExternalTable, label, planner,
177+
Optional.of(insertCtx.orElse((new HiveInsertCommandContext()))));
177178
// set hive query options
178179
} else {
179180
// TODO: support other table types

‎fe/fe-core/src/main/java/org/apache/doris/planner/HiveTableSink.java‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -141,6 +141,7 @@ public void bindDataSink(List<Column> insertCols, Optional<InsertCommandContext>
141141
if (insertCtx.isPresent()) {
142142
HiveInsertCommandContext context = (HiveInsertCommandContext) insertCtx.get();
143143
tSink.setOverwrite(context.isOverwrite());
144+
context.setWritePath(writeTempPath);
144145
}
145146
tDataSink = new TDataSink(getDataSinkType());
146147
tDataSink.setHiveTableSink(tSink);

0 commit comments

Comments
 (0)