2727import org .apache .doris .fs .FileSystem ;
2828import org .apache .doris .fs .FileSystemUtil ;
2929import org .apache .doris .fs .remote .RemoteFile ;
30+ import org .apache .doris .nereids .trees .plans .commands .insert .HiveInsertCommandContext ;
3031import org .apache .doris .qe .ConnectContext ;
3132import org .apache .doris .thrift .THivePartitionUpdate ;
3233import org .apache .doris .thrift .TUpdateMode ;
@@ -72,13 +73,15 @@ public class HMSTransaction implements Transaction {
7273 private String dbName ;
7374 private String tbName ;
7475 private Optional <SummaryProfile > summaryProfile = Optional .empty ();
76+ private String queryId ;
7577
7678 private final Map <DatabaseTableName , Action <TableAndMore >> tableActions = new HashMap <>();
7779 private final Map <DatabaseTableName , Map <List <String >, Action <PartitionAndMore >>>
7880 partitionActions = new HashMap <>();
7981
8082 private HmsCommitter hmsCommitter ;
8183 private List <THivePartitionUpdate > hivePartitionUpdates = Lists .newArrayList ();
84+ private String declaredIntentionsToWrite ;
8285
8386 public HMSTransaction (HiveMetadataOps hiveOps ) {
8487 this .hiveOps = hiveOps ;
@@ -124,6 +127,11 @@ public void rollback() {
124127 }
125128 }
126129
130+ public void beginInsertTable (HiveInsertCommandContext ctx ) {
131+ declaredIntentionsToWrite = ctx .getWritePath ();
132+ queryId = ctx .getQueryId ();
133+ }
134+
127135 public void finishInsertTable (String dbName , String tbName ) {
128136 this .dbName = dbName ;
129137 this .tbName = tbName ;
@@ -239,8 +247,12 @@ public void doCommit() {
239247 hmsCommitter .doCommit ();
240248 } catch (Throwable t ) {
241249 LOG .warn ("Failed to commit for {}.{}, abort it." , dbName , tbName );
242- hmsCommitter .abort ();
243- hmsCommitter .rollback ();
250+ try {
251+ hmsCommitter .abort ();
252+ hmsCommitter .rollback ();
253+ } catch (RuntimeException e ) {
254+ t .addSuppressed (new Exception ("Failed to roll back after commit failure" , e ));
255+ }
244256 throw t ;
245257 } finally {
246258 hmsCommitter .runClearPathsForFinish ();
@@ -560,10 +572,10 @@ private DeleteRecursivelyResult recursiveDeleteFiles(Path directory, boolean del
560572 return new DeleteRecursivelyResult (false , notDeletedEligibleItems .build ());
561573 }
562574
563- return doRecursiveDeleteFiles (directory , deleteEmptyDir );
575+ return doRecursiveDeleteFiles (directory , deleteEmptyDir , queryId );
564576 }
565577
566- private DeleteRecursivelyResult doRecursiveDeleteFiles (Path directory , boolean deleteEmptyDir ) {
578+ private DeleteRecursivelyResult doRecursiveDeleteFiles (Path directory , boolean deleteEmptyDir , String queryId ) {
567579 List <RemoteFile > allFiles = new ArrayList <>();
568580 Set <String > allDirs = new HashSet <>();
569581 Status statusFile = fs .listFiles (directory .toString (), true , allFiles );
@@ -577,15 +589,18 @@ private DeleteRecursivelyResult doRecursiveDeleteFiles(Path directory, boolean d
577589 boolean allDescendentsDeleted = true ;
578590 ImmutableList .Builder <String > notDeletedEligibleItems = ImmutableList .builder ();
579591 for (RemoteFile file : allFiles ) {
580- String fileName = file .getName ();
581- if (!deleteIfExists (new Path (fileName ))) {
592+ if (file .getName ().startsWith (queryId )) {
593+ if (!deleteIfExists (file .getPath ())) {
594+ allDescendentsDeleted = false ;
595+ notDeletedEligibleItems .add (file .getPath ().toString ());
596+ }
597+ } else {
582598 allDescendentsDeleted = false ;
583- notDeletedEligibleItems .add (fileName );
584599 }
585600 }
586601
587602 for (String dir : allDirs ) {
588- DeleteRecursivelyResult subResult = doRecursiveDeleteFiles (new Path (dir ), deleteEmptyDir );
603+ DeleteRecursivelyResult subResult = doRecursiveDeleteFiles (new Path (dir ), deleteEmptyDir , queryId );
589604 if (!subResult .dirNotExists ()) {
590605 allDescendentsDeleted = false ;
591606 }
@@ -1038,6 +1053,9 @@ private void undoUpdateStatisticsTasks() {
10381053 }
10391054
10401055 private void undoAddPartitionsTask () {
1056+ if (addPartitionsTask .isEmpty ()) {
1057+ return ;
1058+ }
10411059
10421060 HivePartition firstPartition = addPartitionsTask .getPartitions ().get (0 ).getPartition ();
10431061 String dbName = firstPartition .getDbName ();
@@ -1319,6 +1337,10 @@ private void doUpdateStatisticsTasks() {
13191337 summaryProfile .ifPresent (SummaryProfile ::setHmsUpdatePartitionTime );
13201338 }
13211339
1340+ public void pruneAndDeleteStagingDirectories () {
1341+ recursiveDeleteItems (new Path (declaredIntentionsToWrite ), true );
1342+ }
1343+
13221344 public void doNothing () {
13231345 // do nothing
13241346 // only for regression test and unit test to throw exception
@@ -1342,6 +1364,7 @@ public void abort() {
13421364
13431365 public void rollback () {
13441366 //delete write path
1367+ pruneAndDeleteStagingDirectories ();
13451368 }
13461369 }
13471370
@@ -1372,22 +1395,22 @@ public Status wrapperDeleteWithProfileSummary(String remotePath) {
13721395 }
13731396
13741397 public void wrapperAsyncRenameWithProfileSummary (Executor executor ,
1375- List <CompletableFuture <?>> renameFileFutures ,
1376- AtomicBoolean cancelled ,
1377- String origFilePath ,
1378- String destFilePath ,
1379- List <String > fileNames ) {
1398+ List <CompletableFuture <?>> renameFileFutures ,
1399+ AtomicBoolean cancelled ,
1400+ String origFilePath ,
1401+ String destFilePath ,
1402+ List <String > fileNames ) {
13801403 FileSystemUtil .asyncRenameFiles (
13811404 fs , executor , renameFileFutures , cancelled , origFilePath , destFilePath , fileNames );
13821405 summaryProfile .ifPresent (profile -> profile .addRenameFileCnt (fileNames .size ()));
13831406 }
13841407
13851408 public void wrapperAsyncRenameDirWithProfileSummary (Executor executor ,
1386- List <CompletableFuture <?>> renameFileFutures ,
1387- AtomicBoolean cancelled ,
1388- String origFilePath ,
1389- String destFilePath ,
1390- Runnable runWhenPathNotExist ) {
1409+ List <CompletableFuture <?>> renameFileFutures ,
1410+ AtomicBoolean cancelled ,
1411+ String origFilePath ,
1412+ String destFilePath ,
1413+ Runnable runWhenPathNotExist ) {
13911414 FileSystemUtil .asyncRenameDir (
13921415 fs , executor , renameFileFutures , cancelled , origFilePath , destFilePath , runWhenPathNotExist );
13931416 summaryProfile .ifPresent (SummaryProfile ::incRenameDirCnt );
0 commit comments