2323
2424import org .apache .doris .backup .Status ;
2525import org .apache .doris .common .Pair ;
26+ import org .apache .doris .fs .FileSystem ;
2627import org .apache .doris .fs .remote .RemoteFile ;
27- import org .apache .doris .fs .remote .RemoteFileSystem ;
2828import org .apache .doris .thrift .THivePartitionUpdate ;
2929import org .apache .doris .thrift .TUpdateMode ;
3030import org .apache .doris .transaction .Transaction ;
4747
4848import java .util .ArrayList ;
4949import java .util .HashMap ;
50+ import java .util .HashSet ;
5051import java .util .List ;
5152import java .util .Map ;
5253import java .util .Objects ;
5354import java .util .Optional ;
5455import java .util .Queue ;
56+ import java .util .Set ;
5557import java .util .StringJoiner ;
5658import java .util .concurrent .CompletableFuture ;
5759import java .util .concurrent .ConcurrentLinkedQueue ;
6365public class HMSTransaction implements Transaction {
6466 private static final Logger LOG = LogManager .getLogger (HMSTransaction .class );
6567 private final HiveMetadataOps hiveOps ;
66- private final RemoteFileSystem fs ;
68+ private final FileSystem fs ;
6769 private String dbName ;
6870 private String tbName ;
6971
@@ -115,8 +117,8 @@ public void rollback() {
115117 }
116118
117119 public void finishInsertTable (String dbName , String tbName ) {
118- this .tbName = tbName ;
119120 this .dbName = dbName ;
121+ this .tbName = tbName ;
120122 List <THivePartitionUpdate > mergedPUs = mergePartitions (hivePartitionUpdates );
121123 Table table = getTable (dbName , tbName );
122124 List <Pair <THivePartitionUpdate , HivePartitionStatistics >> insertExistsPartitions = new ArrayList <>();
@@ -226,17 +228,10 @@ public void doCommit() {
226228 }
227229 }
228230
229- hmsCommitter .waitForAsyncFileSystemTasks ();
230- hmsCommitter .doAddPartitionsTask ();
231- hmsCommitter .doUpdateStatisticsTasks ();
231+ hmsCommitter .doCommit ();
232232 } catch (Throwable t ) {
233233 LOG .warn ("Failed to commit for {}.{}, abort it." , dbName , tbName );
234- hmsCommitter .cancelUnStartedAsyncFileSystemTask ();
235- hmsCommitter .undoUpdateStatisticsTasks ();
236- hmsCommitter .undoAddPartitionsTask ();
237- hmsCommitter .waitForAsyncFileSystemTaskSuppressThrowable ();
238- hmsCommitter .runDirectoryClearUpTasksForAbort ();
239- hmsCommitter .runRenameDirTasksForAbort ();
234+ hmsCommitter .rollback ();
240235 throw t ;
241236 } finally {
242237 hmsCommitter .runClearPathsForFinish ();
@@ -354,7 +349,7 @@ private static void addSuppressedExceptions(
354349 }
355350 }
356351
357- private static class UpdateStatisticsTask {
352+ public static class UpdateStatisticsTask {
358353 private final String dbName ;
359354 private final String tableName ;
360355 private final Optional <String > partitionName ;
@@ -442,7 +437,6 @@ public void run(HiveMetadataOps hiveOps) {
442437 throw t ;
443438 }
444439 }
445- partitions .clear ();
446440 }
447441
448442 public List <List <String >> rollback (HiveMetadataOps hiveOps ) {
@@ -548,7 +542,7 @@ private void recursiveDeleteItems(Path directory, boolean deleteEmptyDir) {
548542
549543 private DeleteRecursivelyResult recursiveDeleteFiles (Path directory , boolean deleteEmptyDir ) {
550544 try {
551- if (!fs .exists (directory .getName ()).ok ()) {
545+ if (!fs .exists (directory .toString ()).ok ()) {
552546 return new DeleteRecursivelyResult (true , ImmutableList .of ());
553547 }
554548 } catch (Exception e ) {
@@ -561,57 +555,53 @@ private DeleteRecursivelyResult recursiveDeleteFiles(Path directory, boolean del
561555 }
562556
563557 private DeleteRecursivelyResult doRecursiveDeleteFiles (Path directory , boolean deleteEmptyDir ) {
564- List <RemoteFile > remoteFiles = new ArrayList <>();
565-
566- Status status = fs .list (directory .getName (), remoteFiles );
567- if (!status .ok ()) {
558+ List <RemoteFile > allFiles = new ArrayList <>();
559+ Set <String > allDirs = new HashSet <>();
560+ Status statusFile = fs .listFiles (directory .toString (), allFiles );
561+ Status statusDir = fs .listDirectories (directory .toString (), allDirs );
562+ if (!statusFile .ok () || !statusDir .ok ()) {
568563 ImmutableList .Builder <String > notDeletedEligibleItems = ImmutableList .builder ();
569564 notDeletedEligibleItems .add (directory + "/*" );
570565 return new DeleteRecursivelyResult (false , notDeletedEligibleItems .build ());
571566 }
572567
573- boolean isEmptyDir = true ;
574- List <String > notDeletedEligibleItems = new ArrayList <>();
575- for (RemoteFile file : remoteFiles ) {
576- if (file .isFile ()) {
577- Path filePath = file .getPath ();
578- isEmptyDir = false ;
579- // TODO Check if this file was created by this query
580- if (!deleteIfExists (filePath )) {
581- notDeletedEligibleItems .add (filePath .toString ());
582- }
583- } else if (file .isDirectory ()) {
584- DeleteRecursivelyResult subResult = doRecursiveDeleteFiles (file .getPath (), deleteEmptyDir );
585- if (!subResult .dirNotExists ()) {
586- isEmptyDir = false ;
587- }
588- if (!subResult .getNotDeletedEligibleItems ().isEmpty ()) {
589- notDeletedEligibleItems .addAll (subResult .getNotDeletedEligibleItems ());
590- }
591- } else {
592- isEmptyDir = false ;
593- notDeletedEligibleItems .add (file .getPath ().toString ());
568+ boolean allDescendentsDeleted = true ;
569+ ImmutableList .Builder <String > notDeletedEligibleItems = ImmutableList .builder ();
570+ for (RemoteFile file : allFiles ) {
571+ String fileName = file .getName ();
572+ if (!deleteIfExists (new Path (fileName ))) {
573+ allDescendentsDeleted = false ;
574+ notDeletedEligibleItems .add (fileName );
594575 }
595576 }
596577
597- if (isEmptyDir && deleteEmptyDir ) {
598- Verify .verify (notDeletedEligibleItems .isEmpty ());
578+ for (String dir : allDirs ) {
579+ DeleteRecursivelyResult subResult = doRecursiveDeleteFiles (new Path (dir ), deleteEmptyDir );
580+ if (!subResult .dirNotExists ()) {
581+ allDescendentsDeleted = false ;
582+ }
583+ if (!subResult .getNotDeletedEligibleItems ().isEmpty ()) {
584+ notDeletedEligibleItems .addAll (subResult .getNotDeletedEligibleItems ());
585+ }
586+ }
587+
588+ if (allDescendentsDeleted && deleteEmptyDir ) {
589+ Verify .verify (notDeletedEligibleItems .build ().isEmpty ());
599590 if (!deleteIfExists (directory )) {
600591 return new DeleteRecursivelyResult (false , ImmutableList .of (directory + "/" ));
601592 }
602593 // all items of the location have been deleted.
603594 return new DeleteRecursivelyResult (true , ImmutableList .of ());
604595 }
605-
606- return new DeleteRecursivelyResult (false , notDeletedEligibleItems );
596+ return new DeleteRecursivelyResult (false , notDeletedEligibleItems .build ());
607597 }
608598
609599 public boolean deleteIfExists (Path path ) {
610- Status status = fs .delete (path .getName ());
600+ Status status = fs .delete (path .toString ());
611601 if (status .ok ()) {
612602 return true ;
613603 }
614- return !fs .exists (path .getName ()).ok ();
604+ return !fs .exists (path .toString ()).ok ();
615605 }
616606
617607 public static class DatabaseTableName {
@@ -1039,9 +1029,6 @@ private void undoUpdateStatisticsTasks() {
10391029 }
10401030
10411031 private void undoAddPartitionsTask () {
1042- if (addPartitionsTask .isEmpty ()) {
1043- return ;
1044- }
10451032
10461033 HivePartition firstPartition = addPartitionsTask .getPartitions ().get (0 ).getPartition ();
10471034 String dbName = firstPartition .getDbName ();
@@ -1304,10 +1291,16 @@ private void doUpdateStatisticsTasks() {
13041291 }
13051292 }
13061293
1294+ public void doNothing () {
1295+ // do nothing
1296+ // only for regression test and unit test to throw exception
1297+ }
1298+
13071299 public void doCommit () {
13081300 waitForAsyncFileSystemTasks ();
13091301 doAddPartitionsTask ();
13101302 doUpdateStatisticsTasks ();
1303+ doNothing ();
13111304 }
13121305
13131306 public void rollback () {
0 commit comments