Fix sandbox cleanup crashing after server restart

oquenchil · copybara-github · commit 488b63072687 · 2024-03-19T06:23:04.000-07:00
We try to clean up the sandbox base from previous server instances asynchronously, however sometimes this is not possible due to the old directories being in a different filesystem. This can happen with overlays on Docker after running bazel in different RUN commands. See #21719 This change fixes the crash by catching the IOException and falling back to synchronous deletion. Fixes #21719. RELNOTES:none PiperOrigin-RevId: 617150522 Change-Id: I82a07ac0ade66cfb1e5732a90a5f3ab4e2e8caa7
diff --git a/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java b/src/main/java/com/google/devtools/build/lib/sandbox/SandboxModule.java
@@ -227,27 +227,36 @@ private void setup(CommandEnvironment cmdEnv, SpawnStrategyRegistry.Builder buil
     // previous builds. However, on the very first build of an instance of the server, we must
     // wipe old contents to avoid reusing stale directories.
     if (firstBuild && sandboxBase.exists()) {
-      if (trashBase.exists()) {
-        // Delete stale trash from a previous server instance.
-        Path staleTrash = getStaleTrashDir(trashBase);
-        trashBase.renameTo(staleTrash);
-        trashBase.createDirectory();
-        treeDeleter.deleteTree(staleTrash);
-      } else {
-        trashBase.createDirectory();
-      }
-      // We can delete other dirs asynchronously (if the flag is on).
-      for (Path entry : sandboxBase.getDirectoryEntries()) {
-        if (entry.getBaseName().equals(AsynchronousTreeDeleter.MOVED_TRASH_DIR)) {
-          continue;
-        }
-        if (entry.getBaseName().equals(SandboxHelpers.INACCESSIBLE_HELPER_DIR)) {
-          entry.deleteTree();
-        } else if (entry.isDirectory()) {
-          treeDeleter.deleteTree(entry);
+      try {
+        if (trashBase.exists()) {
+          // Delete stale trash from a previous server instance.
+          Path staleTrash = getStaleTrashDir(trashBase);
+          trashBase.renameTo(staleTrash);
+          trashBase.createDirectory();
+          treeDeleter.deleteTree(staleTrash);
         } else {
-          entry.delete();
+          trashBase.createDirectory();
         }
+        // We can delete other dirs asynchronously (if the flag is on).
+        for (Path entry : sandboxBase.getDirectoryEntries()) {
+          if (entry.getBaseName().equals(AsynchronousTreeDeleter.MOVED_TRASH_DIR)) {
+            continue;
+          }
+          if (entry.getBaseName().equals(SandboxHelpers.INACCESSIBLE_HELPER_DIR)) {
+            entry.deleteTree();
+          } else if (entry.isDirectory()) {
+            treeDeleter.deleteTree(entry);
+          } else {
+            entry.delete();
+          }
+        }
+      } catch (IOException e) {
+        // We have observed asynchronous deletion failing when running Bazel under Docker, see
+        // #21719. Different RUN commands with `bazel build` will write to different layers in the
+        // docker image. The overlay filesystem is different and the renaming of the directories
+        // that we need to do for asynchronous deletion will fail. When that happens we fall back to
+        // synchronous deletion here.
+        sandboxBase.deleteTree();
       }
     }
     firstBuild = false;
