Publish the new execution log format to the build event protocol.

tjgq · copybara-github · commit 9eb3b0a340c6 · 2024-02-19T12:40:39.000-08:00
For now, we don't publish the old formats, because in our experience their size tends to get out of control, and that doesn't play well with the automatic uploading of files referenced by the BEP.

PiperOrigin-RevId: 608385023
Change-Id: I772cedbf844648ae708abc3ae7e2e5579060a1e4
diff --git a/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java b/src/main/java/com/google/devtools/build/lib/bazel/SpawnLogModule.java
@@ -15,8 +15,10 @@
 
 import static com.google.common.base.Preconditions.checkNotNull;
 
+import com.google.common.eventbus.Subscribe;
 import com.google.common.primitives.Booleans;
 import com.google.devtools.build.lib.buildtool.BuildRequest;
+import com.google.devtools.build.lib.buildtool.buildevent.BuildCompleteEvent;
 import com.google.devtools.build.lib.events.Event;
 import com.google.devtools.build.lib.exec.CompactSpawnLogContext;
 import com.google.devtools.build.lib.exec.ExecutionOptions;
@@ -40,10 +42,16 @@
 
 /** Module providing on-demand spawn logging. */
 public final class SpawnLogModule extends BlazeModule {
+
   @Nullable private SpawnLogContext spawnLogContext;
+  @Nullable private Path outputPath;
+
+  @Nullable private AbruptExitException abruptExit = null;
 
   private void clear() {
     spawnLogContext = null;
+    outputPath = null;
+    abruptExit = null;
   }
 
   private void initOutputs(CommandEnvironment env) throws IOException {
@@ -88,20 +96,21 @@ private void initOutputs(CommandEnvironment env) throws IOException {
     Path outputBase = env.getOutputBase();
 
     if (executionOptions.executionLogCompactFile != null) {
+      outputPath = workingDirectory.getRelative(executionOptions.executionLogCompactFile);
+
       try {
-      spawnLogContext =
-          new CompactSpawnLogContext(
-              workingDirectory.getRelative(executionOptions.executionLogCompactFile),
-              env.getExecRoot().asFragment(),
-              env.getOptions().getOptions(RemoteOptions.class),
-              env.getRuntime().getFileSystem().getDigestFunction(),
-              env.getXattrProvider());
+        spawnLogContext =
+            new CompactSpawnLogContext(
+                outputPath,
+                env.getExecRoot().asFragment(),
+                env.getOptions().getOptions(RemoteOptions.class),
+                env.getRuntime().getFileSystem().getDigestFunction(),
+                env.getXattrProvider());
       } catch (InterruptedException e) {
         env.getReporter()
             .handle(Event.error("Error while setting up the execution log: " + e.getMessage()));
       }
     } else {
-      Path outputPath = null;
       Encoding encoding = null;
 
       if (executionOptions.executionLogBinaryFile != null) {
@@ -159,19 +168,27 @@ public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorB
     }
   }
 
-  @Override
-  public void afterCommand() throws AbruptExitException {
+  @Subscribe
+  public void buildComplete(BuildCompleteEvent event) {
+    // The log must be finalized in buildComplete() instead of afterCommand(), because it's our
+    // last chance to publish it to the build event protocol.
+
     if (spawnLogContext == null) {
       // No logging requested.
+      clear();
       return;
     }
 
     try {
       spawnLogContext.close();
+      event.getResult().getBuildToolLogCollection().addLocalFile("execution.log", outputPath);
     } catch (IOException e) {
-      String message = e.getMessage() == null ? "Error writing execution log" : e.getMessage();
-      throw new AbruptExitException(
-          createDetailedExitCode(message, Code.EXECUTION_LOG_WRITE_FAILURE), e);
+      abruptExit =
+          new AbruptExitException(
+              createDetailedExitCode(
+                  String.format("Error writing execution log: %s", e.getMessage()),
+                  Code.EXECUTION_LOG_WRITE_FAILURE),
+              e);
     } finally {
       clear();
     }
@@ -184,4 +201,11 @@ private static DetailedExitCode createDetailedExitCode(String message, Code deta
             .setExecution(Execution.newBuilder().setCode(detailedCode))
             .build());
   }
+
+  @Override
+  public void afterCommand() throws AbruptExitException {
+    if (abruptExit != null) {
+      throw abruptExit;
+    }
+  }
 }
diff --git a/src/main/java/com/google/devtools/build/lib/exec/CompactSpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/CompactSpawnLogContext.java
@@ -186,6 +186,12 @@ private void logInvocation() throws IOException, InterruptedException {
                         .setHashFunctionName(digestHashFunction.toString())));
   }
 
+  @Override
+  public boolean shouldPublish() {
+    // The compact log is small enough to be uploaded to a remote store.
+    return true;
+  }
+
   @Override
   public void logSpawn(
       Spawn spawn,
diff --git a/src/main/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/ExpandedSpawnLogContext.java
@@ -134,6 +134,12 @@ private MessageOutputStream<SpawnExec> getConvertedOutputStream(Path path) throw
         String.format("invalid execution log encoding: %s", encoding));
   }
 
+  @Override
+  public boolean shouldPublish() {
+    // The expanded log tends to be too large to be uploaded to a remote store.
+    return false;
+  }
+
   @Override
   public void logSpawn(
       Spawn spawn,
diff --git a/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java b/src/main/java/com/google/devtools/build/lib/exec/SpawnLogContext.java
@@ -77,6 +77,9 @@ public abstract void logSpawn(
   /** Finishes writing the log and performs any required post-processing. */
   public abstract void close() throws IOException;
 
+  /** Whether the log should be published to the build event protocol. */
+  public abstract boolean shouldPublish();
+
   /** Computes the environment variables. */
   protected ImmutableList<EnvironmentVariable> getEnvironmentVariables(Spawn spawn) {
     ImmutableMap<String, String> environment = spawn.getEnvironment();
