Skip to content

Commit 4ca8946

Browse files
coeuvrecopybara-github
coeuvre
authored and
copybara-github
committed
Remote: Add --experimental_capture_corrupted_outputs flag.
Which when set, Bazel will save outputs whose digest does not match the expected value to the target directories. Also use OutputDigestMismatchException to indicate the error and include output path in the error message. The message for such an error will become e.g.: ``` com.google.devtools.build.lib.remote.common.OutputDigestMismatchException: Output bazel-out/darwin-fastbuild/bin/output.txt download failed: Expected digest '872af2fe77729717832d0a020ae87a93b8b944146a2af6b3490491e1eaf1dc74/29229' does not match received digest '872af2fe77729717832d0a020ae87a93b8b944146a2af6b3490491e1eaf1dc74/29229'. ``` Used to support bazelbuild/continuous-integration#1175. Closes bazelbuild#13568. PiperOrigin-RevId: 378624823
1 parent a48937f commit 4ca8946

6 files changed

Lines changed: 154 additions & 10 deletions

File tree

‎src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -73,22 +73,37 @@ public static RemoteActionContextProvider createForPlaceholder(
7373
env, /*cache=*/ null, /*executor=*/ null, retryScheduler, digestUtil, /*logDir=*/ null);
7474
}
7575

76+
private static void maybeSetCaptureCorruptedOutputsDir(
77+
RemoteOptions remoteOptions, RemoteCache remoteCache, Path workingDirectory) {
78+
if (remoteOptions.remoteCaptureCorruptedOutputs != null
79+
&& !remoteOptions.remoteCaptureCorruptedOutputs.isEmpty()) {
80+
remoteCache.setCaptureCorruptedOutputsDir(
81+
workingDirectory.getRelative(remoteOptions.remoteCaptureCorruptedOutputs));
82+
}
83+
}
84+
7685
public static RemoteActionContextProvider createForRemoteCaching(
7786
CommandEnvironment env,
87+
RemoteOptions options,
7888
RemoteCache cache,
7989
ListeningScheduledExecutorService retryScheduler,
8090
DigestUtil digestUtil) {
91+
maybeSetCaptureCorruptedOutputsDir(options, cache, env.getWorkingDirectory());
92+
8193
return new RemoteActionContextProvider(
8294
env, cache, /*executor=*/ null, retryScheduler, digestUtil, /*logDir=*/ null);
8395
}
8496

8597
public static RemoteActionContextProvider createForRemoteExecution(
8698
CommandEnvironment env,
99+
RemoteOptions options,
87100
RemoteExecutionCache cache,
88101
RemoteExecutionClient executor,
89102
ListeningScheduledExecutorService retryScheduler,
90103
DigestUtil digestUtil,
91104
Path logDir) {
105+
maybeSetCaptureCorruptedOutputsDir(options, cache, env.getWorkingDirectory());
106+
92107
return new RemoteActionContextProvider(
93108
env, cache, executor, retryScheduler, digestUtil, logDir);
94109
}

‎src/main/java/com/google/devtools/build/lib/remote/RemoteCache.java‎

Lines changed: 65 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,7 @@
3939
import com.google.common.util.concurrent.FutureCallback;
4040
import com.google.common.util.concurrent.Futures;
4141
import com.google.common.util.concurrent.ListenableFuture;
42+
import com.google.common.util.concurrent.MoreExecutors;
4243
import com.google.common.util.concurrent.SettableFuture;
4344
import com.google.devtools.build.lib.actions.ActionInput;
4445
import com.google.devtools.build.lib.actions.Artifact;
@@ -55,6 +56,7 @@
5556
import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.DirectoryMetadata;
5657
import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.FileMetadata;
5758
import com.google.devtools.build.lib.remote.RemoteCache.ActionResultMetadata.SymlinkMetadata;
59+
import com.google.devtools.build.lib.remote.common.OutputDigestMismatchException;
5860
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
5961
import com.google.devtools.build.lib.remote.common.RemoteActionFileArtifactValue;
6062
import com.google.devtools.build.lib.remote.common.RemoteCacheClient;
@@ -110,13 +112,19 @@ interface OutputFilesLocker {
110112
protected final RemoteOptions options;
111113
protected final DigestUtil digestUtil;
112114

115+
private Path captureCorruptedOutputsDir;
116+
113117
public RemoteCache(
114118
RemoteCacheClient cacheProtocol, RemoteOptions options, DigestUtil digestUtil) {
115119
this.cacheProtocol = cacheProtocol;
116120
this.options = options;
117121
this.digestUtil = digestUtil;
118122
}
119123

124+
public void setCaptureCorruptedOutputsDir(Path captureCorruptedOutputsDir) {
125+
this.captureCorruptedOutputsDir = captureCorruptedOutputsDir;
126+
}
127+
120128
public ActionResult downloadActionResult(
121129
RemoteActionExecutionContext context, ActionKey actionKey, boolean inlineOutErr)
122130
throws IOException, InterruptedException {
@@ -334,7 +342,11 @@ public void download(
334342
(file) -> {
335343
try {
336344
ListenableFuture<Void> download =
337-
downloadFile(context, toTmpDownloadPath(file.path()), file.digest());
345+
downloadFile(
346+
context,
347+
remotePathResolver.localPathToOutputPath(file.path()),
348+
toTmpDownloadPath(file.path()),
349+
file.digest());
338350
return Futures.transform(download, (d) -> file, directExecutor());
339351
} catch (IOException e) {
340352
return Futures.<FileMetadata>immediateFailedFuture(e);
@@ -355,6 +367,30 @@ public void download(
355367
try {
356368
waitForBulkTransfer(downloads, /* cancelRemainingOnInterrupt=*/ true);
357369
} catch (Exception e) {
370+
if (captureCorruptedOutputsDir != null) {
371+
if (e instanceof BulkTransferException) {
372+
for (Throwable suppressed : e.getSuppressed()) {
373+
if (suppressed instanceof OutputDigestMismatchException) {
374+
// Capture corrupted outputs
375+
try {
376+
String outputPath = ((OutputDigestMismatchException) suppressed).getOutputPath();
377+
Path localPath = ((OutputDigestMismatchException) suppressed).getLocalPath();
378+
Path dst = captureCorruptedOutputsDir.getRelative(outputPath);
379+
dst.createDirectoryAndParents();
380+
381+
// Make sure dst is still under captureCorruptedOutputsDir, otherwise
382+
// IllegalArgumentException will be thrown.
383+
dst.relativeTo(captureCorruptedOutputsDir);
384+
385+
FileSystemUtils.copyFile(localPath, dst);
386+
} catch (Exception ee) {
387+
ee.addSuppressed(ee);
388+
}
389+
}
390+
}
391+
}
392+
}
393+
358394
try {
359395
// Delete any (partially) downloaded output files.
360396
for (OutputFile file : result.getOutputFilesList()) {
@@ -461,6 +497,34 @@ private void createSymlinks(Iterable<SymlinkMetadata> symlinks) throws IOExcepti
461497
}
462498
}
463499

500+
public ListenableFuture<Void> downloadFile(
501+
RemoteActionExecutionContext context, String outputPath, Path localPath, Digest digest)
502+
throws IOException {
503+
SettableFuture<Void> outerF = SettableFuture.create();
504+
ListenableFuture<Void> f = downloadFile(context, localPath, digest);
505+
Futures.addCallback(
506+
f,
507+
new FutureCallback<Void>() {
508+
@Override
509+
public void onSuccess(Void unused) {
510+
outerF.set(null);
511+
}
512+
513+
@Override
514+
public void onFailure(Throwable throwable) {
515+
if (throwable instanceof OutputDigestMismatchException) {
516+
OutputDigestMismatchException e = ((OutputDigestMismatchException) throwable);
517+
e.setOutputPath(outputPath);
518+
e.setLocalPath(localPath);
519+
}
520+
outerF.setException(throwable);
521+
}
522+
},
523+
MoreExecutors.directExecutor());
524+
525+
return outerF;
526+
}
527+
464528
/** Downloads a file (that is not a directory). The content is fetched from the digest. */
465529
public ListenableFuture<Void> downloadFile(
466530
RemoteActionExecutionContext context, Path path, Digest digest) throws IOException {

‎src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java‎

Lines changed: 3 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -223,7 +223,7 @@ private void initHttpAndDiskCache(
223223
RemoteCache remoteCache = new RemoteCache(cacheClient, remoteOptions, digestUtil);
224224
actionContextProvider =
225225
RemoteActionContextProvider.createForRemoteCaching(
226-
env, remoteCache, /* retryScheduler= */ null, digestUtil);
226+
env, remoteOptions, remoteCache, /* retryScheduler= */ null, digestUtil);
227227
}
228228

229229
@Override
@@ -530,7 +530,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
530530
new RemoteExecutionCache(cacheClient, remoteOptions, digestUtil);
531531
actionContextProvider =
532532
RemoteActionContextProvider.createForRemoteExecution(
533-
env, remoteCache, remoteExecutor, retryScheduler, digestUtil, logDir);
533+
env, remoteOptions, remoteCache, remoteExecutor, retryScheduler, digestUtil, logDir);
534534
repositoryRemoteExecutorFactoryDelegate.init(
535535
new RemoteRepositoryRemoteExecutorFactory(
536536
remoteCache,
@@ -560,7 +560,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
560560
RemoteCache remoteCache = new RemoteCache(cacheClient, remoteOptions, digestUtil);
561561
actionContextProvider =
562562
RemoteActionContextProvider.createForRemoteCaching(
563-
env, remoteCache, retryScheduler, digestUtil);
563+
env, remoteOptions, remoteCache, retryScheduler, digestUtil);
564564
}
565565

566566
if (enableRemoteDownloader) {

‎src/main/java/com/google/devtools/build/lib/remote/common/OutputDigestMismatchException.java‎

Lines changed: 60 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,60 @@
1+
// Copyright 2021 The Bazel Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
package com.google.devtools.build.lib.remote.common;
15+
16+
import build.bazel.remote.execution.v2.Digest;
17+
import com.google.devtools.build.lib.vfs.Path;
18+
import java.io.IOException;
19+
20+
/** An exception to indicate the digest of downloaded output does not match the expected value. */
21+
public class OutputDigestMismatchException extends IOException {
22+
private final Digest expected;
23+
private final Digest actual;
24+
25+
private Path localPath;
26+
private String outputPath;
27+
28+
public OutputDigestMismatchException(Digest expected, Digest actual) {
29+
this.expected = expected;
30+
this.actual = actual;
31+
}
32+
33+
public void setOutputPath(String outputPath) {
34+
this.outputPath = outputPath;
35+
}
36+
37+
public String getOutputPath() {
38+
return outputPath;
39+
}
40+
41+
public Path getLocalPath() {
42+
return localPath;
43+
}
44+
45+
public void setLocalPath(Path localPath) {
46+
this.localPath = localPath;
47+
}
48+
49+
@Override
50+
public String getMessage() {
51+
return String.format(
52+
"Output %s download failed: Expected digest '%s/%d' does not match "
53+
+ "received digest '%s/%d'.",
54+
outputPath,
55+
expected.getHash(),
56+
expected.getSizeBytes(),
57+
actual.getHash(),
58+
actual.getSizeBytes());
59+
}
60+
}

‎src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java‎

Lines changed: 9 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,15 @@ public final class RemoteOptions extends OptionsBase {
8888
help = "Whether to use keepalive for remote execution calls.")
8989
public boolean remoteExecutionKeepalive;
9090

91+
@Option(
92+
name = "experimental_remote_capture_corrupted_outputs",
93+
defaultValue = "null",
94+
documentationCategory = OptionDocumentationCategory.REMOTE,
95+
effectTags = {OptionEffectTag.UNKNOWN},
96+
converter = OptionsUtils.PathFragmentConverter.class,
97+
help = "A path to a directory where the corrupted outputs will be captured to.")
98+
public PathFragment remoteCaptureCorruptedOutputs;
99+
91100
@Option(
92101
name = "remote_cache",
93102
oldName = "remote_http_cache",

‎src/main/java/com/google/devtools/build/lib/remote/util/Utils.java‎

Lines changed: 2 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -36,6 +36,7 @@
3636
import com.google.devtools.build.lib.authandtls.CallCredentialsProvider;
3737
import com.google.devtools.build.lib.remote.ExecutionStatusException;
3838
import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
39+
import com.google.devtools.build.lib.remote.common.OutputDigestMismatchException;
3940
import com.google.devtools.build.lib.remote.common.RemoteCacheClient.ActionKey;
4041
import com.google.devtools.build.lib.remote.options.RemoteOutputsMode;
4142
import com.google.devtools.build.lib.server.FailureDetails;
@@ -390,12 +391,7 @@ public static ListenableFuture<ActionResult> downloadAsActionResult(
390391

391392
public static void verifyBlobContents(Digest expected, Digest actual) throws IOException {
392393
if (!expected.equals(actual)) {
393-
String msg =
394-
String.format(
395-
"Output download failed: Expected digest '%s/%d' does not match "
396-
+ "received digest '%s/%d'.",
397-
expected.getHash(), expected.getSizeBytes(), actual.getHash(), actual.getSizeBytes());
398-
throw new IOException(msg);
394+
throw new OutputDigestMismatchException(expected, actual);
399395
}
400396
}
401397

0 commit comments

Comments
 (0)