Skip to content

Commit a0fa77c

Browse files
coeuvreShreeM01keertk
authored
Exit with code 39 if remote cache evicted blobs that Bazel need during an invocation (bazelbuild#17496)
Part of bazelbuild#16660. Closes bazelbuild#17358. PiperOrigin-RevId: 509494072 Change-Id: Id6944da5d9a556dc9154fcb702948586b474875e Co-authored-by: kshyanashree <[email protected]> Co-authored-by: keertk <[email protected]>
1 parent 5932b3b commit a0fa77c

File tree

11 files changed

+135
-30
lines changed

11 files changed

+135
-30
lines changed

‎site/en/run/scripts.md‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -55,6 +55,7 @@ Bazel execution can result in following exit codes:
5555
- `36` - Local Environmental Issue, suspected permanent.
5656
- `37` - Unhandled Exception / Internal Bazel Error.
5757
- `38` - Reserved for Google-internal use.
58+
- `39` - Blobs required by Bazel are evicted from Remote Cache.
5859
- `41-44` - Reserved for Google-internal use.
5960
- `45` - Error publishing results to the Build Event Service.
6061
- `47` - Reserved for Google-internal use.

‎src/main/java/com/google/devtools/build/lib/exec/SpawnRunner.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -162,7 +162,7 @@ interface SpawnExecutionContext {
162162
* @see #prefetchInputs()
163163
*/
164164
default void prefetchInputsAndWait()
165-
throws IOException, InterruptedException, ForbiddenActionInputException {
165+
throws IOException, ExecException, InterruptedException, ForbiddenActionInputException {
166166
ListenableFuture<Void> future = prefetchInputs();
167167
try (SilentCloseable s =
168168
Profiler.instance().profile(ProfilerTask.REMOTE_DOWNLOAD, "stage remote inputs")) {
@@ -171,6 +171,7 @@ default void prefetchInputsAndWait()
171171
Throwable cause = e.getCause();
172172
if (cause != null) {
173173
throwIfInstanceOf(cause, IOException.class);
174+
throwIfInstanceOf(cause, ExecException.class);
174175
throwIfInstanceOf(cause, ForbiddenActionInputException.class);
175176
throwIfInstanceOf(cause, RuntimeException.class);
176177
}

‎src/main/java/com/google/devtools/build/lib/remote/RemoteActionInputFetcher.java‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -20,17 +20,20 @@
2020
import com.google.common.base.Preconditions;
2121
import com.google.common.collect.ImmutableList;
2222
import com.google.common.util.concurrent.ListenableFuture;
23+
import com.google.devtools.build.lib.actions.EnvironmentalExecException;
2324
import com.google.devtools.build.lib.actions.FileArtifactValue;
2425
import com.google.devtools.build.lib.actions.cache.VirtualActionInput;
2526
import com.google.devtools.build.lib.actions.cache.VirtualActionInput.EmptyActionInput;
2627
import com.google.devtools.build.lib.events.Reporter;
2728
import com.google.devtools.build.lib.remote.common.BulkTransferException;
28-
import com.google.devtools.build.lib.remote.common.CacheNotFoundException;
2929
import com.google.devtools.build.lib.remote.common.RemoteActionExecutionContext;
3030
import com.google.devtools.build.lib.remote.util.DigestUtil;
3131
import com.google.devtools.build.lib.remote.util.TempPathGenerator;
3232
import com.google.devtools.build.lib.remote.util.TracingMetadataUtils;
3333
import com.google.devtools.build.lib.sandbox.SandboxHelpers;
34+
import com.google.devtools.build.lib.server.FailureDetails;
35+
import com.google.devtools.build.lib.server.FailureDetails.FailureDetail;
36+
import com.google.devtools.build.lib.server.FailureDetails.Spawn.Code;
3437
import com.google.devtools.build.lib.vfs.Path;
3538
import com.google.devtools.build.lib.vfs.PathFragment;
3639
import io.reactivex.rxjava3.core.Completable;
@@ -48,6 +51,7 @@ class RemoteActionInputFetcher extends AbstractActionInputPrefetcher {
4851
private final String buildRequestId;
4952
private final String commandId;
5053
private final RemoteCache remoteCache;
54+
private final boolean useNewExitCodeForLostInputs;
5155

5256
RemoteActionInputFetcher(
5357
Reporter reporter,
@@ -56,11 +60,13 @@ class RemoteActionInputFetcher extends AbstractActionInputPrefetcher {
5660
RemoteCache remoteCache,
5761
Path execRoot,
5862
TempPathGenerator tempPathGenerator,
59-
ImmutableList<Pattern> patternsToDownload) {
63+
ImmutableList<Pattern> patternsToDownload,
64+
boolean useNewExitCodeForLostInputs) {
6065
super(reporter, execRoot, tempPathGenerator, patternsToDownload);
6166
this.buildRequestId = Preconditions.checkNotNull(buildRequestId);
6267
this.commandId = Preconditions.checkNotNull(commandId);
6368
this.remoteCache = Preconditions.checkNotNull(remoteCache);
69+
this.useNewExitCodeForLostInputs = useNewExitCodeForLostInputs;
6470
}
6571

6672
@Override
@@ -99,19 +105,18 @@ protected ListenableFuture<Void> doDownloadFile(
99105
protected Completable onErrorResumeNext(Throwable error) {
100106
if (error instanceof BulkTransferException) {
101107
if (((BulkTransferException) error).onlyCausedByCacheNotFoundException()) {
102-
BulkTransferException bulkAnnotatedException = new BulkTransferException();
103-
for (Throwable t : error.getSuppressed()) {
104-
IOException annotatedException =
105-
new IOException(
106-
String.format(
107-
"Failed to fetch file with hash '%s' because it does not"
108-
+ " exist remotely. --remote_download_outputs=minimal"
109-
+ " does not work if your remote cache evicts files"
110-
+ " during builds.",
111-
((CacheNotFoundException) t).getMissingDigest().getHash()));
112-
bulkAnnotatedException.add(annotatedException);
113-
}
114-
error = bulkAnnotatedException;
108+
var code =
109+
useNewExitCodeForLostInputs ? Code.REMOTE_CACHE_EVICTED : Code.REMOTE_CACHE_FAILED;
110+
error =
111+
new EnvironmentalExecException(
112+
(BulkTransferException) error,
113+
FailureDetail.newBuilder()
114+
.setMessage(
115+
"Failed to fetch blobs because they do not exist remotely."
116+
+ " Build without the Bytes does not work if your remote"
117+
+ " cache evicts blobs during builds")
118+
.setSpawn(FailureDetails.Spawn.newBuilder().setCode(code))
119+
.build());
115120
}
116121
}
117122
return Completable.error(error);

‎src/main/java/com/google/devtools/build/lib/remote/RemoteModule.java‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -917,7 +917,8 @@ public void executorInit(CommandEnvironment env, BuildRequest request, ExecutorB
917917
actionContextProvider.getRemoteCache(),
918918
env.getExecRoot(),
919919
tempPathGenerator,
920-
patternsToDownload);
920+
patternsToDownload,
921+
remoteOptions.useNewExitCodeForLostInputs);
921922
env.getEventBus().register(actionInputFetcher);
922923
builder.setActionInputPrefetcher(actionInputFetcher);
923924
remoteOutputService.setActionInputFetcher(actionInputFetcher);

‎src/main/java/com/google/devtools/build/lib/remote/options/RemoteOptions.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -658,6 +658,17 @@ public RemoteOutputsStrategyConverter() {
658658
+ "cache misses and retries.")
659659
public boolean remoteDiscardMerkleTrees;
660660

661+
@Option(
662+
name = "incompatible_remote_use_new_exit_code_for_lost_inputs",
663+
defaultValue = "false",
664+
documentationCategory = OptionDocumentationCategory.REMOTE,
665+
effectTags = {OptionEffectTag.UNKNOWN},
666+
metadataTags = {OptionMetadataTag.INCOMPATIBLE_CHANGE},
667+
help =
668+
"If set to true, Bazel will use new exit code 39 instead of 34 if remote cache evicts"
669+
+ " blobs during the build.")
670+
public boolean useNewExitCodeForLostInputs;
671+
661672
// The below options are not configurable by users, only tests.
662673
// This is part of the effort to reduce the overall number of flags.
663674

‎src/main/java/com/google/devtools/build/lib/util/ExitCode.java‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -65,6 +65,8 @@ public final class ExitCode {
6565
ExitCode.createInfrastructureFailure(37, "BLAZE_INTERNAL_ERROR");
6666
public static final ExitCode TRANSIENT_BUILD_EVENT_SERVICE_UPLOAD_ERROR =
6767
ExitCode.createInfrastructureFailure(38, "PUBLISH_ERROR");
68+
public static final ExitCode REMOTE_CACHE_EVICTED =
69+
ExitCode.createInfrastructureFailure(39, "REMOTE_CACHE_EVICTED");
6870
public static final ExitCode PERSISTENT_BUILD_EVENT_SERVICE_UPLOAD_ERROR =
6971
ExitCode.create(45, "PERSISTENT_BUILD_EVENT_SERVICE_UPLOAD_ERROR");
7072
public static final ExitCode EXTERNAL_DEPS_ERROR = ExitCode.create(48, "EXTERNAL_DEPS_ERROR");

‎src/main/protobuf/failure_details.proto‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -226,6 +226,7 @@ message Spawn {
226226
// refactored to prohibit undetailed failures
227227
UNSPECIFIED_EXECUTION_FAILURE = 12 [(metadata) = { exit_code: 1 }];
228228
FORBIDDEN_INPUT = 13 [(metadata) = { exit_code: 1 }];
229+
REMOTE_CACHE_EVICTED = 14 [(metadata) = { exit_code: 39 }];
229230
}
230231
Code code = 1;
231232

‎src/test/java/com/google/devtools/build/lib/remote/ActionInputPrefetcherTestBase.java‎

Lines changed: 11 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -37,12 +37,12 @@
3737
import com.google.devtools.build.lib.actions.Artifact.TreeFileArtifact;
3838
import com.google.devtools.build.lib.actions.ArtifactRoot;
3939
import com.google.devtools.build.lib.actions.ArtifactRoot.RootType;
40+
import com.google.devtools.build.lib.actions.ExecException;
4041
import com.google.devtools.build.lib.actions.FileArtifactValue;
4142
import com.google.devtools.build.lib.actions.FileArtifactValue.RemoteFileArtifactValue;
4243
import com.google.devtools.build.lib.actions.MetadataProvider;
4344
import com.google.devtools.build.lib.actions.util.ActionsTestUtil;
4445
import com.google.devtools.build.lib.clock.JavaClock;
45-
import com.google.devtools.build.lib.remote.common.BulkTransferException;
4646
import com.google.devtools.build.lib.remote.util.StaticMetadataProvider;
4747
import com.google.devtools.build.lib.remote.util.TempPathGenerator;
4848
import com.google.devtools.build.lib.skyframe.TreeArtifactValue;
@@ -160,7 +160,8 @@ protected Pair<SpecialArtifact, ImmutableList<TreeFileArtifact>> createRemoteTre
160160
protected abstract AbstractActionInputPrefetcher createPrefetcher(Map<HashCode, byte[]> cas);
161161

162162
@Test
163-
public void prefetchFiles_fileExists_doNotDownload() throws IOException, InterruptedException {
163+
public void prefetchFiles_fileExists_doNotDownload()
164+
throws IOException, ExecException, InterruptedException {
164165
Map<ActionInput, FileArtifactValue> metadata = new HashMap<>();
165166
Map<HashCode, byte[]> cas = new HashMap<>();
166167
Artifact a = createRemoteArtifact("file", "hello world", metadata, cas);
@@ -177,7 +178,7 @@ public void prefetchFiles_fileExists_doNotDownload() throws IOException, Interru
177178

178179
@Test
179180
public void prefetchFiles_fileExistsButContentMismatches_download()
180-
throws IOException, InterruptedException {
181+
throws IOException, ExecException, InterruptedException {
181182
Map<ActionInput, FileArtifactValue> metadata = new HashMap<>();
182183
Map<HashCode, byte[]> cas = new HashMap<>();
183184
Artifact a = createRemoteArtifact("file", "hello world remote", metadata, cas);
@@ -304,7 +305,7 @@ public void prefetchFiles_missingFiles_fails() throws Exception {
304305
AbstractActionInputPrefetcher prefetcher = createPrefetcher(new HashMap<>());
305306

306307
assertThrows(
307-
BulkTransferException.class,
308+
Exception.class,
308309
() -> wait(prefetcher.prefetchFiles(ImmutableList.of(a), metadataProvider)));
309310

310311
assertThat(prefetcher.downloadedFiles()).isEmpty();
@@ -347,7 +348,7 @@ public void prefetchFiles_multipleThreads_downloadIsCancelled() throws Exception
347348
() -> {
348349
try {
349350
wait(prefetcher.prefetchFiles(ImmutableList.of(artifact), metadataProvider));
350-
} catch (IOException | InterruptedException ignored) {
351+
} catch (IOException | ExecException | InterruptedException ignored) {
351352
// do nothing
352353
}
353354
});
@@ -357,7 +358,7 @@ public void prefetchFiles_multipleThreads_downloadIsCancelled() throws Exception
357358
() -> {
358359
try {
359360
wait(prefetcher.prefetchFiles(ImmutableList.of(artifact), metadataProvider));
360-
} catch (IOException | InterruptedException ignored) {
361+
} catch (IOException | ExecException | InterruptedException ignored) {
361362
// do nothing
362363
}
363364
});
@@ -394,7 +395,7 @@ public void prefetchFiles_multipleThreads_downloadIsNotCancelledByOtherThreads()
394395
() -> {
395396
try {
396397
wait(prefetcher.prefetchFiles(ImmutableList.of(artifact), metadataProvider));
397-
} catch (IOException | InterruptedException ignored) {
398+
} catch (IOException | ExecException | InterruptedException ignored) {
398399
// do nothing
399400
}
400401
});
@@ -406,7 +407,7 @@ public void prefetchFiles_multipleThreads_downloadIsNotCancelledByOtherThreads()
406407
try {
407408
wait(prefetcher.prefetchFiles(ImmutableList.of(artifact), metadataProvider));
408409
successful.set(true);
409-
} catch (IOException | InterruptedException ignored) {
410+
} catch (IOException | ExecException | InterruptedException ignored) {
410411
// do nothing
411412
}
412413
});
@@ -489,13 +490,14 @@ public void downloadFile_onInterrupt_deletePartialDownloadedFile() throws Except
489490
}
490491

491492
protected static void wait(ListenableFuture<Void> future)
492-
throws IOException, InterruptedException {
493+
throws IOException, ExecException, InterruptedException {
493494
try {
494495
future.get();
495496
} catch (ExecutionException e) {
496497
Throwable cause = e.getCause();
497498
if (cause != null) {
498499
throwIfInstanceOf(cause, IOException.class);
500+
throwIfInstanceOf(cause, ExecException.class);
499501
throwIfInstanceOf(cause, InterruptedException.class);
500502
throwIfInstanceOf(cause, RuntimeException.class);
501503
}

‎src/test/java/com/google/devtools/build/lib/remote/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -162,6 +162,7 @@ java_test(
162162
"//src/main/java/com/google/devtools/build/lib/remote",
163163
"//src/main/java/com/google/devtools/build/lib/standalone",
164164
"//src/main/java/com/google/devtools/build/lib/util:os",
165+
"//src/main/java/com/google/devtools/build/lib/vfs",
165166
"//src/test/java/com/google/devtools/build/lib/remote/util:integration_test_utils",
166167
"//third_party:guava",
167168
"//third_party:junit4",

‎src/test/java/com/google/devtools/build/lib/remote/BuildWithoutTheBytesIntegrationTest.java‎

Lines changed: 52 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -30,6 +30,7 @@
3030
import com.google.devtools.build.lib.runtime.BuildSummaryStatsModule;
3131
import com.google.devtools.build.lib.standalone.StandaloneModule;
3232
import com.google.devtools.build.lib.util.OS;
33+
import com.google.devtools.build.lib.vfs.FileSystemUtils;
3334
import java.io.IOException;
3435
import org.junit.After;
3536
import org.junit.Test;
@@ -69,6 +70,7 @@ protected void setDownloadAll() {
6970
@Override
7071
protected BlazeRuntime.Builder getRuntimeBuilder() throws Exception {
7172
return super.getRuntimeBuilder()
73+
.addBlazeModule(new RemoteModule())
7274
.addBlazeModule(new BuildSummaryStatsModule())
7375
.addBlazeModule(new BlockWaitingModule());
7476
}
@@ -79,7 +81,6 @@ protected ImmutableList<BlazeModule> getSpawnModules() {
7981
.addAll(super.getSpawnModules())
8082
.add(new StandaloneModule())
8183
.add(new CredentialModule())
82-
.add(new RemoteModule())
8384
.add(new DynamicExecutionModule())
8485
.build();
8586
}
@@ -415,4 +416,54 @@ public void symlinkToNestedDirectory() throws Exception {
415416

416417
buildTarget("//a:one_local", "//a:two_local", "//a:one_remote", "//a:two_remote");
417418
}
419+
420+
@Test
421+
public void remoteCacheEvictBlobs_exitWithCode39() throws Exception {
422+
// Arrange: Prepare workspace and populate remote cache
423+
write(
424+
"a/BUILD",
425+
"genrule(",
426+
" name = 'foo',",
427+
" srcs = ['foo.in'],",
428+
" outs = ['foo.out'],",
429+
" cmd = 'cat $(SRCS) > $@',",
430+
")",
431+
"genrule(",
432+
" name = 'bar',",
433+
" srcs = ['foo.out', 'bar.in'],",
434+
" outs = ['bar.out'],",
435+
" cmd = 'cat $(SRCS) > $@',",
436+
" tags = ['no-remote-exec'],",
437+
")");
438+
write("a/foo.in", "foo");
439+
write("a/bar.in", "bar");
440+
441+
// Populate remote cache
442+
buildTarget("//a:bar");
443+
var bytes = FileSystemUtils.readContent(getOutputPath("a/foo.out"));
444+
var hashCode = getDigestHashFunction().getHashFunction().hashBytes(bytes);
445+
getOutputPath("a/foo.out").delete();
446+
getOutputPath("a/bar.out").delete();
447+
getOutputBase().getRelative("action_cache").deleteTreesBelow();
448+
restartServer();
449+
addOptions("--incompatible_remote_use_new_exit_code_for_lost_inputs");
450+
451+
// Clean build, foo.out isn't downloaded
452+
buildTarget("//a:bar");
453+
assertOutputDoesNotExist("a/foo.out");
454+
455+
// Act: Evict blobs from remote cache and do an incremental build
456+
getFileSystem().getPath(worker.getCasPath().getSafePathString()).deleteTreesBelow();
457+
write("a/bar.in", "updated bar");
458+
var error = assertThrows(BuildFailedException.class, () -> buildTarget("//a:bar"));
459+
460+
// Assert: Exit code is 39
461+
assertThat(error)
462+
.hasMessageThat()
463+
.contains(
464+
"Build without the Bytes does not work if your remote cache evicts blobs"
465+
+ " during builds");
466+
assertThat(error).hasMessageThat().contains(String.format("%s/%s", hashCode, bytes.length));
467+
assertThat(error.getDetailedExitCode().getExitCode().getNumericExitCode()).isEqualTo(39);
468+
}
418469
}

0 commit comments

Comments
 (0)