Skip to content

Commit b56a2aa

Browse files
coeuvrecoeuvre
committed
Remote: Use execRoot as input root and do NOT set working directory by default.
When --experimental_sibling_external_layout is set, use the parent directory of execRoot as input root and set working directory to the base name of execRoot. Paths of output files are relative to working directory. When --incompatible_remote_paths_relative_to_input_root is set, paths of output files are relative to input root. Introduce RemotePathResolver which is used to convert local paths to remote paths and vice versa. We should prefer this class instead of using execRoot directly whenever possible in remote module. execRoot usages in RemoteCache are all replaced. On Windows, shared action results cache for cl.exe across different workspaces causing header dependency checking to fail. This was initially fixed by #9172, but is broken after 24c980b. The reason test didn't fail before this change is that two builds from different workspaces do not share the cache since input files are relative to the parent of exec root which contains workspace name. This change fixes that by adding workspace name as platform property for action running on Windows. Fixes #13188. Closes #13339. PiperOrigin-RevId: 369168230
1 parent c901827 commit b56a2aa

23 files changed

Lines changed: 845 additions & 245 deletions

‎src/main/java/com/google/devtools/build/lib/actions/ExecutionRequirements.java‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,4 +246,8 @@ public enum WorkerProtocolFormat {
246246
* followed by a {@code SIGKILL} after a grace period).
247247
*/
248248
public static final String GRACEFUL_TERMINATION = "supports-graceful-termination";
249+
250+
/** Requires the execution service do NOT share caches across different workspace. */
251+
public static final String DIFFERENTIATE_WORKSPACE_CACHE =
252+
"internal-differentiate-workspace-cache";
249253
}

‎src/main/java/com/google/devtools/build/lib/analysis/platform/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -43,6 +43,7 @@ java_library(
4343
srcs = ["PlatformUtils.java"],
4444
deps = [
4545
"//src/main/java/com/google/devtools/build/lib/actions",
46+
"//src/main/java/com/google/devtools/build/lib/actions:execution_requirements",
4647
"//src/main/java/com/google/devtools/build/lib/remote/options",
4748
"//src/main/protobuf:failure_details_java_proto",
4849
"//third_party:guava",

‎src/main/java/com/google/devtools/build/lib/analysis/platform/PlatformUtils.java‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@
1919
import com.google.common.base.Strings;
2020
import com.google.common.collect.ImmutableSortedMap;
2121
import com.google.common.collect.Ordering;
22+
import com.google.devtools.build.lib.actions.ExecutionRequirements;
2223
import com.google.devtools.build.lib.actions.Spawn;
2324
import com.google.devtools.build.lib.actions.UserExecException;
2425
import com.google.devtools.build.lib.remote.options.RemoteOptions;
@@ -101,6 +102,16 @@ public static Platform getPlatformProto(Spawn spawn, @Nullable RemoteOptions rem
101102
}
102103
}
103104

105+
String workspace =
106+
spawn.getExecutionInfo().get(ExecutionRequirements.DIFFERENTIATE_WORKSPACE_CACHE);
107+
if (workspace != null) {
108+
platformBuilder.addProperties(
109+
Property.newBuilder()
110+
.setName("bazel-differentiate-workspace-cache")
111+
.setValue(workspace)
112+
.build());
113+
}
114+
104115
sortPlatformProperties(platformBuilder);
105116
return platformBuilder.build();
106117
}

‎src/main/java/com/google/devtools/build/lib/remote/BUILD‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -70,6 +70,7 @@ java_library(
7070
"//src/main/java/com/google/devtools/build/lib/exec:spawn_runner",
7171
"//src/main/java/com/google/devtools/build/lib/exec:spawn_strategy_registry",
7272
"//src/main/java/com/google/devtools/build/lib/packages",
73+
"//src/main/java/com/google/devtools/build/lib/packages/semantics",
7374
"//src/main/java/com/google/devtools/build/lib/profiler",
7475
"//src/main/java/com/google/devtools/build/lib/remote/common",
7576
"//src/main/java/com/google/devtools/build/lib/remote/disk",

‎src/main/java/com/google/devtools/build/lib/remote/RemoteActionContextProvider.java‎

Lines changed: 24 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -27,7 +27,11 @@
2727
import com.google.devtools.build.lib.exec.ModuleActionContextRegistry;
2828
import com.google.devtools.build.lib.exec.SpawnCache;
2929
import com.google.devtools.build.lib.exec.SpawnStrategyRegistry;
30+
import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
3031
import com.google.devtools.build.lib.remote.common.RemoteExecutionClient;
32+
import com.google.devtools.build.lib.remote.common.RemotePathResolver;
33+
import com.google.devtools.build.lib.remote.common.RemotePathResolver.DefaultRemotePathResolver;
34+
import com.google.devtools.build.lib.remote.common.RemotePathResolver.SiblingRepositoryLayoutResolver;
3135
import com.google.devtools.build.lib.remote.options.RemoteOptions;
3236
import com.google.devtools.build.lib.remote.util.DigestUtil;
3337
import com.google.devtools.build.lib.runtime.CommandEnvironment;
@@ -80,6 +84,22 @@ public static RemoteActionContextProvider createForRemoteExecution(
8084
env, cache, executor, retryScheduler, digestUtil, logDir);
8185
}
8286

87+
RemotePathResolver createRemotePathResolver() {
88+
Path execRoot = env.getExecRoot();
89+
BuildLanguageOptions buildLanguageOptions =
90+
env.getOptions().getOptions(BuildLanguageOptions.class);
91+
RemotePathResolver remotePathResolver;
92+
if (buildLanguageOptions != null && buildLanguageOptions.experimentalSiblingRepositoryLayout) {
93+
RemoteOptions remoteOptions = checkNotNull(env.getOptions().getOptions(RemoteOptions.class));
94+
remotePathResolver =
95+
new SiblingRepositoryLayoutResolver(
96+
execRoot, remoteOptions.incompatibleRemoteOutputPathsRelativeToInputRoot);
97+
} else {
98+
remotePathResolver = new DefaultRemotePathResolver(execRoot);
99+
}
100+
return remotePathResolver;
101+
}
102+
83103
/**
84104
* Registers a remote spawn strategy if this instance was created with an executor, otherwise does
85105
* nothing.
@@ -108,7 +128,8 @@ public void registerRemoteSpawnStrategyIfApplicable(
108128
retryScheduler,
109129
digestUtil,
110130
logDir,
111-
filesToDownload);
131+
filesToDownload,
132+
createRemotePathResolver());
112133
registryBuilder.registerStrategy(
113134
new RemoteSpawnStrategy(env.getExecRoot(), spawnRunner, verboseFailures), "remote");
114135
}
@@ -129,7 +150,8 @@ public void registerSpawnCache(ModuleActionContextRegistry.Builder registryBuild
129150
env.getCommandId().toString(),
130151
env.getReporter(),
131152
digestUtil,
132-
filesToDownload);
153+
filesToDownload,
154+
createRemotePathResolver());
133155
registryBuilder.register(SpawnCache.class, spawnCache, "remote-cache");
134156
}
135157

0 commit comments

Comments
 (0)