Skip to content

Commit 75b0f3a

Browse files
WyveraldWyverald
authored
Write a repo mapping manifest in the runfiles directory (bazelbuild#16555)
* Write a repo mapping manifest in the runfiles directory To ensure we can use repo mappings in the runfiles library, this change writes an extra file "my_binary_target.repo_mapping", which contains a bunch of (base_repo_canonical_name, apparent_repo_name, canonical_repo_name) triples. See https://github.com/bazelbuild/proposals/blob/main/designs/2022-07-21-locating-runfiles-with-bzlmod.md for more information. The extra file is written using a new action "RepoMappingManifestAction", and it's only executed if Bzlmod is enabled. This avoids generating a lot of extra actions that are essentially useless for monorepo setups such as Google's. Work towards bazelbuild#16124 PiperOrigin-RevId: 483735699 Change-Id: I885b4df093bd2c783c57d19f995f420b9b29b53c * Remove erroneous check about a repo mapping entry with an empty apparent name It turns out that not only the main repo can have such an entry; any repo generated by a module extension hosted in the main repo can (and will) also have such an entry, because they can see all repos that the main repo can see, which includes the main repo itself. PiperOrigin-RevId: 483938753 Change-Id: I6192496f2edf08c9b1723c5b041c0ac44f928b3e
1 parent b88779e commit 75b0f3a

File tree

9 files changed

+612
-23
lines changed

9 files changed

+612
-23
lines changed

‎src/main/java/com/google/devtools/build/lib/analysis/BUILD‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -336,6 +336,7 @@ java_library(
336336
":package_specification_provider",
337337
":platform_options",
338338
":provider_collection",
339+
":repo_mapping_manifest_action",
339340
":required_config_fragments_provider",
340341
":resolved_toolchain_context",
341342
":rule_configured_object_value",
@@ -982,6 +983,25 @@ java_library(
982983
],
983984
)
984985

986+
java_library(
987+
name = "repo_mapping_manifest_action",
988+
srcs = ["RepoMappingManifestAction.java"],
989+
deps = [
990+
":actions/abstract_file_write_action",
991+
":actions/deterministic_writer",
992+
"//src/main/java/com/google/devtools/build/lib/actions",
993+
"//src/main/java/com/google/devtools/build/lib/actions:artifacts",
994+
"//src/main/java/com/google/devtools/build/lib/actions:commandline_item",
995+
"//src/main/java/com/google/devtools/build/lib/cmdline",
996+
"//src/main/java/com/google/devtools/build/lib/collect/nestedset",
997+
"//src/main/java/com/google/devtools/build/lib/util",
998+
"//src/main/java/net/starlark/java/eval",
999+
"//third_party:auto_value",
1000+
"//third_party:guava",
1001+
"//third_party:jsr305",
1002+
],
1003+
)
1004+
9851005
java_library(
9861006
name = "required_config_fragments_provider",
9871007
srcs = ["RequiredConfigFragmentsProvider.java"],

‎src/main/java/com/google/devtools/build/lib/analysis/RepoMappingManifestAction.java‎

Lines changed: 122 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,122 @@
1+
// Copyright 2022 The Bazel Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
package com.google.devtools.build.lib.analysis;
15+
16+
import static java.nio.charset.StandardCharsets.ISO_8859_1;
17+
import static java.util.Comparator.comparing;
18+
19+
import com.google.auto.value.AutoValue;
20+
import com.google.common.collect.ImmutableList;
21+
import com.google.devtools.build.lib.actions.ActionExecutionContext;
22+
import com.google.devtools.build.lib.actions.ActionKeyContext;
23+
import com.google.devtools.build.lib.actions.ActionOwner;
24+
import com.google.devtools.build.lib.actions.Artifact;
25+
import com.google.devtools.build.lib.actions.Artifact.ArtifactExpander;
26+
import com.google.devtools.build.lib.actions.CommandLineExpansionException;
27+
import com.google.devtools.build.lib.actions.ExecException;
28+
import com.google.devtools.build.lib.analysis.actions.AbstractFileWriteAction;
29+
import com.google.devtools.build.lib.analysis.actions.DeterministicWriter;
30+
import com.google.devtools.build.lib.cmdline.RepositoryName;
31+
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
32+
import com.google.devtools.build.lib.collect.nestedset.Order;
33+
import com.google.devtools.build.lib.util.Fingerprint;
34+
import java.io.PrintWriter;
35+
import java.util.List;
36+
import java.util.UUID;
37+
import javax.annotation.Nullable;
38+
import net.starlark.java.eval.EvalException;
39+
40+
/** Creates a manifest file describing the repos and mappings relevant for a runfile tree. */
41+
public class RepoMappingManifestAction extends AbstractFileWriteAction {
42+
private static final UUID MY_UUID = UUID.fromString("458e351c-4d30-433d-b927-da6cddd4737f");
43+
44+
private final ImmutableList<Entry> entries;
45+
private final String workspaceName;
46+
47+
/** An entry in the repo mapping manifest file. */
48+
@AutoValue
49+
public abstract static class Entry {
50+
public static Entry of(
51+
RepositoryName sourceRepo, String targetRepoApparentName, RepositoryName targetRepo) {
52+
return new AutoValue_RepoMappingManifestAction_Entry(
53+
sourceRepo, targetRepoApparentName, targetRepo);
54+
}
55+
56+
public abstract RepositoryName sourceRepo();
57+
58+
public abstract String targetRepoApparentName();
59+
60+
public abstract RepositoryName targetRepo();
61+
}
62+
63+
public RepoMappingManifestAction(
64+
ActionOwner owner, Artifact output, List<Entry> entries, String workspaceName) {
65+
super(owner, NestedSetBuilder.emptySet(Order.STABLE_ORDER), output, /*makeExecutable=*/ false);
66+
this.entries =
67+
ImmutableList.sortedCopyOf(
68+
comparing((Entry e) -> e.sourceRepo().getName())
69+
.thenComparing(Entry::targetRepoApparentName),
70+
entries);
71+
this.workspaceName = workspaceName;
72+
}
73+
74+
@Override
75+
public String getMnemonic() {
76+
return "RepoMappingManifest";
77+
}
78+
79+
@Override
80+
protected String getRawProgressMessage() {
81+
return "writing repo mapping manifest for " + getOwner().getLabel();
82+
}
83+
84+
@Override
85+
protected void computeKey(
86+
ActionKeyContext actionKeyContext,
87+
@Nullable ArtifactExpander artifactExpander,
88+
Fingerprint fp)
89+
throws CommandLineExpansionException, EvalException, InterruptedException {
90+
fp.addUUID(MY_UUID);
91+
fp.addString(workspaceName);
92+
for (Entry entry : entries) {
93+
fp.addString(entry.sourceRepo().getName());
94+
fp.addString(entry.targetRepoApparentName());
95+
fp.addString(entry.targetRepo().getName());
96+
}
97+
}
98+
99+
@Override
100+
public DeterministicWriter newDeterministicWriter(ActionExecutionContext ctx)
101+
throws InterruptedException, ExecException {
102+
return out -> {
103+
PrintWriter writer = new PrintWriter(out, /*autoFlush=*/ false, ISO_8859_1);
104+
for (Entry entry : entries) {
105+
if (entry.targetRepoApparentName().isEmpty()) {
106+
// The apparent repo name can only be empty for the main repo. We skip this line as
107+
// Rlocation paths can't reference an empty apparent name anyway.
108+
continue;
109+
}
110+
// The canonical name of the main repo is the empty string, which is not a valid name for a
111+
// directory, so the "workspace name" is used the name of the directory under the runfiles
112+
// tree for it.
113+
String targetRepoDirectoryName =
114+
entry.targetRepo().isMain() ? workspaceName : entry.targetRepo().getName();
115+
writer.format(
116+
"%s,%s,%s\n",
117+
entry.sourceRepo().getName(), entry.targetRepoApparentName(), targetRepoDirectoryName);
118+
}
119+
writer.flush();
120+
};
121+
}
122+
}

‎src/main/java/com/google/devtools/build/lib/analysis/RunfilesSupport.java‎

Lines changed: 101 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,26 +14,34 @@
1414

1515
package com.google.devtools.build.lib.analysis;
1616

17+
import static com.google.common.collect.ImmutableSet.toImmutableSet;
18+
1719
import com.google.common.annotations.VisibleForTesting;
1820
import com.google.common.base.Preconditions;
21+
import com.google.common.collect.ImmutableList;
1922
import com.google.common.collect.ImmutableSet;
2023
import com.google.devtools.build.lib.actions.ActionEnvironment;
2124
import com.google.devtools.build.lib.actions.Artifact;
2225
import com.google.devtools.build.lib.actions.CommandLine;
26+
import com.google.devtools.build.lib.analysis.RepoMappingManifestAction.Entry;
2327
import com.google.devtools.build.lib.analysis.SourceManifestAction.ManifestType;
2428
import com.google.devtools.build.lib.analysis.actions.ActionConstructionContext;
2529
import com.google.devtools.build.lib.analysis.actions.SymlinkTreeAction;
2630
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
2731
import com.google.devtools.build.lib.analysis.config.RunUnder;
32+
import com.google.devtools.build.lib.cmdline.RepositoryName;
2833
import com.google.devtools.build.lib.collect.nestedset.NestedSet;
2934
import com.google.devtools.build.lib.collect.nestedset.NestedSetBuilder;
3035
import com.google.devtools.build.lib.concurrent.ThreadSafety.Immutable;
36+
import com.google.devtools.build.lib.packages.Package;
3137
import com.google.devtools.build.lib.packages.TargetUtils;
3238
import com.google.devtools.build.lib.packages.Type;
39+
import com.google.devtools.build.lib.packages.semantics.BuildLanguageOptions;
3340
import com.google.devtools.build.lib.vfs.FileSystemUtils;
3441
import com.google.devtools.build.lib.vfs.Path;
3542
import com.google.devtools.build.lib.vfs.PathFragment;
3643
import java.util.Collection;
44+
import java.util.HashSet;
3745
import java.util.LinkedHashSet;
3846
import java.util.List;
3947
import java.util.Map;
@@ -76,11 +84,13 @@ public final class RunfilesSupport {
7684
private static final String RUNFILES_DIR_EXT = ".runfiles";
7785
private static final String INPUT_MANIFEST_EXT = ".runfiles_manifest";
7886
private static final String OUTPUT_MANIFEST_BASENAME = "MANIFEST";
87+
private static final String REPO_MAPPING_MANIFEST_EXT = ".repo_mapping";
7988

8089
private final Runfiles runfiles;
8190

8291
private final Artifact runfilesInputManifest;
8392
private final Artifact runfilesManifest;
93+
private final Artifact repoMappingManifest;
8494
private final Artifact runfilesMiddleman;
8595
private final Artifact owningExecutable;
8696
private final boolean buildRunfileLinks;
@@ -132,15 +142,19 @@ private static RunfilesSupport create(
132142
runfilesInputManifest = null;
133143
runfilesManifest = null;
134144
}
145+
Artifact repoMappingManifest =
146+
createRepoMappingManifestAction(ruleContext, runfiles, owningExecutable);
135147
Artifact runfilesMiddleman =
136-
createRunfilesMiddleman(ruleContext, owningExecutable, runfiles, runfilesManifest);
148+
createRunfilesMiddleman(
149+
ruleContext, owningExecutable, runfiles, runfilesManifest, repoMappingManifest);
137150

138151
boolean runfilesEnabled = ruleContext.getConfiguration().runfilesEnabled();
139152

140153
return new RunfilesSupport(
141154
runfiles,
142155
runfilesInputManifest,
143156
runfilesManifest,
157+
repoMappingManifest,
144158
runfilesMiddleman,
145159
owningExecutable,
146160
buildRunfileLinks,
@@ -153,6 +167,7 @@ private RunfilesSupport(
153167
Runfiles runfiles,
154168
Artifact runfilesInputManifest,
155169
Artifact runfilesManifest,
170+
Artifact repoMappingManifest,
156171
Artifact runfilesMiddleman,
157172
Artifact owningExecutable,
158173
boolean buildRunfileLinks,
@@ -162,6 +177,7 @@ private RunfilesSupport(
162177
this.runfiles = runfiles;
163178
this.runfilesInputManifest = runfilesInputManifest;
164179
this.runfilesManifest = runfilesManifest;
180+
this.repoMappingManifest = repoMappingManifest;
165181
this.runfilesMiddleman = runfilesMiddleman;
166182
this.owningExecutable = owningExecutable;
167183
this.buildRunfileLinks = buildRunfileLinks;
@@ -268,6 +284,17 @@ public Artifact getRunfilesManifest() {
268284
return runfilesManifest;
269285
}
270286

287+
/**
288+
* Returns the foo.repo_mapping file if Bazel is run with transitive package tracking turned on
289+
* (see {@code SkyframeExecutor#getForcedSingleSourceRootIfNoExecrootSymlinkCreation}) and any of
290+
* the transitive packages come from a repository with strict deps (see {@code
291+
* #collectRepoMappings}). Otherwise, returns null.
292+
*/
293+
@Nullable
294+
public Artifact getRepoMappingManifest() {
295+
return repoMappingManifest;
296+
}
297+
271298
/** Returns the root directory of the runfiles symlink farm; otherwise, returns null. */
272299
@Nullable
273300
public Path getRunfilesDirectory() {
@@ -327,12 +354,16 @@ private static Artifact createRunfilesMiddleman(
327354
ActionConstructionContext context,
328355
Artifact owningExecutable,
329356
Runfiles runfiles,
330-
@Nullable Artifact runfilesManifest) {
357+
@Nullable Artifact runfilesManifest,
358+
Artifact repoMappingManifest) {
331359
NestedSetBuilder<Artifact> deps = NestedSetBuilder.stableOrder();
332360
deps.addTransitive(runfiles.getAllArtifacts());
333361
if (runfilesManifest != null) {
334362
deps.add(runfilesManifest);
335363
}
364+
if (repoMappingManifest != null) {
365+
deps.add(repoMappingManifest);
366+
}
336367
return context
337368
.getAnalysisEnvironment()
338369
.getMiddlemanFactory()
@@ -495,4 +526,72 @@ public static Path inputManifestPath(Path runfilesDir) {
495526
public static Path outputManifestPath(Path runfilesDir) {
496527
return runfilesDir.getRelative(OUTPUT_MANIFEST_BASENAME);
497528
}
529+
530+
@Nullable
531+
private static Artifact createRepoMappingManifestAction(
532+
RuleContext ruleContext, Runfiles runfiles, Artifact owningExecutable) {
533+
if (!ruleContext
534+
.getAnalysisEnvironment()
535+
.getStarlarkSemantics()
536+
.getBool(BuildLanguageOptions.ENABLE_BZLMOD)) {
537+
// If Bzlmod is not enabled, we don't need a repo mapping manifest.
538+
return null;
539+
}
540+
541+
PathFragment executablePath =
542+
(owningExecutable != null)
543+
? owningExecutable.getOutputDirRelativePath(
544+
ruleContext.getConfiguration().isSiblingRepositoryLayout())
545+
: ruleContext.getPackageDirectory().getRelative(ruleContext.getLabel().getName());
546+
Artifact repoMappingManifest =
547+
ruleContext.getDerivedArtifact(
548+
executablePath.replaceName(executablePath.getBaseName() + REPO_MAPPING_MANIFEST_EXT),
549+
ruleContext.getBinDirectory());
550+
ruleContext
551+
.getAnalysisEnvironment()
552+
.registerAction(
553+
new RepoMappingManifestAction(
554+
ruleContext.getActionOwner(),
555+
repoMappingManifest,
556+
collectRepoMappings(
557+
Preconditions.checkNotNull(
558+
ruleContext.getTransitivePackagesForRunfileRepoMappingManifest()),
559+
runfiles),
560+
ruleContext.getWorkspaceName()));
561+
return repoMappingManifest;
562+
}
563+
564+
/** Returns the list of entries (unsorted) that should appear in the repo mapping manifest. */
565+
private static ImmutableList<Entry> collectRepoMappings(
566+
NestedSet<Package> transitivePackages, Runfiles runfiles) {
567+
// NOTE: It might appear that the flattening of `transitivePackages` is better suited to the
568+
// execution phase rather than here in the analysis phase, but we can't do that since it would
569+
// necessitate storing `transitivePackages` in an action, which breaks skyframe serialization
570+
// since packages cannot be serialized here.
571+
572+
ImmutableSet<RepositoryName> reposContributingRunfiles =
573+
runfiles.getAllArtifacts().toList().stream()
574+
.filter(a -> a.getOwner() != null)
575+
.map(a -> a.getOwner().getRepository())
576+
.collect(toImmutableSet());
577+
Set<RepositoryName> seenRepos = new HashSet<>();
578+
ImmutableList.Builder<Entry> entries = ImmutableList.builder();
579+
for (Package pkg : transitivePackages.toList()) {
580+
if (!seenRepos.add(pkg.getPackageIdentifier().getRepository())) {
581+
// Any package from the same repo would have the same repo mapping.
582+
continue;
583+
}
584+
for (Map.Entry<String, RepositoryName> repoMappingEntry :
585+
pkg.getRepositoryMapping().entries().entrySet()) {
586+
if (reposContributingRunfiles.contains(repoMappingEntry.getValue())) {
587+
entries.add(
588+
Entry.of(
589+
pkg.getPackageIdentifier().getRepository(),
590+
repoMappingEntry.getKey(),
591+
repoMappingEntry.getValue()));
592+
}
593+
}
594+
}
595+
return entries.build();
596+
}
498597
}

‎src/main/java/com/google/devtools/build/lib/cmdline/RepositoryMapping.java‎

Lines changed: 6 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -37,7 +37,8 @@ public abstract class RepositoryMapping {
3737
// Always fallback to the requested name
3838
public static final RepositoryMapping ALWAYS_FALLBACK = createAllowingFallback(ImmutableMap.of());
3939

40-
abstract ImmutableMap<String, RepositoryName> repositoryMapping();
40+
/** Returns all the entries in this repo mapping. */
41+
public abstract ImmutableMap<String, RepositoryName> entries();
4142

4243
/**
4344
* The owner repo of this repository mapping. It is for providing useful debug information when
@@ -66,7 +67,7 @@ public static RepositoryMapping createAllowingFallback(
6667
*/
6768
public RepositoryMapping withAdditionalMappings(Map<String, RepositoryName> additionalMappings) {
6869
HashMap<String, RepositoryName> allMappings = new HashMap<>(additionalMappings);
69-
allMappings.putAll(repositoryMapping());
70+
allMappings.putAll(entries());
7071
return new AutoValue_RepositoryMapping(ImmutableMap.copyOf(allMappings), ownerRepo());
7172
}
7273

@@ -76,15 +77,15 @@ public RepositoryMapping withAdditionalMappings(Map<String, RepositoryName> addi
7677
* repo of the given additional mappings is ignored.
7778
*/
7879
public RepositoryMapping withAdditionalMappings(RepositoryMapping additionalMappings) {
79-
return withAdditionalMappings(additionalMappings.repositoryMapping());
80+
return withAdditionalMappings(additionalMappings.entries());
8081
}
8182

8283
/**
8384
* Returns the canonical repository name associated with the given apparent repo name. The
8485
* provided apparent repo name is assumed to be valid.
8586
*/
8687
public RepositoryName get(String preMappingName) {
87-
RepositoryName canonicalRepoName = repositoryMapping().get(preMappingName);
88+
RepositoryName canonicalRepoName = entries().get(preMappingName);
8889
if (canonicalRepoName != null) {
8990
return canonicalRepoName;
9091
}
@@ -108,7 +109,7 @@ public boolean usesStrictDeps() {
108109
* Returns the first apparent name in this mapping that maps to the given canonical name, if any.
109110
*/
110111
public Optional<String> getInverse(RepositoryName postMappingName) {
111-
return repositoryMapping().entrySet().stream()
112+
return entries().entrySet().stream()
112113
.filter(e -> e.getValue().equals(postMappingName))
113114
.map(Entry::getKey)
114115
.findFirst();

0 commit comments

Comments
 (0)