Skip to content

Commit c5a562b

Browse files
fmeumcopybara-github
fmeum
authored and
copybara-github
committed
Overlay the registry MODULE.bazel file on the module repo
RELNOTES: Modules backed by `http_archive` or `git_repository` no longer require a MODULE.bazel file to be contained in the source archive. Fixes #26217 Closes #26332. PiperOrigin-RevId: 778139801 Change-Id: Ie1d219ce1c219c8bae7657be929e54b9b467abf9
1 parent 933b941 commit c5a562b

File tree

16 files changed

+282
-119
lines changed

16 files changed

+282
-119
lines changed

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java‎

Lines changed: 7 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -67,18 +67,6 @@ public ArchiveRepoSpecBuilder setPatches(ImmutableList<Label> patches) {
6767
return this;
6868
}
6969

70-
@CanIgnoreReturnValue
71-
public ArchiveRepoSpecBuilder setPatchCmds(ImmutableList<String> patchCmds) {
72-
attrBuilder.put("patch_cmds", patchCmds);
73-
return this;
74-
}
75-
76-
@CanIgnoreReturnValue
77-
public ArchiveRepoSpecBuilder setPatchStrip(int patchStrip) {
78-
attrBuilder.put("patch_args", ImmutableList.of("-p" + patchStrip));
79-
return this;
80-
}
81-
8270
@CanIgnoreReturnValue
8371
public ArchiveRepoSpecBuilder setRemotePatches(ImmutableMap<String, String> remotePatches) {
8472
attrBuilder.put("remote_patches", remotePatches);
@@ -98,6 +86,13 @@ public ArchiveRepoSpecBuilder setOverlay(ImmutableMap<String, RemoteFile> overla
9886
return this;
9987
}
10088

89+
@CanIgnoreReturnValue
90+
public ArchiveRepoSpecBuilder setRemoteModuleFile(RemoteFile remoteModuleFile) {
91+
attrBuilder.put("remote_module_file_urls", remoteModuleFile.urls());
92+
attrBuilder.put("remote_module_file_integrity", remoteModuleFile.integrity());
93+
return this;
94+
}
95+
10196
@CanIgnoreReturnValue
10297
public ArchiveRepoSpecBuilder setRemotePatchStrip(int remotePatchStrip) {
10398
attrBuilder.put("remote_patch_strip", StarlarkInt.of(remotePatchStrip));

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/GitRepoSpecBuilder.java‎

Lines changed: 5 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -75,18 +75,11 @@ public GitRepoSpecBuilder setStripPrefix(String stripPrefix) {
7575
}
7676

7777
@CanIgnoreReturnValue
78-
public GitRepoSpecBuilder setPatches(List<Label> patches) {
79-
return setAttr("patches", patches);
80-
}
81-
82-
@CanIgnoreReturnValue
83-
public GitRepoSpecBuilder setPatchArgs(List<String> patchArgs) {
84-
return setAttr("patch_args", patchArgs);
85-
}
86-
87-
@CanIgnoreReturnValue
88-
public GitRepoSpecBuilder setPatchCmds(List<String> patchCmds) {
89-
return setAttr("patch_cmds", patchCmds);
78+
public GitRepoSpecBuilder setRemoteModuleFile(
79+
ArchiveRepoSpecBuilder.RemoteFile remoteModuleFile) {
80+
attrBuilder.put("remote_module_file_urls", remoteModuleFile.urls());
81+
attrBuilder.put("remote_module_file_integrity", remoteModuleFile.integrity());
82+
return this;
9083
}
9184

9285
public RepoSpec build() {

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/IndexRegistry.java‎

Lines changed: 37 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
import com.google.common.base.Strings;
2323
import com.google.common.collect.ImmutableList;
2424
import com.google.common.collect.ImmutableMap;
25+
import com.google.devtools.build.lib.bazel.bzlmod.ArchiveRepoSpecBuilder.RemoteFile;
2526
import com.google.devtools.build.lib.bazel.bzlmod.Version.ParseException;
2627
import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
2728
import com.google.devtools.build.lib.bazel.repository.downloader.Checksum.MissingChecksumException;
@@ -145,9 +146,10 @@ private byte[] grabFile(
145146
maybeContent = Optional.of(doGrabFile(downloadManager, url, eventHandler, useChecksum));
146147
return maybeContent.get();
147148
} finally {
148-
if ((knownFileHashesMode == KnownFileHashesMode.USE_AND_UPDATE
149-
|| knownFileHashesMode == KnownFileHashesMode.USE_IMMUTABLE_AND_UPDATE)
150-
&& useChecksum) {
149+
// We intentionally don't check knownFileHashesMode here: The checksums of module files are
150+
// always needed for the remote_module_file_integrity attributes of the http_archive backing
151+
// the module repo.
152+
if (useChecksum) {
151153
eventHandler.post(RegistryFileDownloadEvent.create(url, maybeContent));
152154
}
153155
}
@@ -240,12 +242,15 @@ private byte[] doGrabFile(
240242
public ModuleFile getModuleFile(
241243
ModuleKey key, ExtendedEventHandler eventHandler, DownloadManager downloadManager)
242244
throws IOException, InterruptedException, NotFoundException {
243-
String url =
244-
constructUrl(getUrl(), "modules", key.name(), key.version().toString(), "MODULE.bazel");
245+
String url = constructModuleFileUrl(key);
245246
byte[] content = grabFile(url, eventHandler, downloadManager, /* useChecksum= */ true);
246247
return ModuleFile.create(content, url);
247248
}
248249

250+
private String constructModuleFileUrl(ModuleKey key) {
251+
return constructUrl(getUrl(), "modules", key.name(), key.version().toString(), "MODULE.bazel");
252+
}
253+
249254
/** Represents fields available in {@code bazel_registry.json} for the registry. */
250255
private static class BazelRegistryJson {
251256
String[] mirrors;
@@ -333,7 +338,10 @@ private <T> T parseJson(String jsonString, String url, Class<T> klass) throws IO
333338

334339
@Override
335340
public RepoSpec getRepoSpec(
336-
ModuleKey key, ExtendedEventHandler eventHandler, DownloadManager downloadManager)
341+
ModuleKey key,
342+
ImmutableMap<String, Optional<Checksum>> moduleFileHashes,
343+
ExtendedEventHandler eventHandler,
344+
DownloadManager downloadManager)
337345
throws IOException, InterruptedException {
338346
String jsonUrl = getSourceJsonUrl(key);
339347
Optional<String> jsonString =
@@ -348,8 +356,14 @@ public RepoSpec getRepoSpec(
348356
case "archive" -> {
349357
ArchiveSourceJson typedSourceJson =
350358
parseJson(jsonString.get(), jsonUrl, ArchiveSourceJson.class);
359+
var moduleFileUrl = constructModuleFileUrl(key);
360+
var moduleFileChecksum = moduleFileHashes.get(moduleFileUrl).get();
351361
return createArchiveRepoSpec(
352-
typedSourceJson, getBazelRegistryJson(eventHandler, downloadManager), key);
362+
typedSourceJson,
363+
moduleFileUrl,
364+
moduleFileChecksum,
365+
getBazelRegistryJson(eventHandler, downloadManager),
366+
key);
353367
}
354368
case "local_path" -> {
355369
LocalPathSourceJson typedSourceJson =
@@ -360,7 +374,9 @@ public RepoSpec getRepoSpec(
360374
case "git_repository" -> {
361375
GitRepoSourceJson typedSourceJson =
362376
parseJson(jsonString.get(), jsonUrl, GitRepoSourceJson.class);
363-
return createGitRepoSpec(typedSourceJson);
377+
var moduleFileUrl = constructModuleFileUrl(key);
378+
var moduleFileChecksum = moduleFileHashes.get(moduleFileUrl).get();
379+
return createGitRepoSpec(typedSourceJson, moduleFileUrl, moduleFileChecksum);
364380
}
365381
default ->
366382
throw new IOException(
@@ -424,7 +440,11 @@ private RepoSpec createLocalPathRepoSpec(
424440
}
425441

426442
private RepoSpec createArchiveRepoSpec(
427-
ArchiveSourceJson sourceJson, Optional<BazelRegistryJson> bazelRegistryJson, ModuleKey key)
443+
ArchiveSourceJson sourceJson,
444+
String moduleFileUrl,
445+
Checksum moduleFileChecksum,
446+
Optional<BazelRegistryJson> bazelRegistryJson,
447+
ModuleKey key)
428448
throws IOException {
429449
URL sourceUrl = sourceJson.url;
430450
if (sourceUrl == null) {
@@ -498,12 +518,16 @@ private RepoSpec createArchiveRepoSpec(
498518
.setStripPrefix(Strings.nullToEmpty(sourceJson.stripPrefix))
499519
.setRemotePatches(remotePatches.buildOrThrow())
500520
.setOverlay(overlay)
521+
.setRemoteModuleFile(
522+
new RemoteFile(
523+
moduleFileChecksum.toSubresourceIntegrity(), ImmutableList.of(moduleFileUrl)))
501524
.setRemotePatchStrip(sourceJson.patchStrip)
502525
.setArchiveType(sourceJson.archiveType)
503526
.build();
504527
}
505528

506-
private RepoSpec createGitRepoSpec(GitRepoSourceJson sourceJson) {
529+
private RepoSpec createGitRepoSpec(
530+
GitRepoSourceJson sourceJson, String moduleFileUrl, Checksum moduleFileChecksum) {
507531
return new GitRepoSpecBuilder()
508532
.setRemote(sourceJson.remote)
509533
.setCommit(sourceJson.commit)
@@ -512,6 +536,9 @@ private RepoSpec createGitRepoSpec(GitRepoSourceJson sourceJson) {
512536
.setInitSubmodules(sourceJson.initSubmodules)
513537
.setVerbose(sourceJson.verbose)
514538
.setStripPrefix(sourceJson.stripPrefix)
539+
.setRemoteModuleFile(
540+
new RemoteFile(
541+
moduleFileChecksum.toSubresourceIntegrity(), ImmutableList.of(moduleFileUrl)))
515542
.build();
516543
}
517544

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/Registry.java‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
package com.google.devtools.build.lib.bazel.bzlmod;
1717

1818
import com.google.common.collect.ImmutableMap;
19+
import com.google.devtools.build.lib.bazel.repository.downloader.Checksum;
1920
import com.google.devtools.build.lib.bazel.repository.downloader.DownloadManager;
2021
import com.google.devtools.build.lib.events.ExtendedEventHandler;
2122
import com.google.devtools.build.skyframe.NotComparableSkyValue;
@@ -50,7 +51,10 @@ ModuleFile getModuleFile(
5051
* by {@code key} should be materialized as a repo.
5152
*/
5253
RepoSpec getRepoSpec(
53-
ModuleKey key, ExtendedEventHandler eventHandler, DownloadManager downloadManager)
54+
ModuleKey key,
55+
ImmutableMap<String, Optional<Checksum>> moduleFileHashes,
56+
ExtendedEventHandler eventHandler,
57+
DownloadManager downloadManager)
5458
throws IOException, InterruptedException;
5559

5660
/**

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RegistryFileDownloadEvent.java‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -34,8 +34,8 @@ public static RegistryFileDownloadEvent create(String uri, Optional<byte[]> cont
3434
static ImmutableMap<String, Optional<Checksum>> collectToMap(Collection<Postable> postables) {
3535
ImmutableMap.Builder<String, Optional<Checksum>> builder = ImmutableMap.builder();
3636
for (Postable postable : postables) {
37-
if (postable instanceof RegistryFileDownloadEvent event) {
38-
builder.put(event.uri(), event.checksum());
37+
if (postable instanceof RegistryFileDownloadEvent(String uri, Optional<Checksum> checksum)) {
38+
builder.put(uri, checksum);
3939
}
4040
}
4141
return builder.buildKeepingLast();

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/RepoSpecFunction.java‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -45,13 +45,23 @@ public SkyValue compute(SkyKey skyKey, Environment env)
4545
if (registry == null) {
4646
return null;
4747
}
48+
ModuleFileValue moduleFileValue =
49+
(ModuleFileValue) env.getValue(ModuleFileValue.key(key.moduleKey()));
50+
if (moduleFileValue == null) {
51+
return null;
52+
}
4853

4954
StoredEventHandler downloadEvents = new StoredEventHandler();
5055
RepoSpec repoSpec;
5156
try (SilentCloseable c =
5257
Profiler.instance()
5358
.profile(ProfilerTask.BZLMOD, () -> "compute repo spec: " + key.moduleKey())) {
54-
repoSpec = registry.getRepoSpec(key.moduleKey(), downloadEvents, this.downloadManager);
59+
repoSpec =
60+
registry.getRepoSpec(
61+
key.moduleKey(),
62+
moduleFileValue.registryFileHashes(),
63+
downloadEvents,
64+
this.downloadManager);
5565
} catch (IOException e) {
5666
throw new RepoSpecException(
5767
ExternalDepsException.withCauseAndMessage(

‎src/test/java/com/google/devtools/build/lib/bazel/bzlmod/FakeRegistry.java‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -80,7 +80,10 @@ public ModuleFile getModuleFile(
8080

8181
@Override
8282
public RepoSpec getRepoSpec(
83-
ModuleKey key, ExtendedEventHandler eventHandler, DownloadManager downloadManager) {
83+
ModuleKey key,
84+
ImmutableMap<String, Optional<Checksum>> moduleFileHashes,
85+
ExtendedEventHandler eventHandler,
86+
DownloadManager downloadManager) {
8487
RepoSpec repoSpec =
8588
LocalPathRepoSpecs.create(rootPath + "/" + key.getCanonicalRepoNameWithVersion().getName());
8689
eventHandler.post(

0 commit comments

Comments
 (0)