Skip to content

Commit 5afb8b6

Browse files
SalmaSamySalmaSamy
authored
Lockfile cherry picks (bazelbuild#18143)
* Module Lockfile - Created lockfile K/F/V holding module information and its related flags - Updated BazelDepGraph function to use the lockfile - Created type adapters for module, related classes & immutable collections - Updated tests with Module.bazel to work after enabling the lockfile - Added new java test class for the lockfile - Added a flag to enable/disable lockfile (while it's still in progress) - Added integration tests PiperOrigin-RevId: 520925174 Change-Id: Ib8da99aa86a1797da44dc9a899d932f58c4a84c9 # Conflicts: # src/test/py/bazel/bzlmod/bazel_module_test.py * Make lockfile human readable PiperOrigin-RevId: 521009432 Change-Id: I2b2e5e0c845f05c428ff8772aba2abce72a3830f * Split bazel_module_tests into smaller ones PiperOrigin-RevId: 521766028 Change-Id: I44f5f68e4357bcc1219985e983814c5e4afec65d # Conflicts: # src/test/py/bazel/BUILD # src/test/py/bazel/bzlmod/bazel_module_test.py * Update lockfile name PiperOrigin-RevId: 521871482 Change-Id: Ib4357c4d35fa8ebd2e636c7f15740247ee8dfab2 * Add repospec to the module - With this we can get the repospec directly from the module not the registry (no need for internet access) - Since Repospec & Tag classes have an "attributes" field that can hold starlark values, an adapter was needed to serialize/deserialize starlark values PiperOrigin-RevId: 523364539 Change-Id: Ifcdf0c6b4b6fbbcdae9c14a4b0cd7f53ae91c161 * Update lockfile flag from experimental to different modes flag: - Update: run resolution and update the lock file when it mismatches the module - Error: throw an error if the module doesn't match the lockfile - Off: don't read/update the lockfile PiperOrigin-RevId: 524813416 Change-Id: I5cc3577fdbed8339ada50001081b75b4932c017c * Hash local overrides Module files into the lockfile PiperOrigin-RevId: 525405790 Change-Id: I7c9b26469c889435ed4adc4a49197c10e0284af0 # Conflicts: # src/main/java/com/google/devtools/build/lib/bazel/bzlmod/BzlmodFlagsAndEnvVars.java # src/test/java/com/google/devtools/build/lib/bazel/bzlmod/BazelLockFileFunctionTest.java * update functions names * Add auto_value_gson and fixes * update auto value import * increase install base size RELNOTES: Added preliminary support for a lockfile for Bzlmod. It's disabled by default; use `--lockfile_mode=update` to enable it. This lockfile contains only Bazel module information; it does not involve module extensions.
1 parent 76ad4a9 commit 5afb8b6

File tree

68 files changed

+3169
-840
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

68 files changed

+3169
-840
lines changed

‎.bazelci/postsubmit.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -331,6 +331,10 @@ tasks:
331331
- "-//src/java_tools/import_deps_checker/..."
332332
# We hit connection timeout error when downloading multiple URLs on RBE, see b/217865760
333333
- "-//src/test/py/bazel:bazel_module_test"
334+
- "-//src/test/py/bazel:bazel_lockfile_test"
335+
- "-//src/test/py/bazel:bazel_overrides_test"
336+
- "-//src/test/py/bazel:bazel_repo_mapping_test"
337+
- "-//src/test/py/bazel:bazel_yanked_versions_test"
334338
- "-//src/test/shell/bazel:verify_workspace"
335339
include_json_profile:
336340
- build

‎.bazelci/presubmit.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -321,6 +321,10 @@ tasks:
321321
- "-//src/java_tools/import_deps_checker/..."
322322
# We hit connection timeout error when downloading multiple URLs on RBE, see b/217865760
323323
- "-//src/test/py/bazel:bazel_module_test"
324+
- "-//src/test/py/bazel:bazel_lockfile_test"
325+
- "-//src/test/py/bazel:bazel_overrides_test"
326+
- "-//src/test/py/bazel:bazel_repo_mapping_test"
327+
- "-//src/test/py/bazel:bazel_yanked_versions_test"
324328
- "-//src/test/shell/bazel:verify_workspace"
325329
kythe_ubuntu2004:
326330
shell_commands:

‎src/main/java/com/google/devtools/build/lib/bazel/BazelRepositoryModule.java‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -25,7 +25,9 @@
2525
import com.google.devtools.build.lib.analysis.ConfiguredRuleClassProvider;
2626
import com.google.devtools.build.lib.analysis.RuleDefinition;
2727
import com.google.devtools.build.lib.analysis.config.BuildConfigurationValue;
28+
import com.google.devtools.build.lib.bazel.bzlmod.AttributeValues;
2829
import com.google.devtools.build.lib.bazel.bzlmod.BazelDepGraphFunction;
30+
import com.google.devtools.build.lib.bazel.bzlmod.BazelLockFileFunction;
2931
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorFunction;
3032
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleInspectorValue.AugmentedModule.ResolutionReason;
3133
import com.google.devtools.build.lib.bazel.bzlmod.BazelModuleResolutionFunction;
@@ -46,6 +48,7 @@
4648
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions;
4749
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.BazelCompatibilityMode;
4850
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.CheckDirectDepsMode;
51+
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.LockfileMode;
4952
import com.google.devtools.build.lib.bazel.repository.RepositoryOptions.RepositoryOverride;
5053
import com.google.devtools.build.lib.bazel.repository.cache.RepositoryCache;
5154
import com.google.devtools.build.lib.bazel.repository.downloader.DelegatingDownloader;
@@ -139,6 +142,7 @@ public class BazelRepositoryModule extends BlazeModule {
139142
private final AtomicBoolean ignoreDevDeps = new AtomicBoolean(false);
140143
private CheckDirectDepsMode checkDirectDepsMode = CheckDirectDepsMode.WARNING;
141144
private BazelCompatibilityMode bazelCompatibilityMode = BazelCompatibilityMode.ERROR;
145+
private LockfileMode bazelLockfileMode = LockfileMode.OFF;
142146
private List<String> allowedYankedVersions = ImmutableList.of();
143147
private SingleExtensionEvalFunction singleExtensionEvalFunction;
144148

@@ -232,7 +236,8 @@ public void workspaceInit(
232236
public RepoSpec getRepoSpec(RepositoryName repoName) {
233237
return RepoSpec.builder()
234238
.setRuleClassName("local_config_platform")
235-
.setAttributes(ImmutableMap.of("name", repoName.getName()))
239+
.setAttributes(
240+
AttributeValues.create(ImmutableMap.of("name", repoName.getName())))
236241
.build();
237242
}
238243

@@ -249,7 +254,10 @@ public ResolutionReason getResolutionReason() {
249254
.addSkyFunction(
250255
SkyFunctions.MODULE_FILE,
251256
new ModuleFileFunction(registryFactory, directories.getWorkspace(), builtinModules))
252-
.addSkyFunction(SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction())
257+
.addSkyFunction(
258+
SkyFunctions.BAZEL_DEP_GRAPH, new BazelDepGraphFunction(directories.getWorkspace()))
259+
.addSkyFunction(
260+
SkyFunctions.BAZEL_LOCK_FILE, new BazelLockFileFunction(directories.getWorkspace()))
253261
.addSkyFunction(SkyFunctions.BAZEL_MODULE_INSPECTION, new BazelModuleInspectorFunction())
254262
.addSkyFunction(SkyFunctions.BAZEL_MODULE_RESOLUTION, new BazelModuleResolutionFunction())
255263
.addSkyFunction(SkyFunctions.SINGLE_EXTENSION_EVAL, singleExtensionEvalFunction)
@@ -421,6 +429,7 @@ public void beforeCommand(CommandEnvironment env) throws AbruptExitException {
421429
ignoreDevDeps.set(repoOptions.ignoreDevDependency);
422430
checkDirectDepsMode = repoOptions.checkDirectDependencies;
423431
bazelCompatibilityMode = repoOptions.bazelCompatibilityMode;
432+
bazelLockfileMode = repoOptions.lockfileMode;
424433
allowedYankedVersions = repoOptions.allowedYankedVersions;
425434

426435
if (repoOptions.registries != null && !repoOptions.registries.isEmpty()) {
@@ -512,6 +521,7 @@ public ImmutableList<Injected> getPrecomputedValues() {
512521
BazelModuleResolutionFunction.CHECK_DIRECT_DEPENDENCIES, checkDirectDepsMode),
513522
PrecomputedValue.injected(
514523
BazelModuleResolutionFunction.BAZEL_COMPATIBILITY_MODE, bazelCompatibilityMode),
524+
PrecomputedValue.injected(BazelLockFileFunction.LOCKFILE_MODE, bazelLockfileMode),
515525
PrecomputedValue.injected(
516526
BazelModuleResolutionFunction.ALLOWED_YANKED_VERSIONS, allowedYankedVersions));
517527
}

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/ArchiveRepoSpecBuilder.java‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -89,7 +89,7 @@ public RepoSpec build() {
8989
return RepoSpec.builder()
9090
.setBzlFile("@bazel_tools//tools/build_defs/repo:http.bzl")
9191
.setRuleClassName("http_archive")
92-
.setAttributes(attrBuilder.buildOrThrow())
92+
.setAttributes(AttributeValues.create(attrBuilder.buildOrThrow()))
9393
.build();
9494
}
9595
}

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValues.java‎

Lines changed: 58 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,58 @@
1+
// Copyright 2023 The Bazel Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
//
15+
16+
package com.google.devtools.build.lib.bazel.bzlmod;
17+
18+
import com.google.auto.value.AutoValue;
19+
import com.google.common.collect.Maps;
20+
import com.ryanharter.auto.value.gson.GenerateTypeAdapter;
21+
import java.util.List;
22+
import java.util.Map;
23+
import net.starlark.java.eval.Dict;
24+
import net.starlark.java.eval.Starlark;
25+
26+
/** Wraps a dictionary of attribute names and values. Always uses a dict to represent them */
27+
@AutoValue
28+
@GenerateTypeAdapter
29+
public abstract class AttributeValues {
30+
31+
public static AttributeValues create(Dict<String, Object> attribs) {
32+
return new AutoValue_AttributeValues(attribs);
33+
}
34+
35+
public static AttributeValues create(Map<String, Object> attribs) {
36+
return new AutoValue_AttributeValues(
37+
Dict.immutableCopyOf(Maps.transformValues(attribs, AttributeValues::valueToStarlark)));
38+
}
39+
40+
public abstract Dict<String, Object> attributes();
41+
42+
// TODO(salmasamy) this is a copy of Attribute::valueToStarlark, Maybe think of a better place?
43+
private static Object valueToStarlark(Object x) {
44+
// Is x a non-empty string_list_dict?
45+
if (x instanceof Map) {
46+
Map<?, ?> map = (Map<?, ?>) x;
47+
if (!map.isEmpty() && map.values().iterator().next() instanceof List) {
48+
Dict.Builder<Object, Object> dict = Dict.builder();
49+
for (Map.Entry<?, ?> e : map.entrySet()) {
50+
dict.put(e.getKey(), Starlark.fromJava(e.getValue(), null));
51+
}
52+
return dict.buildImmutable();
53+
}
54+
}
55+
// For all other attribute values, shallow conversion is safe.
56+
return Starlark.fromJava(x, null);
57+
}
58+
}

‎src/main/java/com/google/devtools/build/lib/bazel/bzlmod/AttributeValuesAdapter.java‎

Lines changed: 156 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,156 @@
1+
// Copyright 2023 The Bazel Authors. All rights reserved.
2+
//
3+
// Licensed under the Apache License, Version 2.0 (the "License");
4+
// you may not use this file except in compliance with the License.
5+
// You may obtain a copy of the License at
6+
//
7+
// http://www.apache.org/licenses/LICENSE-2.0
8+
//
9+
// Unless required by applicable law or agreed to in writing, software
10+
// distributed under the License is distributed on an "AS IS" BASIS,
11+
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12+
// See the License for the specific language governing permissions and
13+
// limitations under the License.
14+
//
15+
16+
package com.google.devtools.build.lib.bazel.bzlmod;
17+
18+
import com.google.devtools.build.lib.cmdline.Label;
19+
import com.google.gson.JsonArray;
20+
import com.google.gson.JsonElement;
21+
import com.google.gson.JsonNull;
22+
import com.google.gson.JsonObject;
23+
import com.google.gson.JsonParser;
24+
import com.google.gson.JsonPrimitive;
25+
import com.google.gson.TypeAdapter;
26+
import com.google.gson.stream.JsonReader;
27+
import com.google.gson.stream.JsonWriter;
28+
import java.io.IOException;
29+
import java.util.ArrayList;
30+
import java.util.List;
31+
import java.util.Map;
32+
import net.starlark.java.eval.Dict;
33+
import net.starlark.java.eval.EvalException;
34+
import net.starlark.java.eval.Mutability;
35+
import net.starlark.java.eval.Starlark;
36+
import net.starlark.java.eval.StarlarkInt;
37+
import net.starlark.java.eval.StarlarkList;
38+
39+
/** Helps serialize/deserialize {@link AttributeValues}, which contains Starlark values. */
40+
public class AttributeValuesAdapter extends TypeAdapter<AttributeValues> {
41+
42+
@Override
43+
public void write(JsonWriter out, AttributeValues attributeValues) throws IOException {
44+
JsonObject jsonObject = new JsonObject();
45+
for (Map.Entry<String, Object> entry : attributeValues.attributes().entrySet()) {
46+
jsonObject.add(entry.getKey(), serializeObject(entry.getValue()));
47+
}
48+
out.jsonValue(jsonObject.toString());
49+
}
50+
51+
@Override
52+
public AttributeValues read(JsonReader in) throws IOException {
53+
JsonObject jsonObject = JsonParser.parseReader(in).getAsJsonObject();
54+
Dict.Builder<String, Object> dict = Dict.builder();
55+
for (Map.Entry<String, JsonElement> entry : jsonObject.entrySet()) {
56+
dict.put(entry.getKey(), deserializeObject(entry.getValue()));
57+
}
58+
return AttributeValues.create(dict.buildImmutable());
59+
}
60+
61+
/**
62+
* Starlark Object Types Bool Integer String Label List (Int, label, string) Dict (String,list) &
63+
* (Label, String)
64+
*/
65+
private JsonElement serializeObject(Object obj) {
66+
if (obj.equals(Starlark.NONE)) {
67+
return JsonNull.INSTANCE;
68+
} else if (obj instanceof Boolean) {
69+
return new JsonPrimitive((Boolean) obj);
70+
} else if (obj instanceof StarlarkInt) {
71+
try {
72+
return new JsonPrimitive(((StarlarkInt) obj).toInt("serialization into the lockfile"));
73+
} catch (EvalException e) {
74+
throw new IllegalArgumentException("Unable to parse StarlarkInt to Integer: " + e);
75+
}
76+
} else if (obj instanceof String || obj instanceof Label) {
77+
return new JsonPrimitive(serializeObjToString(obj));
78+
} else if (obj instanceof Dict) {
79+
JsonObject jsonObject = new JsonObject();
80+
for (Map.Entry<?, ?> entry : ((Dict<?, ?>) obj).entrySet()) {
81+
jsonObject.add(serializeObjToString(entry.getKey()), serializeObject(entry.getValue()));
82+
}
83+
return jsonObject;
84+
} else if (obj instanceof StarlarkList) {
85+
JsonArray jsonArray = new JsonArray();
86+
for (Object item : (StarlarkList<?>) obj) {
87+
jsonArray.add(serializeObject(item));
88+
}
89+
return jsonArray;
90+
} else {
91+
throw new IllegalArgumentException("Unsupported type: " + obj.getClass());
92+
}
93+
}
94+
95+
private Object deserializeObject(JsonElement json) {
96+
if (json == null || json.isJsonNull()) {
97+
return Starlark.NONE;
98+
} else if (json.isJsonPrimitive()) {
99+
JsonPrimitive jsonPrimitive = json.getAsJsonPrimitive();
100+
if (jsonPrimitive.isBoolean()) {
101+
return jsonPrimitive.getAsBoolean();
102+
} else if (jsonPrimitive.isNumber()) {
103+
return StarlarkInt.of(jsonPrimitive.getAsInt());
104+
} else if (jsonPrimitive.isString()) {
105+
return deserializeStringToObject(jsonPrimitive.getAsString());
106+
} else {
107+
throw new IllegalArgumentException("Unsupported JSON primitive: " + jsonPrimitive);
108+
}
109+
} else if (json.isJsonObject()) {
110+
JsonObject jsonObject = json.getAsJsonObject();
111+
Dict.Builder<Object, Object> dict = Dict.builder();
112+
for (Map.Entry<String, JsonElement> entry : jsonObject.entrySet()) {
113+
dict.put(deserializeStringToObject(entry.getKey()), deserializeObject(entry.getValue()));
114+
}
115+
return dict.buildImmutable();
116+
} else if (json.isJsonArray()) {
117+
JsonArray jsonArray = json.getAsJsonArray();
118+
List<Object> list = new ArrayList<>();
119+
for (JsonElement item : jsonArray) {
120+
list.add(deserializeObject(item));
121+
}
122+
return StarlarkList.copyOf(Mutability.IMMUTABLE, list);
123+
} else {
124+
throw new IllegalArgumentException("Unsupported JSON element: " + json);
125+
}
126+
}
127+
128+
/**
129+
* Serializes an object (Label or String) to String A label is converted to a String as it is. A
130+
* String is being modified with a delimiter to be easily differentiated from the label when
131+
* deserializing.
132+
*
133+
* @param obj String or Label
134+
* @return serialized object
135+
*/
136+
private String serializeObjToString(Object obj) {
137+
if (obj instanceof Label) {
138+
return ((Label) obj).getUnambiguousCanonicalForm();
139+
}
140+
return "--" + obj;
141+
}
142+
143+
/**
144+
* Deserializes a string to either a label or a String depending on the prefix. A string will have
145+
* a delimiter at the start, else it is converted to a label.
146+
*
147+
* @param value String to be deserialized
148+
* @return Object of type String of Label
149+
*/
150+
private Object deserializeStringToObject(String value) {
151+
if (value.startsWith("--")) {
152+
return value.substring(2);
153+
}
154+
return Label.parseCanonicalUnchecked(value);
155+
}
156+
}

0 commit comments

Comments
 (0)