Skip to content

Commit a37ebe9

Browse files
committed
build(cargo): keep conformance lockfile aligned with the root
`cargo --locked` only checks a lockfile against its own manifest. The conformance suites build openshell-sdk from a separate workspace with its own Cargo.lock, so the two lockfiles could resolve shared crates to different versions without any check failing. That is how aws-lc-sys 0.46 was picked up, which does not link against the AWS-LC Nix provides. - Add check_cargo_lockfile_alignment.py: aligned workspaces must lock every registry crate they share with the root at a version the root lockfile has. `--fix` re-locks a workspace from the root lockfile, which keeps coupled crate families such as futures-* consistent. - Enroll tests/suites/conformance and re-lock it from the root lockfile. The other workspaces that path-depend on root crates also drift and can be enrolled once aligned. - Run the check in rust:lockfiles:check and the Cargo lockfiles CI job, and its tests in `mise run test`. - Build test archives and run the test runner with --locked so a stale lockfile fails instead of being re-resolved. Signed-off-by: Evan Lezar <[email protected]>
1 parent 898ca46 commit a37ebe9

10 files changed

Lines changed: 737 additions & 345 deletions

File tree

‎.github/workflows/branch-checks.yml‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -239,6 +239,10 @@ jobs:
239239
shell: nix develop -c bash -euo pipefail {0}
240240
run: tasks/scripts/check-cargo-lockfiles.sh
241241

242+
- name: Verify Cargo lockfile alignment
243+
shell: nix develop -c bash -euo pipefail {0}
244+
run: uv run tasks/scripts/check_cargo_lockfile_alignment.py
245+
242246
rust-lint:
243247
name: Rust lint (${{ matrix.system }})
244248
needs: pr_metadata

‎TESTING.md‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -48,6 +48,17 @@ mise run test:rust # cargo test --workspace
4848

4949
Rust validation checks tracked Cargo lockfiles; run `mise run rust:lockfiles:check` to check them directly. If one is stale, refresh it with Cargo using its adjacent manifest, review the diff, and commit the update.
5050

51+
Workspaces that build root crates through path dependencies keep their own
52+
lockfiles, which `--locked` does not compare with the root's. The check also
53+
requires the workspaces listed in
54+
`tasks/scripts/check_cargo_lockfile_alignment.py` to lock shared crates at the
55+
root's versions. If it fails, re-lock them from the root lockfile and commit the
56+
result:
57+
58+
```shell
59+
uv run tasks/scripts/check_cargo_lockfile_alignment.py --fix
60+
```
61+
5162
### PostgreSQL-backed tests
5263

5364
Tests that need a real PostgreSQL server, such as advisory-lock concurrency

‎tasks/rust.toml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,11 @@ run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts
1818
hide = true
1919

2020
["rust:lockfiles:check"]
21-
description = "Verify all tracked Cargo lockfiles are current"
22-
run = "tasks/scripts/check-cargo-lockfiles.sh"
21+
description = "Verify all tracked Cargo lockfiles are current and aligned with the root"
22+
run = [
23+
"tasks/scripts/check-cargo-lockfiles.sh",
24+
"uv run tasks/scripts/check_cargo_lockfile_alignment.py",
25+
]
2326
run_windows = "powershell -NoProfile -ExecutionPolicy Bypass -File tasks/scripts/check-cargo-lockfiles.ps1"
2427
hide = true
2528

Lines changed: 169 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,169 @@
1+
#!/usr/bin/env python3
2+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
# /// script
6+
# requires-python = ">=3.11"
7+
# ///
8+
9+
"""Check that aligned Cargo workspaces lock shared crates like the root does.
10+
11+
Some workspaces outside the root build root crates through path
12+
dependencies, for example the conformance suites build `openshell-sdk`. Each
13+
workspace has its own Cargo.lock, and `cargo --locked` only checks a lockfile
14+
against its own manifest, so nothing stops the two lockfiles from resolving
15+
the same crate to different versions. The root's versions are the ones CI and
16+
the Nix toolchain are built for; a newer `aws-lc-sys`, for example, does not
17+
link against the AWS-LC that Nix provides.
18+
19+
This check fails when an aligned workspace locks a registry crate that the
20+
root lockfile also contains at a version the root lockfile does not have.
21+
With `--fix`, it re-locks each aligned workspace starting from the root
22+
lockfile.
23+
"""
24+
25+
from __future__ import annotations
26+
27+
import argparse
28+
import re
29+
import shutil
30+
import subprocess
31+
import sys
32+
import tomllib
33+
from dataclasses import dataclass
34+
from pathlib import Path
35+
36+
# Workspaces whose shared crates must match the root lockfile. Add a
37+
# workspace here once its lockfile has been aligned with `--fix`.
38+
ALIGNED_WORKSPACES = ("tests/suites/conformance",)
39+
40+
41+
@dataclass(frozen=True)
42+
class Mismatch:
43+
workspace: str
44+
name: str
45+
version: str
46+
root_versions: tuple[str, ...]
47+
48+
def target(self) -> str | None:
49+
"""The root version semver-compatible with the locked one, if any."""
50+
key = compat_key(self.version)
51+
candidates = [v for v in self.root_versions if compat_key(v) == key]
52+
return max(candidates, key=version_tuple) if candidates else None
53+
54+
55+
def version_tuple(version: str) -> tuple[int, ...]:
56+
core = re.split(r"[-+]", version, maxsplit=1)[0]
57+
return tuple(int(part) for part in core.split("."))
58+
59+
60+
def compat_key(version: str) -> tuple[int, ...]:
61+
"""Cargo's caret-compatibility class: the leftmost non-zero component."""
62+
major, minor, patch = (*version_tuple(version), 0, 0, 0)[:3]
63+
if major:
64+
return (major,)
65+
if minor:
66+
return (0, minor)
67+
return (0, 0, patch)
68+
69+
70+
def registry_versions(lockfile: Path) -> dict[str, set[str]]:
71+
"""Versions of registry (non-path) packages in a lockfile, by name."""
72+
data = tomllib.loads(lockfile.read_text())
73+
versions: dict[str, set[str]] = {}
74+
for package in data.get("package", []):
75+
if "source" in package:
76+
versions.setdefault(package["name"], set()).add(package["version"])
77+
return versions
78+
79+
80+
def mismatches(root: Path, workspace: str) -> list[Mismatch]:
81+
root_versions = registry_versions(root / "Cargo.lock")
82+
found = []
83+
for name, versions in sorted(
84+
registry_versions(root / workspace / "Cargo.lock").items()
85+
):
86+
shared = root_versions.get(name)
87+
if shared is None:
88+
continue
89+
for version in sorted(versions - shared, key=version_tuple):
90+
found.append(
91+
Mismatch(
92+
workspace, name, version, tuple(sorted(shared, key=version_tuple))
93+
)
94+
)
95+
return found
96+
97+
98+
def fix(root: Path, workspace: str) -> list[Mismatch]:
99+
"""Re-lock the workspace starting from the root lockfile.
100+
101+
Coupled crates such as the `futures-*` family pin each other, so they
102+
cannot be moved one at a time. Seeding the workspace lockfile with the
103+
root's lets Cargo keep every root version that still fits, add only the
104+
crates the root does not have, and drop the rest. Returns what is still
105+
misaligned afterwards.
106+
"""
107+
shutil.copyfile(root / "Cargo.lock", root / workspace / "Cargo.lock")
108+
subprocess.run(
109+
[
110+
"cargo",
111+
"metadata",
112+
"--format-version",
113+
"1",
114+
"--manifest-path",
115+
str(root / workspace / "Cargo.toml"),
116+
],
117+
check=True,
118+
stdout=subprocess.DEVNULL,
119+
)
120+
return mismatches(root, workspace)
121+
122+
123+
def report(problems: list[Mismatch]) -> None:
124+
for m in problems:
125+
target = m.target()
126+
hint = (
127+
f"; root has {target}"
128+
if target
129+
else f"; root has {', '.join(m.root_versions)}"
130+
)
131+
print(
132+
f"error: {m.workspace}: {m.name} {m.version} is not in the root Cargo.lock{hint}",
133+
file=sys.stderr,
134+
)
135+
136+
137+
def main(argv: list[str] | None = None) -> int:
138+
parser = argparse.ArgumentParser(description=__doc__.splitlines()[0])
139+
parser.add_argument(
140+
"--fix", action="store_true", help="update aligned lockfiles to root versions"
141+
)
142+
args = parser.parse_args(argv)
143+
144+
root = Path(
145+
subprocess.run(
146+
["git", "rev-parse", "--show-toplevel"],
147+
check=True,
148+
capture_output=True,
149+
text=True,
150+
).stdout.strip()
151+
)
152+
problems: list[Mismatch] = []
153+
for workspace in ALIGNED_WORKSPACES:
154+
print(f"Checking {workspace}/Cargo.lock against Cargo.lock")
155+
problems += fix(root, workspace) if args.fix else mismatches(root, workspace)
156+
157+
if problems:
158+
report(problems)
159+
if not args.fix:
160+
print(
161+
"Run `uv run tasks/scripts/check_cargo_lockfile_alignment.py --fix` to align them.",
162+
file=sys.stderr,
163+
)
164+
return 1
165+
return 0
166+
167+
168+
if __name__ == "__main__":
169+
sys.exit(main())
Lines changed: 85 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,85 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
# SPDX-License-Identifier: Apache-2.0
3+
4+
"""Tests for the Cargo lockfile alignment check."""
5+
6+
from __future__ import annotations
7+
8+
import importlib.util
9+
import sys
10+
from pathlib import Path
11+
12+
_SPEC = importlib.util.spec_from_file_location(
13+
"check_cargo_lockfile_alignment",
14+
Path(__file__).with_name("check_cargo_lockfile_alignment.py"),
15+
)
16+
assert _SPEC and _SPEC.loader
17+
alignment = importlib.util.module_from_spec(_SPEC)
18+
sys.modules[_SPEC.name] = alignment
19+
_SPEC.loader.exec_module(alignment)
20+
21+
REGISTRY = 'source = "registry+https://github.com/rust-lang/crates.io-index"'
22+
23+
24+
def lockfile(path: Path, packages: list[tuple[str, str, bool]]) -> None:
25+
"""Write a lockfile of (name, version, from_registry) packages."""
26+
entries = []
27+
for name, version, from_registry in packages:
28+
entry = f'[[package]]\nname = "{name}"\nversion = "{version}"\n'
29+
if from_registry:
30+
entry += REGISTRY + "\n"
31+
entries.append(entry)
32+
path.parent.mkdir(parents=True, exist_ok=True)
33+
path.write_text("version = 4\n\n" + "\n".join(entries))
34+
35+
36+
def test_compat_key_follows_cargo_caret_rules() -> None:
37+
assert alignment.compat_key("1.52.1") == (1,)
38+
assert alignment.compat_key("0.44.0") == (0, 44)
39+
assert alignment.compat_key("0.0.3") == (0, 0, 3)
40+
assert alignment.compat_key("1.0.4+wasi-0.2.12") == (1,)
41+
42+
43+
def test_reports_shared_registry_crates_at_other_versions(tmp_path: Path) -> None:
44+
lockfile(
45+
tmp_path / "Cargo.lock",
46+
[
47+
("tokio", "1.52.1", True),
48+
("rand", "0.8.6", True),
49+
("rand", "0.9.4", True),
50+
("openshell-sdk", "0.0.0", False),
51+
],
52+
)
53+
lockfile(
54+
tmp_path / "ws" / "Cargo.lock",
55+
[
56+
("tokio", "1.53.1", True),
57+
("rand", "0.9.5", True),
58+
("rand", "0.8.6", True),
59+
("only-here", "2.0.0", True),
60+
("openshell-sdk", "0.0.0", False),
61+
],
62+
)
63+
64+
found = alignment.mismatches(tmp_path, "ws")
65+
66+
assert [(m.name, m.version, m.target()) for m in found] == [
67+
("rand", "0.9.5", "0.9.4"),
68+
("tokio", "1.53.1", "1.52.1"),
69+
]
70+
71+
72+
def test_no_target_without_a_compatible_root_version(tmp_path: Path) -> None:
73+
lockfile(tmp_path / "Cargo.lock", [("syn", "1.0.109", True)])
74+
lockfile(tmp_path / "ws" / "Cargo.lock", [("syn", "2.0.119", True)])
75+
76+
(found,) = alignment.mismatches(tmp_path, "ws")
77+
78+
assert found.target() is None
79+
80+
81+
def test_aligned_lockfiles_pass(tmp_path: Path) -> None:
82+
lockfile(tmp_path / "Cargo.lock", [("tokio", "1.52.1", True)])
83+
lockfile(tmp_path / "ws" / "Cargo.lock", [("tokio", "1.52.1", True)])
84+
85+
assert alignment.mismatches(tmp_path, "ws") == []

‎tasks/test.toml‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -23,6 +23,7 @@ depends = [
2323
"test:docs-website",
2424
"test:docs-nav",
2525
"test:maintainer-approval",
26+
"test:cargo-lockfile-alignment",
2627
]
2728

2829
["test:docs-website"]
@@ -31,6 +32,10 @@ description = "Test the docs-website sync script"
3132
# dependencies and the script's runtime dependency, which live outside the project env.
3233
run = "uv run --no-project --with pytest --with pytest-asyncio --with pyyaml pytest tasks/scripts/sync_docs_website_test.py"
3334

35+
["test:cargo-lockfile-alignment"]
36+
description = "Test the Cargo lockfile alignment check"
37+
run = "uv run --no-project --with pytest pytest tasks/scripts/check_cargo_lockfile_alignment_test.py"
38+
3439
["test:docs-nav"]
3540
description = "Test the docs navigation check"
3641
run = "uv run --no-project --with pytest --with pytest-asyncio --with pyyaml pytest tasks/scripts/check_docs_nav_test.py"

‎tests/artifacts.nix‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,7 @@ let
5858
mkdir -p "$(dirname "$output")"
5959
cd "$root"
6060
cargo nextest archive \
61+
--locked \
6162
--manifest-path "$manifest_path" \
6263
--target ${target} \
6364
-p ${package} \

‎tests/config.nix‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -259,7 +259,7 @@ let
259259
text = ''
260260
root=$(git rev-parse --show-toplevel)
261261
export ANSIBLE_CONFIG="$root/tests/ansible/ansible.cfg"
262-
exec cargo run --quiet --manifest-path "$root/tests/runner/Cargo.toml" --bin openshell-test -- "$@"
262+
exec cargo run --quiet --locked --manifest-path "$root/tests/runner/Cargo.toml" --bin openshell-test -- "$@"
263263
'';
264264
};
265265
in

‎tests/runner/src/bundle.rs‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -60,7 +60,7 @@ pub fn build(
6060

6161
let mut command = Command::new("cargo");
6262
command
63-
.args(["nextest", "archive", "--manifest-path"])
63+
.args(["nextest", "archive", "--locked", "--manifest-path"])
6464
.arg(&manifest)
6565
.args(["--target", target, "--archive-file"])
6666
.arg(&archive);

0 commit comments

Comments
 (0)