Skip to content

Commit 63efb2d

Browse files
authored
CI: Fix compute-sanitizer setup and harden mini-CTK fetching (#3397)
CI: Fix compute-sanitizer setup and harden mini-CTK fetching - Fix sanitizer mode, broken by #2437: the sanitizer step reran fetch_ctk, which deleted the mini CTK. A new install-mode: add input installs on top of the existing toolkit instead. - Install compute-sanitizer from the build CTK (cuda.build.version) rather than a run-time scrape of the latest release; remove ci/tools/guess_latest.sh. - Fail closed when a public CUDA version has no pinned manifest hash. Pin 13.5.0 and test that every CI CUDA version is pinned.
1 parent 5518e2d commit 63efb2d

9 files changed

Lines changed: 98 additions & 65 deletions

File tree

‎.github/actions/fetch_ctk/action.yml‎

Lines changed: 43 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,14 @@ inputs:
1919
description: "where the CTK components will be installed to, relative to $PWD"
2020
required: false
2121
default: "./cuda_toolkit"
22+
install-mode:
23+
description: >
24+
"replace" (default) empties cuda-path before installing.
25+
"add" installs the components on top of the mini CTK that an earlier
26+
fetch_ctk call installed at the same cuda-path, and leaves the
27+
environment that call exported unchanged.
28+
required: false
29+
default: "replace"
2230
redist-base-url:
2331
description: >
2432
Base URL of a redistrib server (manifest and component archives).
@@ -43,6 +51,34 @@ runs:
4351
with:
4452
dependent_exes: "curl unzip tar sha256sum cygpath"
4553

54+
- name: Check install mode
55+
shell: bash --noprofile --norc -xeuo pipefail {0}
56+
run: |
57+
case "${{ inputs.install-mode }}" in
58+
replace)
59+
;;
60+
add)
61+
# The environment exports are skipped in this mode, so the target
62+
# must be the CUDA_PATH an earlier fetch_ctk call exported.
63+
if [[ -z "${CUDA_PATH:-}" ]]; then
64+
echo "::error::install-mode 'add' requires CUDA_PATH from an earlier fetch_ctk call"
65+
exit 1
66+
fi
67+
EXISTING_PATH="$CUDA_PATH"
68+
if [[ "${{ inputs.host-platform }}" == win* ]]; then
69+
EXISTING_PATH="$(cygpath -u "$EXISTING_PATH")"
70+
fi
71+
if [[ "$(realpath -e "${{ inputs.cuda-path }}")" != "$(realpath -e "$EXISTING_PATH")" ]]; then
72+
echo "::error::install-mode 'add' requires cuda-path to be the existing CUDA_PATH ($CUDA_PATH)"
73+
exit 1
74+
fi
75+
;;
76+
*)
77+
echo "::error::install-mode must be 'replace' or 'add', got '${{ inputs.install-mode }}'"
78+
exit 1
79+
;;
80+
esac
81+
4682
- name: Set up CTK cache variable
4783
shell: bash --noprofile --norc -xeuo pipefail {0}
4884
run: |
@@ -170,7 +206,9 @@ runs:
170206
# symlinks from a prerelease CTK at ./cuda_toolkit/include etc., causing
171207
# "cp: cannot overwrite non-directory" failures on the next restore).
172208
CUDA_PATH="${{ inputs.cuda-path }}"
173-
rm -rf $CUDA_PATH
209+
if [[ "${{ inputs.install-mode }}" == replace ]]; then
210+
rm -rf $CUDA_PATH
211+
fi
174212
mkdir -p $CUDA_PATH
175213
# Unfortunately we cannot use "rsync -av $CACHE_TMP_DIR/ $CUDA_PATH" because
176214
# not all runners have rsync pre-installed (or even installable, such as
@@ -194,7 +232,9 @@ runs:
194232
ls -l
195233
CACHE_TMP_DIR="./cache_tmp_dir"
196234
CUDA_PATH="${{ inputs.cuda-path }}"
197-
rm -rf $CUDA_PATH
235+
if [[ "${{ inputs.install-mode }}" == replace ]]; then
236+
rm -rf $CUDA_PATH
237+
fi
198238
mkdir -p $CUDA_PATH
199239
tar -xzvf $CTK_CACHE_FILENAME
200240
# Can't use rsync here, see above
@@ -203,6 +243,7 @@ runs:
203243
ls -l $CUDA_PATH
204244
205245
- name: Set output environment variables
246+
if: ${{ inputs.install-mode == 'replace' }}
206247
shell: bash --noprofile --norc -xeuo pipefail {0}
207248
run: |
208249
# mimics actual CTK installation

‎.github/workflows/test-wheel-linux.yml‎

Lines changed: 5 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -366,14 +366,17 @@ jobs:
366366
host-platform: ${{ inputs.host-platform }}
367367
cuda-version: ${{ matrix.CUDA_VER }}
368368

369-
- name: Set up latest cuda_sanitizer_api
369+
- name: Set up cuda_sanitizer_api
370370
if: ${{ (env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true') && env.SETUP_SANITIZER == '1' }}
371371
uses: ./.github/actions/fetch_ctk
372372
continue-on-error: false
373373
with:
374374
host-platform: ${{ inputs.host-platform }}
375-
cuda-version: ${{ env.LATEST_CUDA_VERSION }}
375+
# The build CTK's sanitizer, not the matrix CTK's: older sanitizers
376+
# crash on the cuda.core tests, and newer ones carry the fixes.
377+
cuda-version: ${{ inputs.build-ctk-ver }}
376378
cuda-components: "cuda_sanitizer_api"
379+
install-mode: add
377380

378381
- name: Set up compute-sanitizer
379382
if: ${{ env.TEST_BINDINGS == 'true' || env.TEST_CORE == 'true' }}

‎ci/tools/compute_ci_plan.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -57,7 +57,6 @@
5757
".github/workflows/test-wheel-linux.yml": "linux",
5858
".github/workflows/test-wheel-windows.yml": "windows",
5959
"ci/tools/configure_driver_mode.ps1": "windows",
60-
"ci/tools/guess_latest.sh": "linux",
6160
"ci/tools/install_gpu_driver.ps1": "windows",
6261
"ci/tools/install_gpu_driver.sh": "linux",
6362
"ci/tools/setup-sanitizer": "linux",

‎ci/tools/env-vars‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,7 +119,6 @@ elif [[ "${1}" == "test" ]]; then
119119
# We only test compute-sanitizer on python 3.12 arbitrarily; we don't need to use sanitizer on the entire matrix
120120
# Only local ctk installs have compute-sanitizer; there is no wheel for it
121121
if [[ "${PY_VER}" == "3.12" && "${CUDA_VER}" != "11.8.0" && "${LOCAL_CTK}" == 1 && "${HOST_PLATFORM}" == linux* ]]; then
122-
echo "LATEST_CUDA_VERSION=$(bash ci/tools/guess_latest.sh $TEST_CUDA_MAJOR)" >> $GITHUB_ENV
123122
SETUP_SANITIZER=1
124123
else
125124
SETUP_SANITIZER=0

‎ci/tools/fetch_ctk_redistrib.py‎

Lines changed: 10 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -29,10 +29,11 @@
2929

3030
# Known-good sha256 of the redistrib metadata JSON, keyed by CUDA version.
3131
# Per-archive verification is only trustworthy if the JSON that carries the
32-
# archive hashes is itself trusted, so we pin the metadata too. A version that
33-
# is absent here is not blocked -- failing closed would break CI on every new
34-
# CTK -- but `verify-metadata` emits a warning; add the version's hash below as
35-
# it enters the matrix to pin it. Regenerate with:
32+
# archive hashes is itself trusted, so we pin the metadata too. `verify-metadata`
33+
# fails on a public-server version that is absent here, so add a version's hash
34+
# below in the same change that adds it to ci/versions.yml or a LOCAL_CTK row of
35+
# ci/test-matrix.yml; tests/test_fetch_ctk_redistrib.py enforces this.
36+
# Regenerate with:
3637
# curl -fsSL \
3738
# https://developer.download.nvidia.com/compute/cuda/redist/redistrib_<ver>.json \
3839
# | sha256sum
@@ -41,6 +42,7 @@
4142
"12.9.1": "8335301010b0023ee1ff61eb11e2600ca62002d76780de4089011ad77e0c7630",
4243
"13.0.2": "fce66717a81c510ffeb89ecc3e79849ab34af3b80139f750876d9033e31d71c2",
4344
"13.4.2": "015b97aa3977d4d0bade1631b226b6d4cbe48751cc841aad9193295df7f9047a",
45+
"13.5.0": "401529e951b845788177f999c0756c520c4d0db1cd4cd05c0a295e331884b596",
4446
}
4547

4648

@@ -154,8 +156,8 @@ def verify_metadata_sha256(*, cuda_version: str, metadata_path: str, metadata_so
154156
manifest under the same version number, so it is never checked against
155157
them; it only gets a warning.
156158
157-
Returns a process exit code: 1 on a mismatch (fail closed), 0 when the hash
158-
matches, the version is not pinned yet, or the source is custom (warn only).
159+
Returns a process exit code: 1 on a mismatch or an unpinned public version
160+
(fail closed), 0 when the hash matches or the source is custom (warn only).
159161
"""
160162
actual = compute_file_sha256(metadata_path)
161163
if metadata_source == "custom":
@@ -168,11 +170,11 @@ def verify_metadata_sha256(*, cuda_version: str, metadata_path: str, metadata_so
168170
expected = KNOWN_REDISTRIB_SHA256.get(cuda_version)
169171
if expected is None:
170172
print(
171-
f"WARNING: no pinned redistrib sha256 for CUDA {cuda_version!r}; got {actual}. "
173+
f"ERROR: no pinned redistrib sha256 for CUDA {cuda_version!r}; got {actual}. "
172174
f"Add it to KNOWN_REDISTRIB_SHA256 in {Path(__file__).name} to pin it.",
173175
file=sys.stderr,
174176
)
175-
return 0
177+
return 1
176178
if actual != expected:
177179
print(
178180
f"ERROR: redistrib metadata sha256 mismatch for CUDA {cuda_version!r}: expected {expected}, got {actual}",
@@ -207,7 +209,6 @@ def parse_args(argv: list[str] | None = None) -> argparse.Namespace:
207209
verify_parser.add_argument("--cuda-version", required=True)
208210
verify_parser.add_argument("--metadata-path", required=True)
209211
verify_parser.add_argument("--metadata-source", choices=("public", "custom"), default="public")
210-
211212
return parser.parse_args(argv)
212213

213214

‎ci/tools/guess_latest.sh‎

Lines changed: 0 additions & 42 deletions
This file was deleted.

‎ci/tools/tests/test_compute_ci_plan.py‎

Lines changed: 0 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,6 @@ def test_test_infrastructure_platforms(self) -> None:
8585
".github/workflows/test-wheel-linux.yml": {"linux"},
8686
".github/workflows/test-wheel-windows.yml": {"windows"},
8787
"ci/tools/configure_driver_mode.ps1": {"windows"},
88-
"ci/tools/guess_latest.sh": {"linux"},
8988
"ci/tools/install_gpu_driver.ps1": {"windows"},
9089
"ci/tools/install_gpu_driver.sh": {"linux"},
9190
"ci/tools/setup-sanitizer": {"linux"},

‎ci/tools/tests/test_fetch_ctk_redistrib.py‎

Lines changed: 35 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -4,20 +4,26 @@
44
from __future__ import annotations
55

66
import json
7+
import re
78
import sys
89
from pathlib import Path
910

1011
import pytest
12+
import yaml
1113

1214
sys.path.insert(0, str(Path(__file__).parent.parent))
1315
from fetch_ctk_redistrib import (
16+
KNOWN_REDISTRIB_SHA256,
1417
compute_file_sha256,
1518
get_component_relative_path,
1619
get_component_sha256,
1720
main,
1821
verify_metadata_sha256,
1922
)
2023

24+
CI_DIR = Path(__file__).resolve().parents[2]
25+
SHA256_RE = re.compile(r"^[0-9a-f]{64}$")
26+
2127
# Minimal redistrib-shaped metadata. "cccl" exercises the cuda_cccl -> cccl
2228
# alias; "no_sha" lacks the sha256 field.
2329
FAKE_METADATA = {
@@ -101,14 +107,37 @@ def test_verify_metadata_mismatch_fails_closed(tmp_path, monkeypatch):
101107
assert verify_metadata_sha256(cuda_version="1.2.3", metadata_path=str(f)) == 1
102108

103109

104-
@pytest.mark.agent_authored(model="claude-opus-4.8")
105-
def test_verify_metadata_unknown_version_warns(tmp_path, capsys):
110+
@pytest.mark.agent_authored(model="claude-opus-5.5")
111+
def test_verify_metadata_unknown_version_fails_closed(tmp_path, capsys):
106112
f = tmp_path / "redistrib.json"
107113
f.write_bytes(b"whatever")
108-
# An unpinned version must not fail closed (it would break CI on every new
109-
# CTK), but it must emit a warning.
110-
assert verify_metadata_sha256(cuda_version="99.9.9", metadata_path=str(f)) == 0
111-
assert "WARNING" in capsys.readouterr().err
114+
assert verify_metadata_sha256(cuda_version="99.9.9", metadata_path=str(f)) == 1
115+
assert "ERROR" in capsys.readouterr().err
116+
117+
118+
@pytest.mark.agent_authored(model="claude-opus-5.5")
119+
def test_known_redistrib_sha256_covers_ci_cuda_versions():
120+
# Every CUDA version CI fetches from the public server must be pinned, so a
121+
# version bump fails here instead of in the fetch_ctk step.
122+
versions = yaml.safe_load((CI_DIR / "versions.yml").read_text(encoding="utf-8"))
123+
matrix = yaml.safe_load((CI_DIR / "test-matrix.yml").read_text(encoding="utf-8"))
124+
required = {str(entry["version"]) for entry in versions["cuda"].values()}
125+
required |= {
126+
str(row["CUDA_VER"])
127+
for platform in ("linux", "windows")
128+
for rows in matrix[platform].values()
129+
for row in rows
130+
if str(row.get("LOCAL_CTK")) == "1"
131+
}
132+
assert required
133+
missing = sorted(required - KNOWN_REDISTRIB_SHA256.keys())
134+
assert not missing, (
135+
f"CUDA versions used by CI have no pinned redistrib manifest hash: {missing}. "
136+
"Add them to KNOWN_REDISTRIB_SHA256 in ci/tools/fetch_ctk_redistrib.py; compute each hash with "
137+
"`curl -fsSL https://developer.download.nvidia.com/compute/cuda/redist/redistrib_<ver>.json | sha256sum`."
138+
)
139+
for version, sha256 in KNOWN_REDISTRIB_SHA256.items():
140+
assert SHA256_RE.fullmatch(sha256), f"invalid pinned sha256 for CUDA {version}"
112141

113142

114143
@pytest.mark.agent_authored(model="claude-opus-4.8")

‎ci/versions.yml‎

Lines changed: 5 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,9 +3,13 @@
33

44
backport_branch: "12.9.x" # keep in sync with target-branch in .github/dependabot.yml
55

6-
# This controls the version of the upstream CTK to install to build the wheels.
6+
# This controls the version of the upstream CTK to install to build the wheels,
7+
# and the compute-sanitizer version used in all sanitizer test runs.
78
# It is not the version of cuda_bindings/cuda_python being released, which is
89
# solely determined by git tags via setuptools_scm.
10+
# Each version here, and in every LOCAL_CTK: '1' row of ci/test-matrix.yml,
11+
# needs a pinned manifest hash in KNOWN_REDISTRIB_SHA256 in
12+
# ci/tools/fetch_ctk_redistrib.py; fetch_ctk fails without one.
913
cuda:
1014
build:
1115
version: "13.5.0"

0 commit comments

Comments
 (0)