Skip to content

Commit decf58d

Browse files
mangeshraut712mangeshraut712
authored
Merge branch 'main' into fix/multi-gpu-visible-devices
2 parents 0f1192a + 94625c1 commit decf58d

39 files changed

Lines changed: 1027 additions & 189 deletions

‎.github/actions/install-sccache/action.yml‎

Lines changed: 75 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,75 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
name: Install pinned rapidsai/sccache
6+
description: Download, verify, and install a pinned rapidsai/sccache binary on Linux
7+
8+
# Regenerate hashes when bumping SCCACHE_VERSION:
9+
# for arch in x86_64 aarch64; do
10+
# curl -fsSL "https://github.com/rapidsai/sccache/releases/download/<tag>/sccache-${arch}-unknown-linux-musl.tar.gz" | sha256sum
11+
# done
12+
# Cross-check against the `.sha256` files published on the GitHub release.
13+
#
14+
# Security note (eMudhra/emSign CA incident):
15+
# - Every version bump MUST repeat the embedded-roots check on the extracted
16+
# `sccache` binary:
17+
# grep -ac 'ISRG Root X1' sccache # >0 means a root list is embedded
18+
# grep -ac 'emSign ECC Root CA - G3' sccache # must be 0
19+
# Pass criteria: either no embedded root list (both return 0; binary defers
20+
# to the OS trust store via `rustls-platform-verifier`, as verified for
21+
# `rapidsai/sccache` v0.18.0-rapids.7), or an embedded `webpki-roots` list
22+
# at 1.0.10+ where `ISRG Root X1` matches and `emSign ECC Root CA - G3`
23+
# returns 0. Match the full root name: 1.0.10 removes only that emSign root
24+
# and still ships other emSign roots.
25+
# - Do NOT switch to or bump `mozilla/sccache` to v0.19.0 (or later without
26+
# repeating the roots check): `mozilla/sccache` v0.19.0 re-embeds
27+
# `webpki-roots` 1.0.5, which still trusts `emSign ECC Root CA - G3`.
28+
29+
runs:
30+
using: composite
31+
steps:
32+
- name: Install pinned rapidsai/sccache
33+
shell: bash --noprofile --norc -xeuo pipefail {0}
34+
env:
35+
SCCACHE_VERSION: v0.18.0-rapids.7
36+
run: |
37+
if [[ "$(uname -s)" != "Linux" ]]; then
38+
echo "::error::install-sccache only supports Linux (got $(uname -s))" >&2
39+
exit 1
40+
fi
41+
42+
arch="$(uname -m)"
43+
case "${arch}" in
44+
x86_64)
45+
expected_sha256="44106a2e01aeff99767e66b1a5ae6be074306c518458b367f75287de4532974c"
46+
;;
47+
aarch64)
48+
expected_sha256="1bad10208481786f2c2e943d7cca8006eaa7e620d122859ef2d61861a733d3d1"
49+
;;
50+
*)
51+
echo "::error::Unsupported architecture for install-sccache: ${arch}" >&2
52+
exit 1
53+
;;
54+
esac
55+
56+
tarball="sccache-${arch}-unknown-linux-musl.tar.gz"
57+
url="https://github.com/rapidsai/sccache/releases/download/${SCCACHE_VERSION}/${tarball}"
58+
59+
tmp_dir="$(mktemp -d)"
60+
trap 'rm -rf "${tmp_dir}"' EXIT
61+
62+
curl -fsSL -o "${tmp_dir}/${tarball}" "${url}"
63+
(
64+
cd "${tmp_dir}"
65+
echo "${expected_sha256} ${tarball}" | sha256sum --check --strict -
66+
)
67+
68+
if [[ "$(id -u)" -eq 0 ]]; then
69+
tar -C /usr/local/bin -xvzf "${tmp_dir}/${tarball}" --wildcards --strip-components=1 -x '*/sccache'
70+
else
71+
sudo tar -C /usr/local/bin -xvzf "${tmp_dir}/${tarball}" --wildcards --strip-components=1 -x '*/sccache'
72+
fi
73+
74+
echo "SCCACHE_PATH=/usr/local/bin/sccache" >> "${GITHUB_ENV}"
75+
/usr/local/bin/sccache --version

‎.github/workflows/build-wheel.yml‎

Lines changed: 2 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -79,12 +79,9 @@ jobs:
7979
# the job level instead of linking into the diff.
8080
run: echo "::add-matcher::.github/matchers/compiler.json"
8181

82-
- name: Install latest rapidsai/sccache
82+
- name: Install rapidsai/sccache
8383
if: ${{ startsWith(inputs.host-platform, 'linux') && (env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true') }}
84-
run: |
85-
curl -fsSL "https://github.com/rapidsai/sccache/releases/latest/download/sccache-$(uname -m)-unknown-linux-musl.tar.gz" \
86-
| sudo tar -C /usr/local/bin -xvzf - --wildcards --strip-components=1 -x '*/sccache'
87-
echo "SCCACHE_PATH=/usr/local/bin/sccache" >> "$GITHUB_ENV"
84+
uses: ./.github/actions/install-sccache
8885

8986
# The runner only exposes these values to actions, not to `run:` steps or
9087
# the cibuildwheel container, so re-export them for sccache's GHA backend.

‎.github/workflows/runner-trust-probe.yml‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
#
5+
# Temporary, report-only probe of OS and bundled Node.js certificate stores
6+
# across hosted and self-hosted runner labels for the eMudhra/emSign CA
7+
# incident. Delete this workflow and ci/tools/runner_trust_probe* once all
8+
# runner labels pass and the rollout is complete.
9+
10+
name: "CI: Runner trust probe"
11+
12+
on:
13+
schedule:
14+
# Weekly on Mondays at 06:23 UTC
15+
- cron: "23 6 * * 1"
16+
workflow_dispatch:
17+
inputs:
18+
label-filter:
19+
description: "Optional regex to filter runner labels (e.g. '^(ubuntu-latest|linux-amd64-cpu8)$'; empty probes all)"
20+
required: false
21+
default: ""
22+
type: string
23+
24+
permissions:
25+
contents: read
26+
27+
defaults:
28+
run:
29+
shell: bash --noprofile --norc -xeuo pipefail {0}
30+
31+
jobs:
32+
compute-matrix:
33+
name: Compute runner label sets
34+
if: ${{ github.repository_owner == 'nvidia' }}
35+
runs-on: ubuntu-latest
36+
timeout-minutes: 5
37+
outputs:
38+
host: ${{ steps.compute.outputs.host }}
39+
linux_gpu: ${{ steps.compute.outputs.linux_gpu }}
40+
steps:
41+
- name: Checkout ${{ github.event.repository.name }}
42+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
43+
with:
44+
filter: blob:none
45+
persist-credentials: false
46+
47+
- name: Compute runner labels
48+
id: compute
49+
env:
50+
LABEL_FILTER: ${{ inputs.label-filter || '' }}
51+
run: |
52+
# Hosted and self-hosted CPU labels are listed explicitly; GPU labels
53+
# are derived from ci/test-matrix.yml.
54+
labels_json="$(yq -o json '.' ci/test-matrix.yml | jq -c --arg filter "${LABEL_FILTER}" '
55+
{
56+
host: (
57+
[
58+
"ubuntu-latest",
59+
"linux-amd64-cpu8",
60+
"linux-arm64-cpu8",
61+
"linux-amd64-cpu4",
62+
"windows-latest",
63+
"windows-2022",
64+
"windows-11-arm"
65+
] + [
66+
(.windows["pull-request"][]?, .windows["nightly"][]?)
67+
| "windows-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)"
68+
]
69+
),
70+
linux_gpu: [
71+
(.linux["pull-request"][]?, .linux["nightly"][]?)
72+
| "\(.FLAVOR // "linux")-\(.ARCH)-gpu-\(.GPU)-\(if .DRIVER == "latest" or .DRIVER == "earliest" then .DRIVER else "latest" end)-\(.GPU_COUNT)"
73+
]
74+
}
75+
| map_values(unique | if $filter != "" then map(select(test($filter))) else . end)
76+
')"
77+
78+
echo "host=$(jq -c '.host' <<< "${labels_json}")" | tee -a "${GITHUB_OUTPUT}"
79+
echo "linux_gpu=$(jq -c '.linux_gpu' <<< "${labels_json}")" | tee -a "${GITHUB_OUTPUT}"
80+
81+
probe-host:
82+
name: Probe (${{ matrix.label }})
83+
needs: compute-matrix
84+
if: ${{ needs.compute-matrix.outputs.host != '[]' }}
85+
strategy:
86+
fail-fast: false
87+
matrix:
88+
label: ${{ fromJSON(needs.compute-matrix.outputs.host) }}
89+
runs-on: ${{ matrix.label }}
90+
timeout-minutes: 15
91+
steps:
92+
- name: Checkout ${{ github.event.repository.name }}
93+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
94+
with:
95+
filter: blob:none
96+
persist-credentials: false
97+
98+
- name: Probe Linux OS trust store
99+
if: ${{ runner.os != 'Windows' }}
100+
env:
101+
TRUST_PROBE_LABEL: "${{ matrix.label == 'ubuntu-latest' && 'OS trust store (before ca-certificates upgrade)' || 'OS trust store' }}"
102+
run: ./ci/tools/runner_trust_probe.sh
103+
104+
- name: Probe Linux OS trust store after ca-certificates upgrade (ubuntu-latest)
105+
if: ${{ always() && matrix.label == 'ubuntu-latest' }}
106+
env:
107+
TRUST_PROBE_LABEL: "OS trust store (after ca-certificates upgrade)"
108+
run: |
109+
sudo apt-get update
110+
sudo apt-get install --only-upgrade -y ca-certificates
111+
./ci/tools/runner_trust_probe.sh
112+
113+
- name: Probe Windows OS trust store
114+
if: ${{ runner.os == 'Windows' }}
115+
shell: pwsh
116+
run: ./ci/tools/runner_trust_probe.ps1
117+
118+
- name: Probe Node 20 action runtime (informational)
119+
if: ${{ always() }}
120+
continue-on-error: true
121+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 (declares node20)
122+
with:
123+
script: |
124+
const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
125+
await probe({ core, heading: 'Node 20 action runtime (informational)', gating: false });
126+
127+
- name: Probe bundled Node runtime and trust store
128+
if: ${{ always() }}
129+
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
130+
with:
131+
script: |
132+
const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
133+
await probe({ core });
134+
135+
probe-linux-gpu:
136+
name: Probe (${{ matrix.label }})
137+
needs: compute-matrix
138+
if: ${{ needs.compute-matrix.outputs.linux_gpu != '[]' }}
139+
strategy:
140+
fail-fast: false
141+
matrix:
142+
label: ${{ fromJSON(needs.compute-matrix.outputs.linux_gpu) }}
143+
runs-on: ${{ matrix.label }}
144+
timeout-minutes: 15
145+
# Match test-wheel-linux.yml container setup so the probe inspects the exact
146+
# container environment that Linux GPU test jobs see.
147+
container:
148+
options: -u root --security-opt seccomp=unconfined --shm-size 16g
149+
image: ubuntu:24.04
150+
steps:
151+
- name: Checkout ${{ github.event.repository.name }}
152+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
153+
with:
154+
filter: blob:none
155+
persist-credentials: false
156+
157+
- name: Setup proxy cache
158+
uses: nv-gha-runners/setup-proxy-cache@main
159+
continue-on-error: true
160+
161+
- name: Install container CA certificates and probe tools
162+
uses: ./.github/actions/install_unix_deps
163+
with:
164+
dependencies: "ca-certificates curl openssl"
165+
dependent_exes: "curl openssl"
166+
167+
- name: Probe container OS trust store
168+
env:
169+
TRUST_PROBE_LABEL: "Container OS trust store (ubuntu:24.04)"
170+
run: ./ci/tools/runner_trust_probe.sh
171+
172+
- name: Probe Node 20 action runtime (informational)
173+
if: ${{ always() }}
174+
continue-on-error: true
175+
uses: actions/github-script@60a0d83039c74a4aee543508d2ffcb1c3799cdea # v7.0.1 (declares node20)
176+
with:
177+
script: |
178+
const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
179+
await probe({ core, heading: 'Node 20 action runtime (informational)', gating: false });
180+
181+
- name: Probe bundled Node runtime and trust store
182+
if: ${{ always() }}
183+
uses: actions/github-script@3a2844b7e9c422d3c10d287c895573f7108da1b3 # v9.0.0
184+
with:
185+
script: |
186+
const probe = require(`${process.env.GITHUB_WORKSPACE}/ci/tools/runner_trust_probe_node.js`);
187+
await probe({ core });

‎.github/workflows/test-sdist-linux.yml‎

Lines changed: 3 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -87,15 +87,10 @@ jobs:
8787
pathfinder_uri="file://$(realpath "${pathfinder_wheels[0]}")"
8888
printf 'cuda-pathfinder @ %s\n' "${pathfinder_uri}" | tee wheel-constraints/cuda-bindings.txt
8989
90-
# Cython packages need CTK + sccache.
91-
# The env vars ACTIONS_CACHE_SERVICE_V2, ACTIONS_RESULTS_URL, and ACTIONS_RUNTIME_TOKEN
92-
# are exposed by this action.
93-
- name: Enable sccache
90+
# Cython packages need CTK + sccache (local disk cache only in this job).
91+
- name: Install rapidsai/sccache
9492
if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }}
95-
uses: mozilla-actions/sccache-action@fc920bf0ec8de6ee65d409111f7ec508035751ba # 0.0.11
96-
with:
97-
version: v0.18.0
98-
disable_annotations: 'true'
93+
uses: ./.github/actions/install-sccache
9994

10095
- name: Setup proxy cache
10196
if: ${{ env.BUILD_BINDINGS == 'true' || env.BUILD_CORE == 'true' }}

‎.github/workflows/test-wheel-linux.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -146,12 +146,22 @@ jobs:
146146
dependencies: "jq wget libgl1 libegl1 g++ util-linux yq"
147147
dependent_exes: "jq wget yq"
148148

149+
- name: Checkout driver install scripts (workflow revision)
150+
if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
151+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
152+
with:
153+
ref: ${{ github.sha }}
154+
path: .driver-tools
155+
sparse-checkout: ci/tools
156+
persist-credentials: false
157+
149158
- name: Install GPU driver
150159
if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
151160
env:
152161
DRIVER: ${{ matrix.DRIVER }}
162+
DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }}
153163
GPU_TYPE: ${{ matrix.GPU }}
154-
run: ./ci/tools/install_gpu_driver.sh
164+
run: ./.driver-tools/ci/tools/install_gpu_driver.sh
155165

156166
- name: Ensure GPU is working
157167
run: nvidia-smi

‎.github/workflows/test-wheel-windows.yml‎

Lines changed: 11 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -119,13 +119,23 @@ jobs:
119119
with:
120120
enable-apt: true
121121

122+
- name: Checkout driver install scripts (workflow revision)
123+
if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
124+
uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
125+
with:
126+
ref: ${{ github.sha }}
127+
path: .driver-tools
128+
sparse-checkout: ci/tools
129+
persist-credentials: false
130+
122131
- name: Install GPU driver
123132
if: ${{ matrix.DRIVER != 'latest' && matrix.DRIVER != 'earliest' }}
124133
env:
125134
DRIVER: ${{ matrix.DRIVER }}
135+
DRIVER_SHA256: ${{ matrix.DRIVER_SHA256 }}
126136
GPU_TYPE: ${{ matrix.GPU }}
127137
run: |
128-
ci/tools/install_gpu_driver.ps1
138+
.driver-tools/ci/tools/install_gpu_driver.ps1
129139
130140
- name: Configure driver mode
131141
env:

‎.pre-commit-config.yaml‎

Lines changed: 12 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -219,8 +219,18 @@ repos:
219219
rev: "35dbc940c2bd4499ebf39aba00ad717e05cec9b5" # frozen: v0.22.0
220220
hooks:
221221
- id: cython-lint
222-
args: [--no-pycodestyle]
223-
exclude: ^cuda_bindings_(?:12|13)/
222+
# Some rules (e.g. "inline" on a function declared in a .pxd) match
223+
# declarations in one file against implementations in another, and the
224+
# linter only remembers that within a single invocation. Pre-commit
225+
# splits filenames across parallel processes and only passes changed
226+
# files, so a .pyx edited without its .pxd would escape the check.
227+
# Instead, whenever any Cython file under these trees changes, lint
228+
# all of them in one process. Generated cuda_bindings_* and
229+
# third-party submodules are not listed.
230+
args: [--no-pycodestyle, cuda_core/cuda]
231+
files: ^cuda_core/cuda
232+
pass_filenames: false
233+
require_serial: true
224234
additional_dependencies:
225235
- Cython==3.2.9
226236

0 commit comments

Comments
 (0)