Skip to content

Commit ae4ee3b

Browse files
committed
Copy qa/helpers/download_from_kitpicks.py from PR NVIDIA#162 to make it more accessible sooner.
1 parent 78fd567 commit ae4ee3b

1 file changed

Lines changed: 310 additions & 0 deletions

File tree

Lines changed: 310 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,310 @@
1+
#!/usr/bin/env python3
2+
3+
# SPDX-FileCopyrightText: Copyright (c) 2025 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
4+
# SPDX-License-Identifier: LicenseRef-NVIDIA-SOFTWARE-LICENSE
5+
6+
"""
7+
Download CUDA kitpick local installers for the current platform (or a chosen one).
8+
9+
USAGE EXAMPLES
10+
--------------
11+
1) Full URL (auto-arch detection; downloads only the matching file)
12+
download_from_kitpicks.py https://cuda-repo.nvidia.com/release-candidates/kitpicks/cuda-r13-1/13.1.0/028/local_installers/
13+
14+
2) Full URL, override architecture
15+
download_from_kitpicks.py https://.../local_installers/ --arch=linux
16+
17+
3) Full URL, download all three files
18+
download_from_kitpicks.py https://.../local_installers/ --all
19+
20+
4) Shorthand (VERSION KITPICK), auto-arch detection
21+
download_from_kitpicks.py 13.1 028
22+
23+
5) Shorthand with explicit arch
24+
download_from_kitpicks.py 13.1 028 --arch=linux-sbsa
25+
26+
6) Shorthand, download all
27+
download_from_kitpicks.py 13.1 028 --all
28+
"""
29+
30+
import argparse
31+
import os
32+
import platform
33+
import re
34+
import subprocess
35+
import sys
36+
import urllib.request
37+
from html.parser import HTMLParser
38+
from urllib.parse import urljoin
39+
40+
BASE_URL = "https://cuda-repo.nvidia.com/release-candidates/kitpicks"
41+
42+
ARCH_ALIASES = {
43+
# Windows
44+
"windows": "win-64",
45+
"win-64": "win-64",
46+
# Linux x86_64
47+
"linux": "linux-64",
48+
"linux-64": "linux-64",
49+
# Linux aarch64
50+
"linux-sbsa": "linux-aarch64",
51+
"linux-aarch64": "linux-aarch64",
52+
}
53+
54+
55+
def debug(msg: str) -> None:
56+
print(msg, flush=True)
57+
58+
59+
def extract_kitpick_from_url(url: str) -> str | None:
60+
"""
61+
Extract the 3-digit kitpick number from a full URL (e.g., '028').
62+
"""
63+
m = re.search(r"/(\d{3})/local_installers/?$", url.rstrip("/"))
64+
if m:
65+
return m.group(1)
66+
# fallback: last 3 consecutive digits in the URL
67+
m2 = re.findall(r"(\d{3})(?!\d)", url)
68+
return m2[-1] if m2 else None
69+
70+
71+
def normalize_version_tag(version_str: str) -> tuple[str, str]:
72+
"""
73+
Given '13.1' or '13.1.0', return (r_tag, semver) where:
74+
r_tag = 'cuda-r13-1'
75+
semver = '13.1.0'
76+
"""
77+
m = re.fullmatch(r"\s*(\d+)\.(\d+)(?:\.(\d+))?\s*", version_str)
78+
if not m:
79+
raise ValueError(f"Invalid VERSION '{version_str}'. Expected like '13.1' or '13.1.0'.")
80+
maj, min_, patch = m.group(1), m.group(2), m.group(3) or "0"
81+
r_tag = f"cuda-r{maj}-{min_}"
82+
semver = f"{maj}.{min_}.{patch}"
83+
return r_tag, semver
84+
85+
86+
def build_full_url(version_str: str, kitpick: str) -> str:
87+
r_tag, semver = normalize_version_tag(version_str)
88+
kitpick3 = kitpick.strip()
89+
if not re.fullmatch(r"\d{3}", kitpick3):
90+
raise ValueError(f"Invalid KITPICK '{kitpick}'. Expected a 3-digit value like '028'.")
91+
url = f"{BASE_URL}/{r_tag}/{semver}/{kitpick3}/local_installers/"
92+
return url
93+
94+
95+
class LinkExtractor(HTMLParser):
96+
def __init__(self) -> None:
97+
super().__init__()
98+
self.links: list[str] = []
99+
100+
def handle_starttag(self, tag, attrs):
101+
if tag.lower() == "a":
102+
href = dict(attrs).get("href")
103+
if href and (href.endswith(".run") or href.endswith(".exe")):
104+
self.links.append(href)
105+
106+
107+
def get_file_links(url: str) -> list[tuple[str, str]]:
108+
"""
109+
Scrape the directory listing page to find .run and .exe files.
110+
Returns list of (filename, absolute_url).
111+
"""
112+
with urllib.request.urlopen(url) as response: # noqa: S310
113+
html = response.read().decode("utf-8")
114+
115+
parser = LinkExtractor()
116+
parser.feed(html)
117+
return [(href, urljoin(url, href)) for href in parser.links]
118+
119+
120+
def generate_new_filename(original_filename: str, kitpick: str) -> str:
121+
"""
122+
Add _kitpickXYZ before extension.
123+
"""
124+
name, ext = os.path.splitext(original_filename)
125+
return f"{name}_kitpick{kitpick}{ext}"
126+
127+
128+
def download_file(file_url: str, original_filename: str, new_filename: str) -> bool:
129+
"""
130+
Download file using wget with timestamp preservation.
131+
"""
132+
try:
133+
cmd = ["wget", "--timestamping", "--output-document", new_filename, file_url]
134+
debug(f"Downloading: {original_filename} -> {new_filename}")
135+
result = subprocess.run(cmd, capture_output=True, text=True) # noqa: S603
136+
if result.returncode == 0:
137+
print(f"✓ {new_filename}")
138+
os.chmod(new_filename, 0o555) # noqa: S103
139+
return True
140+
print(f"✗ Failed: {original_filename}\n{result.stderr}")
141+
return False
142+
except FileNotFoundError:
143+
print("Error: wget not found. Please install wget first.")
144+
return False
145+
146+
147+
# ---------------------------------------------------------------------------
148+
# Architecture handling
149+
# ---------------------------------------------------------------------------
150+
151+
152+
def auto_detect_arch() -> str:
153+
sysname = platform.system().lower()
154+
machine = platform.machine().lower()
155+
156+
if sysname == "windows":
157+
return "win-64"
158+
159+
if sysname == "linux":
160+
if machine in ("x86_64", "amd64"):
161+
return "linux-64"
162+
if machine in ("aarch64", "arm64"):
163+
return "linux-aarch64"
164+
165+
raise RuntimeError(f"Unsupported platform for auto-detection: {sysname} / {machine}")
166+
167+
168+
def normalize_arch(name: str) -> str:
169+
key = name.strip().lower().replace("_", "-")
170+
canonical = ARCH_ALIASES.get(key)
171+
if canonical is None:
172+
raise ValueError(f"Unknown --arch '{name}'. Supported aliases: {', '.join(ARCH_ALIASES.keys())}")
173+
return canonical
174+
175+
176+
def select_links_for_arch(file_links: list[tuple[str, str]], arch: str) -> list[tuple[str, str]]:
177+
"""
178+
Pick only the links that match the requested arch, based on filename.
179+
We rely on naming conventions:
180+
- Windows: contains 'windows' and ends with .exe
181+
- linux-64: contains 'linux' but NOT 'sbsa', ends with .run
182+
- linux-aarch64: contains 'linux_sbsa' or 'sbsa', ends with .run
183+
"""
184+
out: list[tuple[str, str]] = []
185+
186+
for fname, href in file_links:
187+
lower = fname.lower()
188+
189+
if arch == "win-64":
190+
if lower.endswith(".exe") and "windows" in lower:
191+
out.append((fname, href))
192+
193+
elif arch == "linux-64":
194+
# include 'linux' but exclude sbsa variants
195+
if lower.endswith(".run") and "linux" in lower and "sbsa" not in lower:
196+
out.append((fname, href))
197+
198+
elif arch == "linux-aarch64": # noqa: SIM102
199+
# include sbsa variants
200+
if lower.endswith(".run") and ("linux_sbsa" in lower or "sbsa" in lower):
201+
out.append((fname, href))
202+
203+
return out
204+
205+
206+
# ---------------------------------------------------------------------------
207+
# CLI
208+
# ---------------------------------------------------------------------------
209+
210+
211+
def parse_args(argv: list[str]) -> argparse.Namespace:
212+
p = argparse.ArgumentParser(
213+
description="Download CUDA kitpick local installers for the current platform or a chosen one."
214+
)
215+
216+
# Two invocation modes:
217+
# 1) download_from_kitpicks.py <FULL_URL>
218+
# 2) download_from_kitpicks.py <VERSION> <KITPICK>
219+
p.add_argument("arg1", help="Either FULL_URL or VERSION (e.g., 13.1)")
220+
p.add_argument("arg2", nargs="?", help="KITPICK (e.g., 028) if using VERSION form")
221+
222+
grp = p.add_mutually_exclusive_group()
223+
grp.add_argument("--arch", help="Target architecture (see ARCH ALIASES in script header)")
224+
grp.add_argument("--all", action="store_true", help="Download all three installers")
225+
226+
return p.parse_args(argv)
227+
228+
229+
def is_url(s: str) -> bool:
230+
return s.startswith("http://") or s.startswith("https://")
231+
232+
233+
def main(argv: list[str]) -> int:
234+
args = parse_args(argv)
235+
236+
# Determine full URL & kitpick suffix
237+
if args.arg2: # VERSION + KITPICK
238+
full_url = build_full_url(args.arg1, args.arg2)
239+
kitpick_suffix = args.arg2
240+
debug(f"Constructed URL: {full_url}")
241+
else: # FULL URL
242+
if not is_url(args.arg1):
243+
print(
244+
"ERROR: With one positional argument, it must be a FULL URL.\n"
245+
" Or use two args: VERSION KITPICK (e.g., '13.1 028')."
246+
)
247+
return 2
248+
full_url = args.arg1 if args.arg1.endswith("/") else args.arg1 + "/"
249+
kitpick_suffix = extract_kitpick_from_url(full_url) or "000"
250+
debug(f"Using provided URL: {full_url}")
251+
252+
# Resolve architecture target(s)
253+
targets: list[str]
254+
if args.all:
255+
targets = ["win-64", "linux-64", "linux-aarch64"]
256+
debug("Mode: download all architectures")
257+
else:
258+
if args.arch:
259+
arch = normalize_arch(args.arch)
260+
debug(f"Mode: explicit arch = {arch}")
261+
else:
262+
arch = auto_detect_arch()
263+
debug(f"Mode: auto-detected arch = {arch}")
264+
targets = [arch]
265+
266+
# Fetch directory listing
267+
debug("Fetching directory listing...")
268+
try:
269+
file_links = get_file_links(full_url)
270+
except Exception as e:
271+
print(f"Error fetching directory listing: {e}")
272+
return 1
273+
274+
if not file_links:
275+
print("No .run or .exe files found in the directory listing.")
276+
return 1
277+
278+
# Select links by arch
279+
selected: list[tuple[str, str]] = []
280+
for arch in targets:
281+
chosen = select_links_for_arch(file_links, arch)
282+
if not chosen:
283+
debug(f"Warning: no matching files found for arch '{arch}'.")
284+
selected.extend(chosen)
285+
286+
if not selected:
287+
print("No matching files to download after applying architecture filters.")
288+
return 1
289+
290+
# Show plan
291+
print("\nFull URL:", full_url)
292+
print(f"Kitpick: {kitpick_suffix}")
293+
print(f"Selected {len(selected)} file(s):")
294+
for fname, _ in selected:
295+
print(f" - {fname} -> {generate_new_filename(fname, kitpick_suffix)}")
296+
297+
# Download
298+
print("\nStarting downloads...")
299+
ok = 0
300+
for fname, href in selected:
301+
new_name = generate_new_filename(fname, kitpick_suffix)
302+
if download_file(href, fname, new_name):
303+
ok += 1
304+
305+
print(f"\nDone: {ok}/{len(selected)} files downloaded successfully.")
306+
return 0
307+
308+
309+
if __name__ == "__main__":
310+
raise SystemExit(main(sys.argv[1:]))

0 commit comments

Comments
 (0)