|
11 | 11 | import contextlib |
12 | 12 | import functools |
13 | 13 | import glob |
| 14 | +import hashlib |
14 | 15 | import os |
15 | 16 | import shutil |
16 | 17 | import sys |
17 | 18 | import sysconfig |
18 | 19 | import tempfile |
| 20 | +import uuid |
19 | 21 | from pathlib import Path |
20 | 22 | from warnings import warn |
21 | 23 |
|
|
31 | 33 | get_requires_for_build_editable = _build_meta.get_requires_for_build_editable |
32 | 34 |
|
33 | 35 | # Note: There is no support guarantee for environment variables like |
34 | | -# CUDA_PYTHON_TOOLCHAIN, etc. They may be removed or changed in the future. |
| 36 | +# CUDA_PYTHON_TOOLCHAIN, CUDA_PYTHON_CYTHON_CACHE_DIR, etc. They may be |
| 37 | +# removed or changed in the future. |
35 | 38 |
|
36 | 39 | # Populated by _build_cuda_bindings(); consumed by setup.py. |
37 | 40 | _extensions = None |
@@ -86,13 +89,14 @@ def _get_cuda_path() -> str: |
86 | 89 | # ----------------------------------------------------------------------- |
87 | 90 | # Toolchain selection |
88 | 91 | # |
89 | | -# The helpers below (down to the end-of-shared-block marker) are duplicated |
90 | | -# verbatim in cuda_core/build_hooks.py. Keep them in sync. Only the |
91 | | -# per-package _resolve_toolchain() flag assembly that follows is package- |
92 | | -# specific (it differs because the two packages use different C++ standards |
93 | | -# and opt levels). |
94 | | - |
95 | | -# --- begin shared toolchain helpers (keep in sync) --- |
| 92 | +# There is one shared helper block below, duplicated verbatim in |
| 93 | +# cuda_core/build_hooks.py (keep it in sync; enforced by |
| 94 | +# toolshed/check_build_hooks_sync.py). It contains the toolchain helpers and |
| 95 | +# the Cython cache helpers. Only the per-package _resolve_toolchain() flag |
| 96 | +# assembly that follows the shared block is package-specific (it differs |
| 97 | +# because the two packages use different C++ standards and opt levels). |
| 98 | + |
| 99 | +# --- begin shared build helpers (keep in sync) --- |
96 | 100 | _TOOLCHAINS_LINUX = ("gnu", "llvm") |
97 | 101 | _TOOLCHAINS_WINDOWS = ("msvc",) |
98 | 102 | _TOOLCHAIN_COMPILERS = { |
@@ -157,7 +161,132 @@ def _check_toolchain_available(name): |
157 | 161 | ) |
158 | 162 |
|
159 | 163 |
|
160 | | -# --- end shared toolchain helpers --- |
| 164 | +# === Cython generated-source cache (opt-in via CUDA_PYTHON_CYTHON_CACHE_DIR) === |
| 165 | +# Workaround for Cython issue #7532: Cython's native cache fingerprint omits |
| 166 | +# `compiler_directives`, so builds with different directives (e.g. linetrace |
| 167 | +# for coverage) could reuse stale generated C/C++ output. This helper |
| 168 | +# namespaces the Cython cache by package and a digest of output-affecting |
| 169 | +# build configuration so distinct configurations get distinct caches. |
| 170 | +# |
| 171 | +# Removal: once cython/cython#7532 is resolved in a released Cython version |
| 172 | +# and cuda-python's minimum Cython version includes the fix, this helper |
| 173 | +# and its workaround-specific tests can be deleted; cythonize() can then be |
| 174 | +# called with `cache=<root>` (or `cache=True`) without per-config namespacing. |
| 175 | +# See https://github.com/cython/cython/issues/7532 |
| 176 | +def _cython_cache_path( |
| 177 | + package, |
| 178 | + *, |
| 179 | + compiler_directives=None, |
| 180 | + compile_time_env=None, |
| 181 | + language_level=None, |
| 182 | + cplus=None, |
| 183 | + debug=False, |
| 184 | + cuda_major=None, |
| 185 | +): |
| 186 | + """Return a per-configuration Cython cache directory, or None to disable caching. |
| 187 | +
|
| 188 | + Returns None when CUDA_PYTHON_CYTHON_CACHE_DIR is unset, so cythonize() |
| 189 | + is called without ``cache=`` and existing workflows are unchanged. |
| 190 | + """ |
| 191 | + cache_root = os.environ.get("CUDA_PYTHON_CYTHON_CACHE_DIR") |
| 192 | + if not cache_root: |
| 193 | + return None |
| 194 | + if sys.platform == "win32": |
| 195 | + warn( |
| 196 | + "CUDA_PYTHON_CYTHON_CACHE_DIR is set but Cython caching via symlinks " |
| 197 | + "is not supported on Windows; caching will be disabled.", |
| 198 | + stacklevel=2, |
| 199 | + ) |
| 200 | + return None |
| 201 | + |
| 202 | + h = hashlib.sha256() |
| 203 | + h.update(package.encode("utf-8")) |
| 204 | + # The Python version running cythonize affects generated C code |
| 205 | + # (e.g. CYTHON_COMPRESS_STRINGS: zstd on 3.14, zlib on 3.12/3.13). |
| 206 | + h.update(f"python={sys.version_info.major}.{sys.version_info.minor}".encode()) |
| 207 | + |
| 208 | + def _update(name, value): |
| 209 | + h.update(name.encode("utf-8")) |
| 210 | + h.update(repr(value).encode("utf-8")) |
| 211 | + |
| 212 | + # compiler_directives are not in Cython's native fingerprint (#7532). |
| 213 | + if compiler_directives: |
| 214 | + for key in sorted(compiler_directives): |
| 215 | + _update(f"directive:{key}", compiler_directives[key]) |
| 216 | + # compile_time_env, language_level, and cplus are already in Cython's |
| 217 | + # fingerprint, but we include them so the namespace stays correct even |
| 218 | + # if Cython's fingerprint logic changes. |
| 219 | + if compile_time_env: |
| 220 | + for key in sorted(compile_time_env): |
| 221 | + _update(f"compile_time_env:{key}", compile_time_env[key]) |
| 222 | + if language_level is not None: |
| 223 | + _update("language_level", language_level) |
| 224 | + if cplus is not None: |
| 225 | + _update("cplus", cplus) |
| 226 | + # debug toggles gdb_debug in cythonize(), which affects generated code. |
| 227 | + _update("debug", debug) |
| 228 | + if cuda_major is not None: |
| 229 | + _update("cuda_major", cuda_major) |
| 230 | + |
| 231 | + return os.path.join(cache_root, f"{package}-{h.hexdigest()[:16]}") |
| 232 | + |
| 233 | + |
| 234 | +@contextlib.contextmanager |
| 235 | +def _stable_cython_alias(target: Path, alias: Path): |
| 236 | + """Atomically create a stable directory symlink alias for a Cython include tree. |
| 237 | +
|
| 238 | + Cython's cache fingerprint includes the absolute path of each resolved |
| 239 | + .pxd dependency (via ``file_hash()``). PEP 517 build environments install |
| 240 | + dependencies under randomized temporary prefixes, making those paths |
| 241 | + unstable across runs. This context manager creates a fixed, worktree- |
| 242 | + relative symlink so Cython sees a stable lexical path. |
| 243 | +
|
| 244 | + The symlink is created in the *package directory* (the directory containing |
| 245 | + this build_hooks.py), not in the cwd, to keep aliases package-local and |
| 246 | + avoid cross-package races. |
| 247 | +
|
| 248 | + alias must not already exist as a real file or directory; if it is a |
| 249 | + symlink (including a dangling one) it is atomically replaced. |
| 250 | +
|
| 251 | + On exit the alias is removed only if it still points at ``target`` (a |
| 252 | + racing replacement will not be deleted). |
| 253 | +
|
| 254 | + POSIX only: directory symlinks require no elevated privileges on Linux. |
| 255 | + """ |
| 256 | + # Resolve the *parent* directory (must exist), then append the name. |
| 257 | + # We deliberately do not follow a symlink that may already sit at alias. |
| 258 | + if not alias.is_absolute(): |
| 259 | + alias = Path(__file__).parent / alias |
| 260 | + alias = alias.parent.resolve() / alias.name |
| 261 | + target = target.resolve() |
| 262 | + |
| 263 | + if alias.exists() and not alias.is_symlink(): |
| 264 | + raise RuntimeError( |
| 265 | + f"Cannot create Cython include alias at {alias}: a real file or directory already exists there." |
| 266 | + ) |
| 267 | + |
| 268 | + tmp_alias = alias.with_name(f".{alias.name}.{uuid.uuid4().hex[:8]}.tmp") |
| 269 | + try: |
| 270 | + os.symlink(target, tmp_alias, target_is_directory=True) |
| 271 | + try: |
| 272 | + os.replace(tmp_alias, alias) |
| 273 | + except BaseException: |
| 274 | + tmp_alias.unlink(missing_ok=True) |
| 275 | + raise |
| 276 | + rel = os.path.relpath(alias, start=Path.cwd()) |
| 277 | + yield rel |
| 278 | + finally: |
| 279 | + tmp_alias.unlink(missing_ok=True) |
| 280 | + # Only remove the alias we created; leave it alone if something else |
| 281 | + # has already replaced it (readlink will differ). |
| 282 | + try: |
| 283 | + if alias.is_symlink() and Path(os.readlink(alias)).resolve() == target: |
| 284 | + alias.unlink() |
| 285 | + except OSError: |
| 286 | + pass |
| 287 | + |
| 288 | + |
| 289 | +# --- end shared build helpers --- |
161 | 290 |
|
162 | 291 |
|
163 | 292 | def _resolve_toolchain(debug=False, compile_for_coverage=False): |
@@ -313,6 +442,7 @@ def _build_cuda_bindings(debug=False): |
313 | 442 | All CUDA-dependent logic (cythonization) is deferred to this function so |
314 | 443 | that metadata queries do not require a CUDA toolkit installation. |
315 | 444 | """ |
| 445 | + import Cython |
316 | 446 | from Cython.Build import cythonize |
317 | 447 | from Cython.Compiler import Options as _CythonOptions |
318 | 448 |
|
@@ -403,14 +533,37 @@ def get_static_libraries(f): |
403 | 533 | # build, so a stale .so from a previous toolchain is never packaged. |
404 | 534 | _check_build_toolchain(toolchain) |
405 | 535 |
|
406 | | - _extensions = cythonize( |
407 | | - extensions, |
408 | | - nthreads=nthreads, |
409 | | - build_dir="." if compile_for_coverage else "build/cython", |
| 536 | + cache_path = _cython_cache_path( |
| 537 | + "cuda-bindings", |
410 | 538 | compiler_directives=cython_directives, |
411 | | - **extra_cythonize_kwargs, |
| 539 | + language_level=3, |
| 540 | + cplus=True, |
| 541 | + debug=debug, |
412 | 542 | ) |
413 | 543 |
|
| 544 | + def _do_cythonize(cython_include_path): |
| 545 | + global _extensions |
| 546 | + _extensions = cythonize( |
| 547 | + extensions, |
| 548 | + nthreads=nthreads, |
| 549 | + build_dir="." if compile_for_coverage else "build/cython", |
| 550 | + compiler_directives=cython_directives, |
| 551 | + include_path=cython_include_path, |
| 552 | + cache=cache_path, |
| 553 | + **extra_cythonize_kwargs, |
| 554 | + ) |
| 555 | + |
| 556 | + if cache_path is not None: |
| 557 | + # Alias Cython's bundled .pxd declarations under a stable worktree-relative |
| 558 | + # path so Cython's cache fingerprint sees the same path on every run |
| 559 | + # despite PEP 517 build environments landing under randomized temp prefixes. |
| 560 | + stdlib_target = Path(Cython.__file__).parent / "Includes" |
| 561 | + stdlib_alias = Path(__file__).parent / ".cython-stdlib" |
| 562 | + with _stable_cython_alias(stdlib_target, stdlib_alias) as rel_stdlib: |
| 563 | + _do_cythonize([".", rel_stdlib]) |
| 564 | + else: |
| 565 | + _do_cythonize(["."]) |
| 566 | + |
414 | 567 |
|
415 | 568 | # ----------------------------------------------------------------------- |
416 | 569 | # PEP 517 build hooks |
|
0 commit comments