Skip to content

Commit 576ef68

Browse files
authored
build: make sccache's preprocessor cache work in isolated wheel builds (#3139)
* build: drop the empty venv include dir from build_ext include_dirs In a virtualenv, setuptools' build_ext appends <sys.exec_prefix>/include. Under PEP 517 build isolation that is a fresh /tmp/build-env-<random> per build, so every compile command differs and sccache's preprocessor cache, which hashes the -I flags, never hits. Drop the directory when it holds no files; it contributes no headers then, so compilation is unchanged. The helper lives in _build_shared.py and is applied in both packages' build_ext.finalize_options, with tests in the shared build_shared mixins. * build: keep build-env library dirs out of cuda.bindings Extensions Cython copies each Extension's library_dirs into the metadata comment of the generated .cpp. Under PEP 517 build isolation, platlib and <sys.prefix>/lib are /tmp/build-env-<random> paths, so the generated sources differ per build, and sccache's preprocessor cache, which hashes the raw source, misses even when the object cache hits. Drop platlib: nothing links from site-packages itself. Add <sys.prefix>/lib (where a conda environment can provide libcudart_static.a) in build_ext.finalize_options instead, so the link line still has it but the generated sources no longer do. * ci: enable sccache's GHA preprocessor cache in Linux wheel builds rapidsai/sccache can store preprocessor-cache entries in the GHA cache alongside object entries. A preprocessor hit skips running the preprocessor, which the object cache alone always has to do to compute its key. The two preceding commits make the compile commands and the generated sources stable across isolated builds, so these entries can now hit. * build: follow symlinked dirs when checking the venv include dir drop_empty_venv_include_dir() used os.walk() without followlinks, so a <venv>/include whose only content is symlinked directories (for example linked header trees) looked empty. The helper then dropped the -I flag even though headers were reachable through it, breaking compiles that need them. Walk with followlinks=True so headers behind symlinked directories count as files, and add regression tests for a symlinked directory that holds a header and for one that is empty. * build: do not add <sys.prefix>/lib to library_dirs twice build_ext.finalize_options unconditionally appended <sys.prefix>/lib. Skip the append when the directory is already in library_dirs, so a repeated finalize_options call or a pre-populated list does not produce a duplicate -L flag.
1 parent 6bc28b0 commit 576ef68

8 files changed

Lines changed: 132 additions & 4 deletions

File tree

‎.github/workflows/build-wheel.yml‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -246,6 +246,7 @@ jobs:
246246
CC="/host/${{ env.SCCACHE_PATH }} cc"
247247
CXX="/host/${{ env.SCCACHE_PATH }} c++"
248248
SCCACHE_GHA_ENABLED=true
249+
SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true
249250
ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
250251
ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
251252
ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
@@ -343,6 +344,7 @@ jobs:
343344
CC="/host/${{ env.SCCACHE_PATH }} cc"
344345
CXX="/host/${{ env.SCCACHE_PATH }} c++"
345346
SCCACHE_GHA_ENABLED=true
347+
SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true
346348
ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
347349
ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
348350
ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}
@@ -620,6 +622,7 @@ jobs:
620622
CC="/host/${{ env.SCCACHE_PATH }} cc"
621623
CXX="/host/${{ env.SCCACHE_PATH }} c++"
622624
SCCACHE_GHA_ENABLED=true
625+
SCCACHE_PREPROCESSOR_CACHE_GHA_ENABLED=true
623626
ACTIONS_RUNTIME_TOKEN=${{ env.ACTIONS_RUNTIME_TOKEN }}
624627
ACTIONS_RESULTS_URL=${{ env.ACTIONS_RESULTS_URL }}
625628
ACTIONS_CACHE_SERVICE_V2=${{ env.ACTIONS_CACHE_SERVICE_V2 }}

‎cuda_bindings/_build_shared.py‎

Lines changed: 28 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -18,8 +18,9 @@
1818
``tweak`` hook for flags a single package needs.
1919
2020
Besides the toolchain, this module owns the CUDA path lookup and the machinery that both backends use
21-
around cythonize and build_ext: the opt-in Cython generated-source cache and
22-
the build stamps that force a rebuild when the build configuration changed.
21+
around cythonize and build_ext: the opt-in Cython generated-source cache, the
22+
include-dir cleanup that keeps compile commands stable across isolated builds,
23+
and the build stamps that force a rebuild when the build configuration changed.
2324
2425
Note: There is no support guarantee for environment variables like
2526
CUDA_PYTHON_TOOLCHAIN. They may be removed or changed in the future.
@@ -429,6 +430,31 @@ def _stable_cython_alias(target: Path, alias: Path):
429430
pass
430431

431432

433+
# -----------------------------------------------------------------------
434+
# build_ext include dirs
435+
436+
437+
def drop_empty_venv_include_dir(include_dirs):
438+
"""Return ``include_dirs`` without the virtualenv include dir if it holds no files.
439+
440+
In a virtualenv, setuptools' build_ext appends ``<sys.exec_prefix>/include``.
441+
Under PEP 517 build isolation that prefix is a fresh temporary directory
442+
(``/tmp/build-env-<random>``), so the ``-I`` flag differs on every build,
443+
and compiler caches that key on the command line (sccache's preprocessor
444+
cache) never hit. A directory without files contributes no headers, so
445+
dropping it leaves the compilation unchanged.
446+
447+
Symlinked subdirectories are followed: headers reachable through them
448+
still count as files.
449+
"""
450+
if sys.exec_prefix == sys.base_exec_prefix:
451+
return include_dirs
452+
venv_include = os.path.join(sys.exec_prefix, "include")
453+
if any(files for _, _, files in os.walk(venv_include, followlinks=True)):
454+
return include_dirs
455+
return [d for d in include_dirs if d != venv_include]
456+
457+
432458
# -----------------------------------------------------------------------
433459
# Build stamps
434460
#

‎cuda_bindings/build_hooks.py‎

Lines changed: 4 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -284,12 +284,14 @@ def _build_cuda_bindings(debug=False):
284284
include_dirs = [
285285
os.path.dirname(sysconfig.get_path("include")),
286286
] + include_path_list
287-
library_dirs = [sysconfig.get_path("platlib"), os.path.join(os.sys.prefix, "lib")]
287+
# Keep build-environment paths out of the Extensions: Cython copies their
288+
# library_dirs into the generated sources. setup.py's build_ext adds
289+
# <sys.prefix>/lib instead.
288290
if sys.platform == "win32":
289291
cudalib_subdirs = [r"lib\arm64"] if sysconfig.get_platform() == "win-arm64" else [r"lib\x64"]
290292
else:
291293
cudalib_subdirs = ["lib64", "lib"]
292-
library_dirs.extend(os.path.join(cuda_path, subdir) for subdir in cudalib_subdirs)
294+
library_dirs = [os.path.join(cuda_path, subdir) for subdir in cudalib_subdirs]
293295

294296
# Rename architecture-specific files
295297
dst_files = _rename_architecture_specific_files()

‎cuda_bindings/setup.py‎

Lines changed: 17 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3,13 +3,16 @@
33
# SPDX-License-Identifier: Apache-2.0
44

55
import os
6+
import sys
67
from warnings import warn
78

89
import build_hooks
910
from setuptools import setup
1011
from setuptools.command.build_ext import build_ext as _build_ext
1112
from setuptools.command.build_py import build_py as _build_py
1213

14+
from _build_shared import drop_empty_venv_include_dir
15+
1316
if os.environ.get("PARALLEL_LEVEL") is not None:
1417
warn(
1518
"Environment variable PARALLEL_LEVEL is deprecated. Use CUDA_PYTHON_PARALLEL_LEVEL instead",
@@ -24,6 +27,20 @@
2427

2528

2629
class build_ext(_build_ext):
30+
def finalize_options(self):
31+
super().finalize_options()
32+
# A per-build venv path on every -I line defeats sccache's
33+
# preprocessor cache; see _build_shared.drop_empty_venv_include_dir().
34+
self.include_dirs = drop_empty_venv_include_dir(self.include_dirs)
35+
# A conda environment can provide libcudart_static.a in <prefix>/lib.
36+
# Added here rather than to the Extensions: Cython copies Extension
37+
# library_dirs into the generated sources, and under build isolation
38+
# sys.prefix is a fresh /tmp/build-env-<random> per build, which
39+
# defeats sccache's preprocessor cache.
40+
prefix_lib = os.path.join(sys.prefix, "lib")
41+
if prefix_lib not in self.library_dirs:
42+
self.library_dirs.append(prefix_lib)
43+
2744
def build_extensions(self):
2845
if nthreads > 0:
2946
self.parallel = nthreads

‎cuda_bindings/tests/test_build_hooks.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -125,6 +125,7 @@ def test_record_writes_stamp(self, stamp):
125125
BuildKeyStampMixin,
126126
CheckToolchainAvailableMixin,
127127
DistutilsLinkerIntegrationMixin,
128+
DropEmptyVenvIncludeDirMixin,
128129
ForceBuildExtReexportMixin,
129130
ResolveToolchainMixin,
130131
WithCompilerMixin,
@@ -161,6 +162,10 @@ class TestBuildKeyStamp(BuildKeyStampMixin):
161162
build_shared = _build_shared
162163

163164

165+
class TestDropEmptyVenvIncludeDir(DropEmptyVenvIncludeDirMixin):
166+
build_shared = _build_shared
167+
168+
164169
class TestForceBuildExtReexport(ForceBuildExtReexportMixin):
165170
build_hooks = build_hooks
166171
build_shared = _build_shared

‎cuda_core/setup.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -13,6 +13,8 @@
1313
from setuptools.command.build_ext import build_ext as _build_ext
1414
from setuptools.command.build_py import build_py as _build_py
1515

16+
from _build_shared import drop_empty_venv_include_dir
17+
1618
nthreads = int(os.environ.get("CUDA_PYTHON_PARALLEL_LEVEL", os.cpu_count() // 2))
1719
coverage_mode = bool(int(os.environ.get("CUDA_PYTHON_COVERAGE", "0")))
1820
_ROOT_DIR = Path(__file__).resolve().parent
@@ -56,6 +58,9 @@ def _build_aoti_shim_lib(compiler, plat_name):
5658
class build_ext(_build_ext): # noqa: N801
5759
def finalize_options(self):
5860
super().finalize_options()
61+
# A per-build venv path on every -I line defeats sccache's
62+
# preprocessor cache; see _build_shared.drop_empty_venv_include_dir().
63+
self.include_dirs = drop_empty_venv_include_dir(self.include_dirs)
5964
# A stale .so from a previous build configuration (CUDA major,
6065
# toolchain, debug/coverage) looks perfectly fresh; see
6166
# build_hooks._check_build_config().

‎cuda_core/tests/test_build_hooks.py‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -908,6 +908,7 @@ def fake_cythonize(ext_modules, **kwargs):
908908
BuildKeyStampMixin,
909909
CheckToolchainAvailableMixin,
910910
DistutilsLinkerIntegrationMixin,
911+
DropEmptyVenvIncludeDirMixin,
911912
ForceBuildExtReexportMixin,
912913
ResolveToolchainMixin,
913914
WithCompilerMixin,
@@ -944,6 +945,10 @@ class TestBuildKeyStamp(BuildKeyStampMixin):
944945
build_shared = _build_shared
945946

946947

948+
class TestDropEmptyVenvIncludeDir(DropEmptyVenvIncludeDirMixin):
949+
build_shared = _build_shared
950+
951+
947952
class TestForceBuildExtReexport(ForceBuildExtReexportMixin):
948953
build_hooks = build_hooks
949954
build_shared = _build_shared

‎cuda_python_test_helpers/cuda_python_test_helpers/build_shared.py‎

Lines changed: 65 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -470,6 +470,71 @@ def test_record_creates_the_directory_and_writes_the_key(self, stamp_file):
470470
assert stamp_file.read_text(encoding="utf-8") == "cu13-gnu-opt\n"
471471

472472

473+
class DropEmptyVenvIncludeDirMixin:
474+
"""``drop_empty_venv_include_dir``: remove the venv include dir only when it holds no files."""
475+
476+
build_shared = None
477+
478+
@pytest.fixture
479+
def venv(self, tmp_path, monkeypatch):
480+
venv = tmp_path / "build-env-abc123"
481+
monkeypatch.setattr(sys, "exec_prefix", str(venv))
482+
monkeypatch.setattr(sys, "base_exec_prefix", str(tmp_path / "python"))
483+
return venv
484+
485+
@pytest.mark.agent_authored(model="claude-opus-5.5")
486+
def test_outside_a_venv_returns_the_list_unchanged(self, tmp_path, monkeypatch):
487+
monkeypatch.setattr(sys, "exec_prefix", str(tmp_path))
488+
monkeypatch.setattr(sys, "base_exec_prefix", str(tmp_path))
489+
dirs = ["/opt/cuda/include", os.path.join(str(tmp_path), "include")]
490+
assert self.build_shared.drop_empty_venv_include_dir(dirs) == dirs
491+
492+
@pytest.mark.agent_authored(model="claude-opus-5.5")
493+
def test_missing_venv_include_dir_is_dropped(self, venv):
494+
venv_include = os.path.join(str(venv), "include")
495+
dirs = ["/opt/cuda/include", venv_include, "/usr/include/python3"]
496+
assert self.build_shared.drop_empty_venv_include_dir(dirs) == ["/opt/cuda/include", "/usr/include/python3"]
497+
498+
@pytest.mark.agent_authored(model="claude-opus-5.5")
499+
def test_venv_include_dir_with_only_empty_subdirs_is_dropped(self, venv):
500+
(venv / "include" / "site" / "python3.14").mkdir(parents=True)
501+
dirs = ["/opt/cuda/include", os.path.join(str(venv), "include")]
502+
assert self.build_shared.drop_empty_venv_include_dir(dirs) == ["/opt/cuda/include"]
503+
504+
@pytest.mark.agent_authored(model="claude-opus-5.5")
505+
def test_venv_include_dir_with_a_header_is_kept(self, venv):
506+
header_dir = venv / "include" / "site" / "python3.14" / "greenlet"
507+
header_dir.mkdir(parents=True)
508+
(header_dir / "greenlet.h").write_text("", encoding="utf-8")
509+
dirs = ["/opt/cuda/include", os.path.join(str(venv), "include")]
510+
assert self.build_shared.drop_empty_venv_include_dir(dirs) == dirs
511+
512+
@pytest.mark.agent_authored(model="claude-sonnet-5.5")
513+
def test_venv_include_dir_with_a_header_behind_a_symlinked_dir_is_kept(self, venv, tmp_path):
514+
vendor = tmp_path / "vendor-headers"
515+
vendor.mkdir()
516+
(vendor / "config.h").write_text("", encoding="utf-8")
517+
(venv / "include").mkdir(parents=True)
518+
try:
519+
(venv / "include" / "vendor").symlink_to(vendor, target_is_directory=True)
520+
except OSError as exc: # e.g. Windows without symlink privilege
521+
pytest.skip(f"cannot create symlinks: {exc}")
522+
dirs = ["/opt/cuda/include", os.path.join(str(venv), "include")]
523+
assert self.build_shared.drop_empty_venv_include_dir(dirs) == dirs
524+
525+
@pytest.mark.agent_authored(model="claude-sonnet-5.5")
526+
def test_venv_include_dir_with_a_symlink_to_an_empty_dir_is_dropped(self, venv, tmp_path):
527+
empty = tmp_path / "empty-headers"
528+
empty.mkdir()
529+
(venv / "include").mkdir(parents=True)
530+
try:
531+
(venv / "include" / "vendor").symlink_to(empty, target_is_directory=True)
532+
except OSError as exc: # e.g. Windows without symlink privilege
533+
pytest.skip(f"cannot create symlinks: {exc}")
534+
dirs = ["/opt/cuda/include", os.path.join(str(venv), "include")]
535+
assert self.build_shared.drop_empty_venv_include_dir(dirs) == ["/opt/cuda/include"]
536+
537+
473538
class ForceBuildExtReexportMixin:
474539
"""``build_hooks.force_build_ext`` is a live view of ``build_shared.force_build_ext``.
475540

0 commit comments

Comments
 (0)