Skip to content

Commit f8245a5

Browse files
juenglinjuenglin
committed
build: infer toolchain from external CC/CXX; fix externally-supplied-clang regression
When CUDA_PYTHON_TOOLCHAIN is unset on Linux, infer the toolchain from the externally-set CC/CXX (CXX preferred, fall back to CC): a value containing 'clang' selects llvm, else gnu. This fixes the regression reported by sylvesterkaczmarek: previously the default 'gnu' flag set (incl. -fno-var-tracking-assignments) reached an externally-supplied clang because the default path did not override CC/CXX and the old _is_clang strip was removed. Now clang is inferred and the llvm flag set (no gcc-only flags, -fuse-ld=lld) is used, and the external compiler is left in place, so a wrapper like CC='sccache clang' survives and gets the llvm flags. When CUDA_PYTHON_TOOLCHAIN is set it takes precedence over an externally- set CC: a mismatch warns (CC only; CXX commonly defaults to 'c++' and is not a reliable user-intent signal) and the external CC is overridden. The shared toolchain helpers remain byte-identical across the two build_hooks.py via the 'keep in sync' markers. _resolve_toolchain_name now returns an 'explicit' flag so _apply_toolchain_env only overrides CC/CXX when the toolchain was chosen explicitly (not inferred). Tests: regression tests for externally-supplied CC=clang (infer llvm, no gcc-only flags, CC survives), CC=gcc (infer gnu), and the explicit mismatch/no-mismatch cases. A module-level autouse fixture cleans CC/CXX/LDSHARED per test because _apply_toolchain_env sets them directly in os.environ, which monkeypatch does not revert.
1 parent b0289f6 commit f8245a5

4 files changed

Lines changed: 262 additions & 38 deletions

File tree

‎cuda_bindings/build_hooks.py‎

Lines changed: 52 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -99,33 +99,67 @@ def _get_cuda_path() -> str:
9999
}
100100

101101

102-
def _resolve_toolchain_name():
103-
"""Read CUDA_PYTHON_TOOLCHAIN, validate it, return (name, allowed, cc, cxx).
102+
def _infer_compiler_family(value):
103+
"""Return 'llvm' if the compiler string looks like clang, else 'gnu'."""
104+
return "llvm" if "clang" in value else "gnu"
105+
104106

105-
The default toolchain (gnu on Linux, msvc on Windows) is the first entry
106-
of the platform's allowed tuple. cc/cxx are the compiler binaries for the
107-
toolchain (None for msvc, which distutils discovers via the MSVC env).
107+
def _resolve_toolchain_name():
108+
"""Read CUDA_PYTHON_TOOLCHAIN / CC / CXX and pick the toolchain.
109+
110+
Returns (name, allowed, cc, cxx, explicit). On Linux the default is gnu;
111+
on Windows the only value is msvc. When CUDA_PYTHON_TOOLCHAIN is set it
112+
takes precedence over an externally-set CC/CXX: a mismatch warns and the
113+
external CC/CXX is overridden. When CUDA_PYTHON_TOOLCHAIN is unset on
114+
Linux, the toolchain is inferred from the external CC/CXX (CXX preferred,
115+
fall back to CC): a value containing 'clang' selects llvm, else gnu. In
116+
the inferred case the external compiler is left in place (not overridden),
117+
so a wrapper like CC='sccache clang' survives and gets the llvm flags.
108118
"""
109119
if sys.platform == "win32":
110120
platform_key, allowed = "win32", _TOOLCHAINS_WINDOWS
111121
else:
112122
platform_key, allowed = "linux", _TOOLCHAINS_LINUX
113-
name = os.environ.get("CUDA_PYTHON_TOOLCHAIN", allowed[0]).strip().lower()
114-
if name not in allowed:
115-
raise RuntimeError(
116-
f"CUDA_PYTHON_TOOLCHAIN={name!r} is not supported on {platform_key}. Valid values: {', '.join(allowed)}."
117-
)
123+
124+
explicit = os.environ.get("CUDA_PYTHON_TOOLCHAIN", "").strip().lower()
125+
external_cc = os.environ.get("CC", "").strip()
126+
external_cxx = os.environ.get("CXX", "").strip()
127+
128+
if explicit:
129+
name = explicit
130+
if name not in allowed:
131+
raise RuntimeError(
132+
f"CUDA_PYTHON_TOOLCHAIN={name!r} is not supported on {platform_key}. Valid values: {', '.join(allowed)}."
133+
)
134+
# Warn if an explicit toolchain conflicts with an externally-set CC. We check
135+
# CC only (not CXX): CXX commonly defaults to 'c++' in the environment and is
136+
# not a reliable user-intent signal, whereas CC is the canonical override.
137+
tc_cc, _tc_cxx = _TOOLCHAIN_COMPILERS[name]
138+
if tc_cc is not None and external_cc and _infer_compiler_family(external_cc) != _infer_compiler_family(tc_cc):
139+
warn(
140+
f"CUDA_PYTHON_TOOLCHAIN={name} takes precedence over externally-set CC ({external_cc!r}); ignoring it.",
141+
stacklevel=2,
142+
)
143+
elif platform_key == "linux":
144+
# Infer from the external compiler (CXX preferred, fall back to CC).
145+
probe = external_cxx or external_cc
146+
name = "llvm" if probe and "clang" in probe else allowed[0]
147+
else:
148+
name = allowed[0]
149+
118150
cc, cxx = _TOOLCHAIN_COMPILERS[name]
119-
return name, allowed, cc, cxx
151+
return name, allowed, cc, cxx, bool(explicit)
120152

121153

122-
def _apply_toolchain_env(name, allowed, cc, cxx):
123-
"""Set CC/CXX/LDSHARED for a non-default toolchain.
154+
def _apply_toolchain_env(cc, cxx, explicit):
155+
"""Set CC/CXX/LDSHARED for an explicitly-chosen non-default toolchain.
124156
125-
The default path intentionally does not touch the env, so an externally-
126-
set compiler (e.g. CC="sccache cc" in CI) keeps working.
157+
The default path and the inferred path intentionally do not touch the
158+
env, so an externally-set compiler (e.g. CC='sccache cc' or CC='clang' in
159+
CI) keeps working. Only an explicit CUDA_PYTHON_TOOLCHAIN override governs
160+
the compiler.
127161
"""
128-
if name != allowed[0] and cc is not None:
162+
if explicit and cc is not None:
129163
os.environ["CC"] = cc
130164
os.environ["CXX"] = cxx
131165
os.environ["LDSHARED"] = f"{cxx} -shared"
@@ -164,7 +198,7 @@ def _resolve_toolchain(debug=False, compile_for_coverage=False):
164198
Linux) selects clang/clang++ and lld and sets CC/CXX/LDSHARED so distutils'
165199
customize_compiler picks them up.
166200
"""
167-
name, allowed, cc, cxx = _resolve_toolchain_name()
201+
name, _allowed, cc, cxx, explicit = _resolve_toolchain_name()
168202

169203
extra_compile_args = []
170204
extra_link_args = []
@@ -199,7 +233,7 @@ def _resolve_toolchain(debug=False, compile_for_coverage=False):
199233
# related to free-threading builds.
200234
extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"]
201235

202-
_apply_toolchain_env(name, allowed, cc, cxx)
236+
_apply_toolchain_env(cc, cxx, explicit)
203237

204238
return name, cc, cxx, extra_compile_args, extra_link_args
205239

‎cuda_bindings/tests/test_build_hooks.py‎

Lines changed: 76 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -39,6 +39,16 @@ def _load_build_hooks():
3939
build_hooks = _load_build_hooks()
4040

4141

42+
@pytest.fixture(autouse=True)
43+
def _clean_cc_env(monkeypatch):
44+
# _apply_toolchain_env sets CC/CXX/LDSHARED directly in os.environ (so
45+
# distutils picks them up), which monkeypatch does not revert because it
46+
# did not set them. Clean them per test so toolchain state never leaks
47+
# across tests.
48+
for k in ("CUDA_PYTHON_TOOLCHAIN", "CC", "CXX", "LDSHARED"):
49+
monkeypatch.delenv(k, raising=False)
50+
51+
4252
class TestResolveToolchain:
4353
"""_resolve_toolchain: pick compiler/linker/flags from CUDA_PYTHON_TOOLCHAIN.
4454
@@ -135,3 +145,69 @@ def fake_which(name):
135145
def test_llvm_present_passes(self, monkeypatch):
136146
monkeypatch.setattr(build_hooks.shutil, "which", lambda name: "/bin/" + name)
137147
build_hooks._check_toolchain_available("llvm")
148+
149+
150+
class TestInferToolchain:
151+
"""When CUDA_PYTHON_TOOLCHAIN is unset, infer the toolchain from CC/CXX.
152+
153+
Regression for the externally-supplied-clang path (cuda.bindings): previously
154+
the default 'gnu' flag set (incl. -fno-var-tracking-assignments) reached clang because the default path did not override CC/CXX and the old _is_clang strip was removed. Now clang is inferred and the llvm flag set (no gcc-only flags, -fuse-ld=lld) is used, and the external compiler is left in place.
155+
"""
156+
157+
@pytest.mark.agent_authored(model="glm-5.2")
158+
def test_external_cc_clang_infers_llvm(self, monkeypatch):
159+
if sys.platform == "win32":
160+
pytest.skip("inference is Linux-only")
161+
monkeypatch.setenv("CC", "clang")
162+
name, _cc, _cxx, cargs, largs = build_hooks._resolve_toolchain()
163+
assert name == "llvm"
164+
# llvm flags: no gcc-only -fno-var-tracking-assignments, uses lld
165+
assert "-fno-var-tracking-assignments" not in cargs
166+
assert "-fpermissive" not in cargs # cuda.bindings gnu keeps this; llvm drops it
167+
assert "-fuse-ld=lld" in largs
168+
# inferred path does not override the external compiler
169+
assert os.environ["CC"] == "clang"
170+
171+
@pytest.mark.agent_authored(model="glm-5.2")
172+
def test_external_cxx_clang_infers_llvm(self, monkeypatch):
173+
if sys.platform == "win32":
174+
pytest.skip("inference is Linux-only")
175+
monkeypatch.setenv("CXX", "clang++")
176+
name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain()
177+
assert name == "llvm"
178+
179+
@pytest.mark.agent_authored(model="glm-5.2")
180+
def test_external_cc_gcc_infers_gnu(self, monkeypatch):
181+
if sys.platform == "win32":
182+
pytest.skip("inference is Linux-only")
183+
monkeypatch.setenv("CC", "gcc")
184+
name, _cc, _cxx, cargs, _largs = build_hooks._resolve_toolchain()
185+
assert name == "gnu"
186+
# gnu keeps the gcc-only flag (the regression was that clang got it)
187+
assert "-fno-var-tracking-assignments" in cargs
188+
assert os.environ["CC"] == "gcc" # not overridden
189+
190+
@pytest.mark.agent_authored(model="glm-5.2")
191+
def test_explicit_gnu_with_clang_cc_warns_and_overrides(self, monkeypatch):
192+
if sys.platform == "win32":
193+
pytest.skip("Linux-only")
194+
monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "gnu")
195+
monkeypatch.setenv("CC", "clang")
196+
with pytest.warns(UserWarning, match="takes precedence"):
197+
name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain()
198+
assert name == "gnu"
199+
assert os.environ["CC"] == "cc" # explicit toolchain governs
200+
201+
@pytest.mark.agent_authored(model="glm-5.2")
202+
def test_explicit_llvm_with_clang_cc_no_warning(self, monkeypatch):
203+
if sys.platform == "win32":
204+
pytest.skip("Linux-only")
205+
monkeypatch.setenv("CUDA_PYTHON_TOOLCHAIN", "llvm")
206+
monkeypatch.setenv("CC", "clang")
207+
import warnings
208+
209+
with warnings.catch_warnings():
210+
warnings.simplefilter("error") # any warning here would fail the test
211+
name, _cc, _cxx, _cargs, _largs = build_hooks._resolve_toolchain()
212+
assert name == "llvm"
213+
assert os.environ["CC"] == "clang" # overridden to clang (same family)

‎cuda_core/build_hooks.py‎

Lines changed: 53 additions & 18 deletions
Original file line numberDiff line numberDiff line change
@@ -16,6 +16,7 @@
1616
import tempfile
1717
import zipfile
1818
from pathlib import Path
19+
from warnings import warn
1920

2021
from Cython.Build import cythonize
2122
from Cython.Compiler import Options as _CythonOptions
@@ -95,33 +96,67 @@ def _get_cuda_path() -> str:
9596
}
9697

9798

98-
def _resolve_toolchain_name():
99-
"""Read CUDA_PYTHON_TOOLCHAIN, validate it, return (name, allowed, cc, cxx).
99+
def _infer_compiler_family(value):
100+
"""Return 'llvm' if the compiler string looks like clang, else 'gnu'."""
101+
return "llvm" if "clang" in value else "gnu"
102+
100103

101-
The default toolchain (gnu on Linux, msvc on Windows) is the first entry
102-
of the platform's allowed tuple. cc/cxx are the compiler binaries for the
103-
toolchain (None for msvc, which distutils discovers via the MSVC env).
104+
def _resolve_toolchain_name():
105+
"""Read CUDA_PYTHON_TOOLCHAIN / CC / CXX and pick the toolchain.
106+
107+
Returns (name, allowed, cc, cxx, explicit). On Linux the default is gnu;
108+
on Windows the only value is msvc. When CUDA_PYTHON_TOOLCHAIN is set it
109+
takes precedence over an externally-set CC/CXX: a mismatch warns and the
110+
external CC/CXX is overridden. When CUDA_PYTHON_TOOLCHAIN is unset on
111+
Linux, the toolchain is inferred from the external CC/CXX (CXX preferred,
112+
fall back to CC): a value containing 'clang' selects llvm, else gnu. In
113+
the inferred case the external compiler is left in place (not overridden),
114+
so a wrapper like CC='sccache clang' survives and gets the llvm flags.
104115
"""
105116
if sys.platform == "win32":
106117
platform_key, allowed = "win32", _TOOLCHAINS_WINDOWS
107118
else:
108119
platform_key, allowed = "linux", _TOOLCHAINS_LINUX
109-
name = os.environ.get("CUDA_PYTHON_TOOLCHAIN", allowed[0]).strip().lower()
110-
if name not in allowed:
111-
raise RuntimeError(
112-
f"CUDA_PYTHON_TOOLCHAIN={name!r} is not supported on {platform_key}. Valid values: {', '.join(allowed)}."
113-
)
120+
121+
explicit = os.environ.get("CUDA_PYTHON_TOOLCHAIN", "").strip().lower()
122+
external_cc = os.environ.get("CC", "").strip()
123+
external_cxx = os.environ.get("CXX", "").strip()
124+
125+
if explicit:
126+
name = explicit
127+
if name not in allowed:
128+
raise RuntimeError(
129+
f"CUDA_PYTHON_TOOLCHAIN={name!r} is not supported on {platform_key}. Valid values: {', '.join(allowed)}."
130+
)
131+
# Warn if an explicit toolchain conflicts with an externally-set CC. We check
132+
# CC only (not CXX): CXX commonly defaults to 'c++' in the environment and is
133+
# not a reliable user-intent signal, whereas CC is the canonical override.
134+
tc_cc, _tc_cxx = _TOOLCHAIN_COMPILERS[name]
135+
if tc_cc is not None and external_cc and _infer_compiler_family(external_cc) != _infer_compiler_family(tc_cc):
136+
warn(
137+
f"CUDA_PYTHON_TOOLCHAIN={name} takes precedence over externally-set CC ({external_cc!r}); ignoring it.",
138+
stacklevel=2,
139+
)
140+
elif platform_key == "linux":
141+
# Infer from the external compiler (CXX preferred, fall back to CC).
142+
probe = external_cxx or external_cc
143+
name = "llvm" if probe and "clang" in probe else allowed[0]
144+
else:
145+
name = allowed[0]
146+
114147
cc, cxx = _TOOLCHAIN_COMPILERS[name]
115-
return name, allowed, cc, cxx
148+
return name, allowed, cc, cxx, bool(explicit)
116149

117150

118-
def _apply_toolchain_env(name, allowed, cc, cxx):
119-
"""Set CC/CXX/LDSHARED for a non-default toolchain.
151+
def _apply_toolchain_env(cc, cxx, explicit):
152+
"""Set CC/CXX/LDSHARED for an explicitly-chosen non-default toolchain.
120153
121-
The default path intentionally does not touch the env, so an externally-
122-
set compiler (e.g. CC="sccache cc" in CI) keeps working.
154+
The default path and the inferred path intentionally do not touch the
155+
env, so an externally-set compiler (e.g. CC='sccache cc' or CC='clang' in
156+
CI) keeps working. Only an explicit CUDA_PYTHON_TOOLCHAIN override governs
157+
the compiler.
123158
"""
124-
if name != allowed[0] and cc is not None:
159+
if explicit and cc is not None:
125160
os.environ["CC"] = cc
126161
os.environ["CXX"] = cxx
127162
os.environ["LDSHARED"] = f"{cxx} -shared"
@@ -160,7 +195,7 @@ def _resolve_toolchain(debug=False, compile_for_coverage=False):
160195
Linux) selects clang/clang++ and lld and sets CC/CXX/LDSHARED so distutils'
161196
customize_compiler picks them up.
162197
"""
163-
name, allowed, cc, cxx = _resolve_toolchain_name()
198+
name, _allowed, cc, cxx, explicit = _resolve_toolchain_name()
164199

165200
extra_compile_args = []
166201
extra_link_args = []
@@ -190,7 +225,7 @@ def _resolve_toolchain(debug=False, compile_for_coverage=False):
190225
# related to free-threading builds.
191226
extra_compile_args += ["-DCYTHON_TRACE_NOGIL=1", "-DCYTHON_USE_SYS_MONITORING=0"]
192227

193-
_apply_toolchain_env(name, allowed, cc, cxx)
228+
_apply_toolchain_env(cc, cxx, explicit)
194229

195230
return name, cc, cxx, extra_compile_args, extra_link_args
196231

0 commit comments

Comments
 (0)