Skip to content

Commit 57b2709

Browse files
GodlyDonutsleofang
andauthored
Avoid nvJitLink cache crashes during incremental LTO (#3040)
* Avoid nvJitLink cache crashes during incremental LTO * Limit incremental LTO cache workaround to nvJitLink before 13.6 * remove redundant defensive line that is problematic in FT Python --------- Co-authored-by: Leo Fang <[email protected]>
1 parent 01037b1 commit 57b2709

3 files changed

Lines changed: 36 additions & 5 deletions

File tree

‎cuda_core/cuda/core/_linker.pyi‎

Lines changed: 6 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -153,7 +153,9 @@ class LinkerOptions:
153153
incremental : bool, optional
154154
Perform an incremental link. The result can be passed
155155
directly to a later :class:`Linker`. Requires nvJitLink 13.2 or newer
156-
and is not supported by the driver linker backend.
156+
and is not supported by the driver linker backend. Intermediate
157+
caching is disabled on nvJitLink versions older than 13.6 when combined
158+
with link-time optimization, to work around an nvJitLink bug.
157159
Default: False.
158160
ptx : bool, optional
159161
Emit PTX after linking instead of CUBIN; only supported with ``link_time_optimization=True``.
@@ -198,6 +200,9 @@ class LinkerOptions:
198200
Default: 1.
199201
no_cache : bool, optional
200202
Do not cache the intermediate steps of nvJitLink.
203+
Caching is also disabled automatically for incremental links that use
204+
link-time optimization on nvJitLink versions older than 13.6, to work
205+
around an nvJitLink bug.
201206
Default: False.
202207
numba_debug : bool, optional
203208
Non-functional. ``numba_debug`` is an NVVM/NVRTC *compiler* option;

‎cuda_core/cuda/core/_linker.pyx‎

Lines changed: 10 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -271,7 +271,9 @@ class LinkerOptions:
271271
incremental : bool, optional
272272
Perform an incremental link. The result can be passed
273273
directly to a later :class:`Linker`. Requires nvJitLink 13.2 or newer
274-
and is not supported by the driver linker backend.
274+
and is not supported by the driver linker backend. Intermediate
275+
caching is disabled on nvJitLink versions older than 13.6 when combined
276+
with link-time optimization, to work around an nvJitLink bug.
275277
Default: False.
276278
ptx : bool, optional
277279
Emit PTX after linking instead of CUBIN; only supported with ``link_time_optimization=True``.
@@ -316,6 +318,9 @@ class LinkerOptions:
316318
Default: 1.
317319
no_cache : bool, optional
318320
Do not cache the intermediate steps of nvJitLink.
321+
Caching is also disabled automatically for incremental links that use
322+
link-time optimization on nvJitLink versions older than 13.6, to work
323+
around an nvJitLink bug.
319324
Default: False.
320325
numba_debug : bool, optional
321326
Non-functional. ``numba_debug`` is an NVVM/NVRTC *compiler* option;
@@ -431,7 +436,10 @@ class LinkerOptions:
431436
options.append(f"-split-compile={int(self.split_compile)}")
432437
if self.split_compile_extended is not None:
433438
options.append(f"-split-compile-extended={int(self.split_compile_extended)}")
434-
if self.no_cache is True:
439+
# nvbugs 6659652
440+
if self.no_cache is True or (
441+
_nvjitlink_version < (13, 6) and self.incremental and self.link_time_optimization
442+
):
435443
options.append("-no-cache")
436444

437445
if as_bytes:
@@ -782,8 +790,6 @@ def _decide_nvjitlink_or_driver() -> bool:
782790
if _use_nvjitlink_backend is not None:
783791
return not _use_nvjitlink_backend
784792

785-
_nvjitlink_version = None
786-
787793
warn_txt_common = (
788794
"the driver APIs will be used instead, which do not support"
789795
" minor version compatibility or linking LTO IRs."

‎cuda_core/tests/test_linker.py‎

Lines changed: 20 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -244,6 +244,26 @@ def test_linker_options_incremental_as_bytes(value, expected_count):
244244
assert options.as_bytes().count(b"-r") == expected_count
245245

246246

247+
@pytest.mark.agent_authored(model="gpt-6")
248+
@pytest.mark.skipif(is_culink_backend, reason="as_bytes() only supported for nvjitlink backend")
249+
@pytest.mark.parametrize(
250+
"version,needs_workaround", [((13, 4), True), ((13, 5), True), ((13, 6), False), ((14, 0), False)]
251+
)
252+
@pytest.mark.parametrize(
253+
"incremental,link_time_optimization", [(False, False), (False, True), (True, False), (True, True)]
254+
)
255+
@pytest.mark.parametrize("no_cache", [None, False, True])
256+
def test_linker_options_incremental_lto_disables_cache(
257+
monkeypatch, version, needs_workaround, incremental, link_time_optimization, no_cache
258+
):
259+
options = LinkerOptions(
260+
arch="sm_80", incremental=incremental, link_time_optimization=link_time_optimization, no_cache=no_cache
261+
)
262+
monkeypatch.setattr(_linker, "_nvjitlink_version", version)
263+
expected = no_cache is True or (needs_workaround and incremental and link_time_optimization)
264+
assert options.as_bytes().count(b"-no-cache") == int(expected)
265+
266+
247267
@pytest.mark.parametrize("backend", ("invalid", "driver"))
248268
def test_linker_options_as_bytes_invalid_backend(backend):
249269
"""Test LinkerOptions.as_bytes() with invalid backend"""

0 commit comments

Comments
 (0)