Skip to content

Commit 645c86a

Browse files
committed
feat(cuda.core): bridge CUDA driver logs into Python logging
Add register_cuda_log_bridge()/unregister_cuda_log_bridge() to cuda.core.utils, forwarding cuLogs* driver logs to a Python logging.Logger via cuLogsRegisterCallback/cuLogsUnregisterCallback. Refs: NVIDIA#671
1 parent 0271e3a commit 645c86a

3 files changed

Lines changed: 230 additions & 0 deletions

File tree

‎cuda_core/cuda/core/utils/__init__.py‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,10 @@
1717
StridedMemoryView,
1818
args_viewable_as_strided_memory,
1919
)
20+
from cuda.core.utils._logging import (
21+
register_cuda_log_bridge,
22+
unregister_cuda_log_bridge,
23+
)
2024
from cuda.core.utils._program_cache import (
2125
FileStreamProgramCache,
2226
InMemoryProgramCache,
@@ -38,4 +42,6 @@
3842
"discard_prefetch_batch",
3943
"make_program_cache_key",
4044
"prefetch_batch",
45+
"register_cuda_log_bridge",
46+
"unregister_cuda_log_bridge",
4147
]
Lines changed: 134 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,134 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
"""Bridge the CUDA driver's ``cuLogs*`` error-log-management callback into
6+
the standard :mod:`logging` module.
7+
8+
The CUDA driver (CUDA 12.9+) maintains an internal, human-readable error/
9+
warning log stream (see the *Error Log Management* section of the CUDA
10+
driver API docs) that is otherwise only observable via the ``CUDA_LOG_FILE``
11+
environment variable or stderr. This module lets a caller route that stream
12+
into any :class:`logging.Logger`, so it can be filtered, captured (e.g. via
13+
pytest's ``caplog``), and forwarded using ordinary Python logging tooling.
14+
15+
xref: https://github.com/NVIDIA/cuda-python/issues/671
16+
"""
17+
18+
from __future__ import annotations
19+
20+
import ctypes
21+
import logging
22+
import threading
23+
24+
from cuda.bindings import driver as _driver
25+
from cuda.core._utils.cuda_utils import handle_return
26+
27+
__all__ = ["register_cuda_log_bridge", "unregister_cuda_log_bridge"]
28+
29+
# C callback signature expected by cuLogsRegisterCallback:
30+
# void callback(void *userData, CUlogLevel logLevel, char *message, size_t length)
31+
#
32+
# Note: cuLogsRegisterCallback requires an actual C function pointer. The
33+
# cuda.bindings wrapper does *not* accept a plain Python callable directly
34+
# (it only ever converts the `callbackFunc` argument to an integer address),
35+
# so we build a real function pointer via ctypes and pass its address.
36+
_CUlogsCallback_functype = ctypes.CFUNCTYPE(None, ctypes.c_void_p, ctypes.c_int, ctypes.c_char_p, ctypes.c_size_t)
37+
38+
# CUlogLevel currently only defines two levels (CUDA 12.9 cuLogs* API).
39+
# Anything unrecognized is conservatively mapped to ERROR.
40+
_CUDA_LEVEL_TO_PY_LEVEL = {
41+
int(_driver.CUlogLevel.CU_LOG_LEVEL_ERROR): logging.ERROR,
42+
int(_driver.CUlogLevel.CU_LOG_LEVEL_WARNING): logging.WARNING,
43+
}
44+
45+
_lock = threading.Lock()
46+
# Must hold a reference to the ctypes callback for as long as it is
47+
# registered: once it is garbage collected, the driver would be left
48+
# holding a dangling function pointer and crash the process on the next
49+
# log message.
50+
_c_callback: ctypes._FuncPointer | None = None
51+
_callback_handle = None # CUlogsCallbackHandle, needed to unregister
52+
53+
54+
def _make_c_callback(logger: logging.Logger) -> ctypes._FuncPointer:
55+
"""Build a ctypes C function pointer that forwards each driver log
56+
record to ``logger``.
57+
"""
58+
59+
def _on_log(_user_data, log_level, message_ptr, length):
60+
# This runs inside a ctypes callback trampoline, potentially from a
61+
# CUDA-driver-internal thread. It must never raise or block.
62+
try:
63+
raw = ctypes.string_at(message_ptr, length) if message_ptr else b""
64+
msg = raw.decode("utf-8", errors="replace").rstrip("\n")
65+
py_level = _CUDA_LEVEL_TO_PY_LEVEL.get(log_level, logging.ERROR)
66+
logger.log(py_level, "[CUDA Driver] %s", msg)
67+
except Exception:
68+
logger.exception("cuda_log_bridge: error while forwarding driver log")
69+
70+
return _CUlogsCallback_functype(_on_log)
71+
72+
73+
def register_cuda_log_bridge(logger: logging.Logger | None = None) -> None:
74+
"""Register a bridge that forwards CUDA driver ``cuLogs*`` messages to
75+
a Python :class:`logging.Logger`.
76+
77+
This is a process-wide, idempotent operation: the driver only supports
78+
a global log stream (not scoped to a context/stream/thread), so calling
79+
this again while a bridge is already registered is a no-op -- the
80+
logger passed to the first call remains in effect.
81+
82+
Parameters
83+
----------
84+
logger : logging.Logger, optional
85+
The logger that driver messages should be forwarded to. Defaults
86+
to ``logging.getLogger("cuda.driver")``.
87+
88+
Raises
89+
------
90+
cuda.core._utils.cuda_utils.CUDAError
91+
If the underlying ``cuLogsRegisterCallback`` call fails, e.g.
92+
because the loaded CUDA driver predates the 12.9 cuLogs* API.
93+
"""
94+
global _c_callback, _callback_handle
95+
96+
if logger is None:
97+
logger = logging.getLogger("cuda.driver")
98+
99+
with _lock:
100+
if _callback_handle is not None:
101+
return # already registered; idempotent
102+
103+
c_callback = _make_c_callback(logger)
104+
addr = ctypes.cast(c_callback, ctypes.c_void_p).value
105+
106+
handle = handle_return(_driver.cuLogsRegisterCallback(addr, None))
107+
108+
# Keep the ctypes callback object alive for as long as it is
109+
# registered (see the module-level comment on _c_callback above).
110+
_c_callback = c_callback
111+
_callback_handle = handle
112+
113+
114+
def unregister_cuda_log_bridge() -> None:
115+
"""Unregister the bridge previously installed by
116+
:func:`register_cuda_log_bridge`.
117+
118+
Idempotent: safe to call even if no bridge is currently registered.
119+
120+
Raises
121+
------
122+
cuda.core._utils.cuda_utils.CUDAError
123+
If the underlying ``cuLogsUnregisterCallback`` call fails.
124+
"""
125+
global _c_callback, _callback_handle
126+
127+
with _lock:
128+
if _callback_handle is None:
129+
return # nothing registered; idempotent
130+
131+
handle_return(_driver.cuLogsUnregisterCallback(_callback_handle))
132+
133+
_c_callback = None
134+
_callback_handle = None
Lines changed: 90 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,90 @@
1+
# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved.
2+
#
3+
# SPDX-License-Identifier: Apache-2.0
4+
5+
"""Tests for bridging CUDA driver ``cuLogs*`` messages into Python
6+
``logging`` (see design doc / GitHub issue #671).
7+
"""
8+
9+
import logging
10+
11+
import pytest
12+
13+
from cuda.bindings import driver
14+
from cuda.core.utils import register_cuda_log_bridge, unregister_cuda_log_bridge
15+
16+
17+
@pytest.fixture(autouse=True)
18+
def teardown_bridge():
19+
yield
20+
# Guarantee the callback is unregistered after every test, so a leaked
21+
# bridge/slot from one test can't affect later ones.
22+
unregister_cuda_log_bridge()
23+
24+
25+
# 使用方式1: 调用 register(logger), 通过 caplog 捕获 driver log
26+
def test_cuda_driver_log_captured_via_caplog(caplog):
27+
# 1. 初始化 CUDA Driver
28+
driver.cuInit(0)
29+
30+
# 2. 注册桥接,将驱动日志路由到用户提供的 logger
31+
logger_name = "cuda.driver.caplog_test"
32+
logger_obj = logging.getLogger(logger_name)
33+
register_cuda_log_bridge(logger_obj)
34+
35+
# 3. 触发驱动错误:ordinal 9999 必然越界,驱动会写入两条 [E] 日志
36+
with caplog.at_level(logging.ERROR, logger=logger_name):
37+
driver.cuDeviceGet(9999)
38+
39+
# 4. 验证 caplog 中包含两条驱动错误日志
40+
messages = [r.message for r in caplog.records if r.name == logger_name]
41+
42+
assert any("Parameter ordinal must be between 0 and 1" in m for m in messages), (
43+
f"Expected ordinal error in caplog, got:\n{messages}"
44+
)
45+
46+
assert any("CUDA_ERROR_INVALID_DEVICE" in m and "cuDeviceGet" in m for m in messages), (
47+
f"Expected cuDeviceGet return error in caplog, got:\n{messages}"
48+
)
49+
50+
51+
# 使用方式2: 通过 logging handler 把 driver log 倒入到文件
52+
def test_cuda_driver_log_written_to_file(tmp_path):
53+
# 1. 初始化 CUDA Driver
54+
driver.cuInit(0)
55+
56+
# 2. 准备一个独立的 logger,挂上写文件的 FileHandler
57+
# (用独立 logger name,避免污染 root logger / 影响其他用例)
58+
log_file = tmp_path / "cuda_driver.log"
59+
logger_name = "cuda.driver.file_test"
60+
logger_obj = logging.getLogger(logger_name)
61+
logger_obj.setLevel(logging.WARNING)
62+
63+
file_handler = logging.FileHandler(log_file, encoding="utf-8")
64+
file_handler.setLevel(logging.WARNING)
65+
file_handler.setFormatter(logging.Formatter("%(asctime)s %(levelname)s %(name)s: %(message)s"))
66+
logger_obj.addHandler(file_handler)
67+
68+
try:
69+
# 3. 注册桥接,将驱动日志路由到该 logger
70+
register_cuda_log_bridge(logger_obj)
71+
72+
# 4. 触发驱动错误:ordinal 9999 必然越界,驱动会写入两条 [E] 日志
73+
driver.cuDeviceGet(9999)
74+
finally:
75+
# 5. 显式移除并关闭 handler:
76+
# - close() 会 flush 缓冲区,确保日志落盘再读取
77+
# - removeHandler 避免 handler 残留导致文件句柄泄漏 / 影响后续用例
78+
logger_obj.removeHandler(file_handler)
79+
file_handler.close()
80+
81+
# 6. 校验文件内容
82+
content = log_file.read_text(encoding="utf-8")
83+
84+
assert "Parameter ordinal must be between 0 and 1" in content, (
85+
f"Expected ordinal error in log file, got:\n{content}"
86+
)
87+
88+
assert "CUDA_ERROR_INVALID_DEVICE" in content and "cuDeviceGet" in content, (
89+
f"Expected cuDeviceGet return error in log file, got:\n{content}"
90+
)

0 commit comments

Comments
 (0)