Skip to content

Commit 8f7a9eb

Browse files
authored
Work around Orin IPC export FD exhaustion (nvbug 6977893) (#3191)
On Orin, recognize the driver's successful export with an invalid -EMFILE descriptor as a known bug. Use an explicit non-strict xfail restricted to that outcome, so fixed drivers can pass and unrelated failures still fail. Continue checking descriptor rollback and the successful export/import path before reporting the expected failure. Close all test buffers on partial setup and assertion failures. Avoid closing a negative descriptor in the export rollback so the original invalid-handle error is preserved instead of being masked by EBADF. Signed-off-by: Ralf W. Grosse-Kunstleve <[email protected]>
1 parent 70ac738 commit 8f7a9eb

2 files changed

Lines changed: 70 additions & 47 deletions

File tree

‎cuda_core/cuda/core/_memory/_virtual_memory_resource.pyx‎

Lines changed: 3 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1124,7 +1124,9 @@ cdef VirtualMemoryIPCBufferDescriptor _export_ipc_descriptor(Buffer buf):
11241124
try:
11251125
handle = IPCAllocationHandle._init(fd, None)
11261126
except: # noqa: E722 rollback-then-raise: the fd is not owned yet
1127-
os.close(fd)
1127+
# nvbug 6977893: an Orin driver can return a negative fd despite success.
1128+
if fd >= 0:
1129+
os.close(fd)
11281130
raise
11291131
# The handle owns the fd now; if append fails, the handle closes it once.
11301132
handles.append(handle)

‎cuda_core/tests/memory_ipc/test_vmm_ipc.py‎

Lines changed: 67 additions & 46 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"""
1313

1414
import ctypes
15+
import errno
1516
import gc
1617
import json
1718
import multiprocessing as mp
@@ -20,6 +21,7 @@
2021
import pickle
2122
import socket
2223
import threading
24+
from contextlib import ExitStack
2325

2426
import pytest
2527
from helpers.child_processes import child_timeout_sec, kill_subprocesses, noop, run_child, warmup_multiprocessing
@@ -700,66 +702,85 @@ def child_main(device_id, to_child, from_child):
700702
buffer.close()
701703

702704

705+
class _OrinIPCExportFDExhaustionError(Exception):
706+
"""nvbug 6977893: Orin export reports success with an invalid -EMFILE fd."""
707+
708+
703709
class TestFileDescriptorLimit:
704710
@pytest.mark.agent_authored(model="claude-fable-5-1")
705711
@pytest.mark.thread_unsafe(reason="changes the process-wide file descriptor limit")
706-
def test_export_near_the_limit(self, vmm_ipc_device):
712+
def test_export_near_the_limit(self, vmm_ipc_device, request):
707713
"""Under a lowered RLIMIT_NOFILE an export either succeeds or fails with a readable error and no leak."""
708714
import resource # POSIX-only; a module-level import breaks collection on Windows
709715

710716
device = vmm_ipc_device
717+
is_orin = device.name == "Orin"
718+
xfail_reason = "Workaround for nvbug 6977893: Orin IPC export mishandles FD exhaustion"
719+
if is_orin:
720+
request.node.add_marker(
721+
pytest.mark.xfail(reason=xfail_reason, raises=_OrinIPCExportFDExhaustionError, strict=False)
722+
)
723+
orin_export_bug = False
711724
stream = device.default_stream
712725
mr = _resource(device)
713-
first = mr.allocate(1)
714-
gran = first.size
715-
second = mr.modify_allocation(first, 2 * gran)
716-
buf = mr.modify_allocation(second, 3 * gran)
717-
values = (0x61, 0x62, 0x63)
718-
for i, value in enumerate(values):
719-
_fill(buf, value, offset=i * gran, size=gran)
720-
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
721-
fillers = []
722-
imported = desc = None
723-
try:
724-
# Fill the holes in the descriptor table so that the limit alone decides how many more fit.
725-
highest = max(int(name) for name in os.listdir("/proc/self/fd"))
726-
while True:
727-
fd = os.open(os.devnull, os.O_RDONLY)
728-
if fd > highest:
729-
os.close(fd)
730-
break
731-
fillers.append(fd)
732-
baseline = _settled_open_fds()
733-
# Room for two more descriptors: a three-allocation export cannot complete.
734-
resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 2, hard))
735-
failure = None
726+
with ExitStack() as buffers:
727+
first = buffers.enter_context(mr.allocate(1))
728+
gran = first.size
729+
second = buffers.enter_context(mr.modify_allocation(first, 2 * gran))
730+
buf = buffers.enter_context(mr.modify_allocation(second, 3 * gran))
731+
values = (0x61, 0x62, 0x63)
732+
for i, value in enumerate(values):
733+
_fill(buf, value, offset=i * gran, size=gran)
734+
soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE)
735+
fillers = []
736+
desc = None
736737
try:
738+
# Fill the holes in the descriptor table so that the limit alone decides how many more fit.
739+
highest = max(int(name) for name in os.listdir("/proc/self/fd"))
740+
while True:
741+
fd = os.open(os.devnull, os.O_RDONLY)
742+
if fd > highest:
743+
os.close(fd)
744+
break
745+
fillers.append(fd)
746+
baseline = _settled_open_fds()
747+
# Room for two more descriptors: a three-allocation export cannot complete.
748+
resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 2, hard))
749+
export_failed = False
750+
try:
751+
desc = buf.ipc_descriptor
752+
except CUDAError as exc:
753+
if not _has_note(exc, "file descriptor limit"):
754+
raise
755+
export_failed = True
756+
except ValueError as exc:
757+
if not is_orin or str(exc) != (
758+
f"Invalid allocation handle (fd) {-errno.EMFILE}: must be non-negative"
759+
):
760+
raise
761+
orin_export_bug = export_failed = True
762+
if export_failed:
763+
assert _open_fds() == baseline, "a failed export must close the descriptors it already took"
764+
else:
765+
# Acceptable only if the driver did not need one descriptor per allocation.
766+
assert len(desc.fds) == 3
767+
desc = None
768+
gc.collect()
769+
# Room for the three descriptors and the listing: export and import both succeed.
770+
resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 6, hard))
737771
desc = buf.ipc_descriptor
738-
except CUDAError as exc:
739-
failure = exc
740-
if failure is not None:
741-
assert _has_note(failure, "file descriptor limit")
742-
assert _open_fds() == baseline, "a failed export must close the descriptors it already took"
743-
else:
744-
# Acceptable only if the driver did not need one descriptor per allocation.
745772
assert len(desc.fds) == 3
773+
imported = buffers.enter_context(Buffer.from_ipc_descriptor(mr, desc, stream=stream))
774+
assert _open_fds() == baseline + 3, "the import must not take file descriptors of its own"
775+
finally:
776+
resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
777+
for fd in fillers:
778+
os.close(fd)
746779
desc = None
747780
gc.collect()
748-
# Room for the three descriptors and the listing: export and import both succeed.
749-
resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 6, hard))
750-
desc = buf.ipc_descriptor
751-
assert len(desc.fds) == 3
752-
imported = Buffer.from_ipc_descriptor(mr, desc, stream=stream)
753-
assert _open_fds() == baseline + 3, "the import must not take file descriptors of its own"
754-
finally:
755-
resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard))
756-
for fd in fillers:
757-
os.close(fd)
758-
assert [_read(imported, i * gran, 16) for i in range(3)] == [bytes([value]) * 16 for value in values]
759-
del desc
760-
gc.collect()
761-
for buffer in (imported, buf, second, first):
762-
buffer.close()
781+
assert [_read(imported, i * gran, 16) for i in range(3)] == [bytes([value]) * 16 for value in values]
782+
if orin_export_bug:
783+
raise _OrinIPCExportFDExhaustionError(xfail_reason)
763784

764785

765786
class TestRawFileDescriptorTransport:

0 commit comments

Comments
 (0)