|
12 | 12 | """ |
13 | 13 |
|
14 | 14 | import ctypes |
| 15 | +import errno |
15 | 16 | import gc |
16 | 17 | import json |
17 | 18 | import multiprocessing as mp |
|
20 | 21 | import pickle |
21 | 22 | import socket |
22 | 23 | import threading |
| 24 | +from contextlib import ExitStack |
23 | 25 |
|
24 | 26 | import pytest |
25 | 27 | from helpers.child_processes import child_timeout_sec, kill_subprocesses, noop, run_child, warmup_multiprocessing |
@@ -700,66 +702,85 @@ def child_main(device_id, to_child, from_child): |
700 | 702 | buffer.close() |
701 | 703 |
|
702 | 704 |
|
| 705 | +class _OrinIPCExportFDExhaustionError(Exception): |
| 706 | + """nvbug 6977893: Orin export reports success with an invalid -EMFILE fd.""" |
| 707 | + |
| 708 | + |
703 | 709 | class TestFileDescriptorLimit: |
704 | 710 | @pytest.mark.agent_authored(model="claude-fable-5-1") |
705 | 711 | @pytest.mark.thread_unsafe(reason="changes the process-wide file descriptor limit") |
706 | | - def test_export_near_the_limit(self, vmm_ipc_device): |
| 712 | + def test_export_near_the_limit(self, vmm_ipc_device, request): |
707 | 713 | """Under a lowered RLIMIT_NOFILE an export either succeeds or fails with a readable error and no leak.""" |
708 | 714 | import resource # POSIX-only; a module-level import breaks collection on Windows |
709 | 715 |
|
710 | 716 | device = vmm_ipc_device |
| 717 | + is_orin = device.name == "Orin" |
| 718 | + xfail_reason = "Workaround for nvbug 6977893: Orin IPC export mishandles FD exhaustion" |
| 719 | + if is_orin: |
| 720 | + request.node.add_marker( |
| 721 | + pytest.mark.xfail(reason=xfail_reason, raises=_OrinIPCExportFDExhaustionError, strict=False) |
| 722 | + ) |
| 723 | + orin_export_bug = False |
711 | 724 | stream = device.default_stream |
712 | 725 | mr = _resource(device) |
713 | | - first = mr.allocate(1) |
714 | | - gran = first.size |
715 | | - second = mr.modify_allocation(first, 2 * gran) |
716 | | - buf = mr.modify_allocation(second, 3 * gran) |
717 | | - values = (0x61, 0x62, 0x63) |
718 | | - for i, value in enumerate(values): |
719 | | - _fill(buf, value, offset=i * gran, size=gran) |
720 | | - soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) |
721 | | - fillers = [] |
722 | | - imported = desc = None |
723 | | - try: |
724 | | - # Fill the holes in the descriptor table so that the limit alone decides how many more fit. |
725 | | - highest = max(int(name) for name in os.listdir("/proc/self/fd")) |
726 | | - while True: |
727 | | - fd = os.open(os.devnull, os.O_RDONLY) |
728 | | - if fd > highest: |
729 | | - os.close(fd) |
730 | | - break |
731 | | - fillers.append(fd) |
732 | | - baseline = _settled_open_fds() |
733 | | - # Room for two more descriptors: a three-allocation export cannot complete. |
734 | | - resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 2, hard)) |
735 | | - failure = None |
| 726 | + with ExitStack() as buffers: |
| 727 | + first = buffers.enter_context(mr.allocate(1)) |
| 728 | + gran = first.size |
| 729 | + second = buffers.enter_context(mr.modify_allocation(first, 2 * gran)) |
| 730 | + buf = buffers.enter_context(mr.modify_allocation(second, 3 * gran)) |
| 731 | + values = (0x61, 0x62, 0x63) |
| 732 | + for i, value in enumerate(values): |
| 733 | + _fill(buf, value, offset=i * gran, size=gran) |
| 734 | + soft, hard = resource.getrlimit(resource.RLIMIT_NOFILE) |
| 735 | + fillers = [] |
| 736 | + desc = None |
736 | 737 | try: |
| 738 | + # Fill the holes in the descriptor table so that the limit alone decides how many more fit. |
| 739 | + highest = max(int(name) for name in os.listdir("/proc/self/fd")) |
| 740 | + while True: |
| 741 | + fd = os.open(os.devnull, os.O_RDONLY) |
| 742 | + if fd > highest: |
| 743 | + os.close(fd) |
| 744 | + break |
| 745 | + fillers.append(fd) |
| 746 | + baseline = _settled_open_fds() |
| 747 | + # Room for two more descriptors: a three-allocation export cannot complete. |
| 748 | + resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 2, hard)) |
| 749 | + export_failed = False |
| 750 | + try: |
| 751 | + desc = buf.ipc_descriptor |
| 752 | + except CUDAError as exc: |
| 753 | + if not _has_note(exc, "file descriptor limit"): |
| 754 | + raise |
| 755 | + export_failed = True |
| 756 | + except ValueError as exc: |
| 757 | + if not is_orin or str(exc) != ( |
| 758 | + f"Invalid allocation handle (fd) {-errno.EMFILE}: must be non-negative" |
| 759 | + ): |
| 760 | + raise |
| 761 | + orin_export_bug = export_failed = True |
| 762 | + if export_failed: |
| 763 | + assert _open_fds() == baseline, "a failed export must close the descriptors it already took" |
| 764 | + else: |
| 765 | + # Acceptable only if the driver did not need one descriptor per allocation. |
| 766 | + assert len(desc.fds) == 3 |
| 767 | + desc = None |
| 768 | + gc.collect() |
| 769 | + # Room for the three descriptors and the listing: export and import both succeed. |
| 770 | + resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 6, hard)) |
737 | 771 | desc = buf.ipc_descriptor |
738 | | - except CUDAError as exc: |
739 | | - failure = exc |
740 | | - if failure is not None: |
741 | | - assert _has_note(failure, "file descriptor limit") |
742 | | - assert _open_fds() == baseline, "a failed export must close the descriptors it already took" |
743 | | - else: |
744 | | - # Acceptable only if the driver did not need one descriptor per allocation. |
745 | 772 | assert len(desc.fds) == 3 |
| 773 | + imported = buffers.enter_context(Buffer.from_ipc_descriptor(mr, desc, stream=stream)) |
| 774 | + assert _open_fds() == baseline + 3, "the import must not take file descriptors of its own" |
| 775 | + finally: |
| 776 | + resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) |
| 777 | + for fd in fillers: |
| 778 | + os.close(fd) |
746 | 779 | desc = None |
747 | 780 | gc.collect() |
748 | | - # Room for the three descriptors and the listing: export and import both succeed. |
749 | | - resource.setrlimit(resource.RLIMIT_NOFILE, (highest + 1 + 6, hard)) |
750 | | - desc = buf.ipc_descriptor |
751 | | - assert len(desc.fds) == 3 |
752 | | - imported = Buffer.from_ipc_descriptor(mr, desc, stream=stream) |
753 | | - assert _open_fds() == baseline + 3, "the import must not take file descriptors of its own" |
754 | | - finally: |
755 | | - resource.setrlimit(resource.RLIMIT_NOFILE, (soft, hard)) |
756 | | - for fd in fillers: |
757 | | - os.close(fd) |
758 | | - assert [_read(imported, i * gran, 16) for i in range(3)] == [bytes([value]) * 16 for value in values] |
759 | | - del desc |
760 | | - gc.collect() |
761 | | - for buffer in (imported, buf, second, first): |
762 | | - buffer.close() |
| 781 | + assert [_read(imported, i * gran, 16) for i in range(3)] == [bytes([value]) * 16 for value in values] |
| 782 | + if orin_export_bug: |
| 783 | + raise _OrinIPCExportFDExhaustionError(xfail_reason) |
763 | 784 |
|
764 | 785 |
|
765 | 786 | class TestRawFileDescriptorTransport: |
|
0 commit comments