Repository navigation
Expand file tree
/
Copy path_transfer.py
More file actions
98 lines (81 loc) · 3.75 KB
/
Copy path_transfer.py
File metadata and controls
98 lines (81 loc) · 3.75 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
"""Device-to-host copies that run behind the GPU work queued after them."""
from __future__ import annotations
import threading
from typing import NamedTuple
import cupy as cp
import numpy as np
from cunibs.solver import host_copy_async
# Two slots, so one drains into its host array while the other fills.
_SLOT_BYTES = 64 << 20
_SLOTS = 2
class Fence(NamedTuple):
"""The point in the copy queue a set of events marks, and how many copies precede it."""
events: list[cp.cuda.Event]
copies: int
class AsyncCopier:
"""Copies device arrays into new host arrays through a pinned ring, off the calling thread.
A copy waits for the GPU work that produced its source, then moves it a slot at a time: the
transfer runs on the copy engine and the slot drains into the host array on the CUDA driver's
callback thread. Each slot has its own stream, so its next transfer waits in stream order for
its last drain.
The copier keeps every source and destination alive until a :meth:`settle` or :meth:`drain`
has seen its copy land, so dropping a result early can never free memory the callback thread
is still writing.
"""
def __init__(self) -> None:
# Each slot is an allocation of its own, released with the copier. CuPy's pinned pool
# would keep the page-locked memory for the life of the process.
self._slots = [
cp.cuda.PinnedMemoryPointer(cp.cuda.PinnedMemory(_SLOT_BYTES), 0)
for _ in range(_SLOTS)
]
self._streams = [cp.cuda.Stream(non_blocking=True) for _ in range(_SLOTS)]
self._turn = 0
self._lock = threading.Lock()
# (sequence number, destination, source) for every copy not yet known to have landed.
self._inflight: list[tuple[int, np.ndarray, cp.ndarray]] = []
self._queued = 0
def copy(self, src: cp.ndarray, after: cp.cuda.Event) -> np.ndarray:
"""A host array holding ``src`` once a later :meth:`fence` has been settled."""
if not src.flags.c_contiguous:
raise ValueError("AsyncCopier.copy needs a C-contiguous source")
dst = np.empty(src.shape, dtype=src.dtype)
with self._lock:
self._queued += 1
self._inflight.append((self._queued, dst, src))
for stream in self._streams:
stream.wait_event(after)
offset = 0
while offset < src.nbytes:
i = self._turn
self._turn = (i + 1) % _SLOTS
n = min(_SLOT_BYTES, src.nbytes - offset)
stream, slot = self._streams[i], self._slots[i].ptr
cp.cuda.runtime.memcpyAsync(
slot,
src.data.ptr + offset,
n,
cp.cuda.runtime.memcpyDeviceToHost,
stream.ptr,
)
host_copy_async(slot, dst.ctypes.data + offset, n, stream.ptr)
offset += n
return dst
def fence(self) -> Fence:
"""A marker that is passed once every copy queued so far has landed."""
with self._lock:
events = []
for stream in self._streams:
event = cp.cuda.Event(disable_timing=True)
event.record(stream)
events.append(event)
return Fence(events, self._queued)
def settle(self, fence: Fence) -> None:
"""Wait for ``fence`` and release the buffers of the copies before it."""
for event in fence.events:
event.synchronize()
with self._lock:
self._inflight = [entry for entry in self._inflight if entry[0] > fence.copies]
def drain(self) -> None:
"""Wait for every queued copy and release all of their buffers."""
self.settle(self.fence())