Skip to content

Commit ffee537

Browse files
GH-42222: [Python] Add bindings for CopyTo on RecordBatch and Array classes (#42223)
### Rationale for this change We have added bindings for the Device and MemoryManager classes (#41126), and as a next step we can expose the functionality to copy a full Array or RecordBatch to a specific memory manager. ### What changes are included in this PR? This adds a `copy_to` method on pyarrow Array and RecordBatch. ### Are these changes tested? Yes * GitHub Issue: #42222 Authored-by: Joris Van den Bossche <[email protected]> Signed-off-by: Joris Van den Bossche <[email protected]>
1 parent e1e7c50 commit ffee537

7 files changed

Lines changed: 143 additions & 50 deletions

File tree

‎python/pyarrow/array.pxi‎

Lines changed: 36 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1702,6 +1702,42 @@ cdef class Array(_PandasConvertible):
17021702
_append_array_buffers(self.sp_array.get().data().get(), res)
17031703
return res
17041704

1705+
def copy_to(self, destination):
1706+
"""
1707+
Construct a copy of the array with all buffers on destination
1708+
device.
1709+
1710+
This method recursively copies the array's buffers and those of its
1711+
children onto the destination MemoryManager device and returns the
1712+
new Array.
1713+
1714+
Parameters
1715+
----------
1716+
destination : pyarrow.MemoryManager or pyarrow.Device
1717+
The destination device to copy the array to.
1718+
1719+
Returns
1720+
-------
1721+
Array
1722+
"""
1723+
cdef:
1724+
shared_ptr[CArray] c_array
1725+
shared_ptr[CMemoryManager] c_memory_manager
1726+
1727+
if isinstance(destination, Device):
1728+
c_memory_manager = (<Device>destination).unwrap().get().default_memory_manager()
1729+
elif isinstance(destination, MemoryManager):
1730+
c_memory_manager = (<MemoryManager>destination).unwrap()
1731+
else:
1732+
raise TypeError(
1733+
"Argument 'destination' has incorrect type (expected a "
1734+
f"pyarrow Device or MemoryManager, got {type(destination)})"
1735+
)
1736+
1737+
with nogil:
1738+
c_array = GetResultValue(self.ap.CopyTo(c_memory_manager))
1739+
return pyarrow_wrap_array(c_array)
1740+
17051741
def _export_to_c(self, out_ptr, out_schema_ptr=0):
17061742
"""
17071743
Export to a C ArrowArray struct, given its pointer.

‎python/pyarrow/device.pxi‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,9 @@ cdef class Device(_Weakrefable):
6464
self.init(device)
6565
return self
6666

67+
cdef inline shared_ptr[CDevice] unwrap(self) nogil:
68+
return self.device
69+
6770
def __eq__(self, other):
6871
if not isinstance(other, Device):
6972
return False
@@ -130,6 +133,9 @@ cdef class MemoryManager(_Weakrefable):
130133
self.init(mm)
131134
return self
132135

136+
cdef inline shared_ptr[CMemoryManager] unwrap(self) nogil:
137+
return self.memory_manager
138+
133139
def __repr__(self):
134140
return "<pyarrow.MemoryManager device: {}>".format(
135141
frombytes(self.memory_manager.get().device().get().ToString())

‎python/pyarrow/includes/libarrow.pxd‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,7 +234,9 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
234234
CStatus Validate() const
235235
CStatus ValidateFull() const
236236
CResult[shared_ptr[CArray]] View(const shared_ptr[CDataType]& type)
237+
237238
CDeviceAllocationType device_type()
239+
CResult[shared_ptr[CArray]] CopyTo(const shared_ptr[CMemoryManager]& to) const
238240

239241
shared_ptr[CArray] MakeArray(const shared_ptr[CArrayData]& data)
240242
CResult[shared_ptr[CArray]] MakeArrayOfNull(
@@ -1027,6 +1029,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
10271029
shared_ptr[CRecordBatch] Slice(int64_t offset)
10281030
shared_ptr[CRecordBatch] Slice(int64_t offset, int64_t length)
10291031

1032+
CResult[shared_ptr[CRecordBatch]] CopyTo(const shared_ptr[CMemoryManager]& to) const
1033+
10301034
CResult[shared_ptr[CTensor]] ToTensor(c_bool null_to_nan, c_bool row_major,
10311035
CMemoryPool* pool) const
10321036

‎python/pyarrow/lib.pxd‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -542,6 +542,8 @@ cdef class Device(_Weakrefable):
542542
@staticmethod
543543
cdef wrap(const shared_ptr[CDevice]& device)
544544

545+
cdef inline shared_ptr[CDevice] unwrap(self) nogil
546+
545547

546548
cdef class MemoryManager(_Weakrefable):
547549
cdef:
@@ -552,6 +554,8 @@ cdef class MemoryManager(_Weakrefable):
552554
@staticmethod
553555
cdef wrap(const shared_ptr[CMemoryManager]& mm)
554556

557+
cdef inline shared_ptr[CMemoryManager] unwrap(self) nogil
558+
555559

556560
cdef class Buffer(_Weakrefable):
557561
cdef:

‎python/pyarrow/table.pxi‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3569,6 +3569,41 @@ cdef class RecordBatch(_Tabular):
35693569
row_major, pool))
35703570
return pyarrow_wrap_tensor(c_tensor)
35713571

3572+
def copy_to(self, destination):
3573+
"""
3574+
Copy the entire RecordBatch to destination device.
3575+
3576+
This copies each column of the record batch to create
3577+
a new record batch where all underlying buffers for the columns have
3578+
been copied to the destination MemoryManager.
3579+
3580+
Parameters
3581+
----------
3582+
destination : pyarrow.MemoryManager or pyarrow.Device
3583+
The destination device to copy the array to.
3584+
3585+
Returns
3586+
-------
3587+
RecordBatch
3588+
"""
3589+
cdef:
3590+
shared_ptr[CRecordBatch] c_batch
3591+
shared_ptr[CMemoryManager] c_memory_manager
3592+
3593+
if isinstance(destination, Device):
3594+
c_memory_manager = (<Device>destination).unwrap().get().default_memory_manager()
3595+
elif isinstance(destination, MemoryManager):
3596+
c_memory_manager = (<MemoryManager>destination).unwrap()
3597+
else:
3598+
raise TypeError(
3599+
"Argument 'destination' has incorrect type (expected a "
3600+
f"pyarrow Device or MemoryManager, got {type(destination)})"
3601+
)
3602+
3603+
with nogil:
3604+
c_batch = GetResultValue(self.batch.CopyTo(c_memory_manager))
3605+
return pyarrow_wrap_batch(c_batch)
3606+
35723607
def _export_to_c(self, out_ptr, out_schema_ptr=0):
35733608
"""
35743609
Export to a C ArrowArray struct, given its pointer.

‎python/pyarrow/tests/test_cuda.py‎

Lines changed: 32 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -827,21 +827,29 @@ def test_IPC(size):
827827
assert p.exitcode == 0
828828

829829

830-
def _arr_copy_to_host(carr):
831-
# TODO replace below with copy to device when exposed in python
832-
buffers = []
833-
for cbuf in carr.buffers():
834-
if cbuf is None:
835-
buffers.append(None)
836-
else:
837-
buf = global_context.foreign_buffer(
838-
cbuf.address, cbuf.size, cbuf
839-
).copy_to_host()
840-
buffers.append(buf)
841-
842-
child = pa.Array.from_buffers(carr.type.value_type, 3, buffers[2:])
843-
new = pa.Array.from_buffers(carr.type, 2, buffers[:2], children=[child])
844-
return new
830+
def test_copy_to():
831+
_, buf = make_random_buffer(size=10, target='device')
832+
mm_cuda = buf.memory_manager
833+
834+
for dest in [mm_cuda, mm_cuda.device]:
835+
arr = pa.array([0, 1, 2])
836+
arr_cuda = arr.copy_to(dest)
837+
assert not arr_cuda.buffers()[1].is_cpu
838+
assert arr_cuda.buffers()[1].device_type == pa.DeviceAllocationType.CUDA
839+
assert arr_cuda.buffers()[1].device == mm_cuda.device
840+
841+
arr_roundtrip = arr_cuda.copy_to(pa.default_cpu_memory_manager())
842+
assert arr_roundtrip.equals(arr)
843+
844+
batch = pa.record_batch({"col": arr})
845+
batch_cuda = batch.copy_to(dest)
846+
buf_cuda = batch_cuda["col"].buffers()[1]
847+
assert not buf_cuda.is_cpu
848+
assert buf_cuda.device_type == pa.DeviceAllocationType.CUDA
849+
assert buf_cuda.device == mm_cuda.device
850+
851+
batch_roundtrip = batch_cuda.copy_to(pa.default_cpu_memory_manager())
852+
assert batch_roundtrip.equals(batch)
845853

846854

847855
def test_device_interface_array():
@@ -856,19 +864,10 @@ def test_device_interface_array():
856864
typ = pa.list_(pa.int32())
857865
arr = pa.array([[1], [2, 42]], type=typ)
858866

859-
# TODO replace below with copy to device when exposed in python
860-
cbuffers = []
861-
for buf in arr.buffers():
862-
if buf is None:
863-
cbuffers.append(None)
864-
else:
865-
cbuf = global_context.new_buffer(buf.size)
866-
cbuf.copy_from_host(buf, position=0, nbytes=buf.size)
867-
cbuffers.append(cbuf)
868-
869-
carr = pa.Array.from_buffers(typ, 2, cbuffers[:2], children=[
870-
pa.Array.from_buffers(typ.value_type, 3, cbuffers[2:])
871-
])
867+
# copy to device
868+
_, buf = make_random_buffer(size=10, target='device')
869+
mm_cuda = buf.memory_manager
870+
carr = arr.copy_to(mm_cuda)
872871

873872
# Type is known up front
874873
carr._export_to_c_device(ptr_array)
@@ -882,7 +881,7 @@ def test_device_interface_array():
882881
del carr
883882
carr_new = pa.Array._import_from_c_device(ptr_array, typ)
884883
assert carr_new.type == pa.list_(pa.int32())
885-
arr_new = _arr_copy_to_host(carr_new)
884+
arr_new = carr_new.copy_to(pa.default_cpu_memory_manager())
886885
assert arr_new.equals(arr)
887886

888887
del carr_new
@@ -891,15 +890,13 @@ def test_device_interface_array():
891890
pa.Array._import_from_c_device(ptr_array, typ)
892891

893892
# Schema is exported and imported at the same time
894-
carr = pa.Array.from_buffers(typ, 2, cbuffers[:2], children=[
895-
pa.Array.from_buffers(typ.value_type, 3, cbuffers[2:])
896-
])
893+
carr = arr.copy_to(mm_cuda)
897894
carr._export_to_c_device(ptr_array, ptr_schema)
898895
# Delete and recreate C++ objects from exported pointers
899896
del carr
900897
carr_new = pa.Array._import_from_c_device(ptr_array, ptr_schema)
901898
assert carr_new.type == pa.list_(pa.int32())
902-
arr_new = _arr_copy_to_host(carr_new)
899+
arr_new = carr_new.copy_to(pa.default_cpu_memory_manager())
903900
assert arr_new.equals(arr)
904901

905902
del carr_new
@@ -908,21 +905,6 @@ def test_device_interface_array():
908905
pa.Array._import_from_c_device(ptr_array, ptr_schema)
909906

910907

911-
def _batch_copy_to_host(cbatch):
912-
# TODO replace below with copy to device when exposed in python
913-
arrs = []
914-
for col in cbatch.columns:
915-
buffers = [
916-
global_context.foreign_buffer(buf.address, buf.size, buf).copy_to_host()
917-
if buf is not None else None
918-
for buf in col.buffers()
919-
]
920-
new = pa.Array.from_buffers(col.type, len(col), buffers)
921-
arrs.append(new)
922-
923-
return pa.RecordBatch.from_arrays(arrs, schema=cbatch.schema)
924-
925-
926908
def test_device_interface_batch_array():
927909
cffi = pytest.importorskip("pyarrow.cffi")
928910
ffi = cffi.ffi
@@ -949,7 +931,7 @@ def test_device_interface_batch_array():
949931
del cbatch
950932
cbatch_new = pa.RecordBatch._import_from_c_device(ptr_array, schema)
951933
assert cbatch_new.schema == schema
952-
batch_new = _batch_copy_to_host(cbatch_new)
934+
batch_new = cbatch_new.copy_to(pa.default_cpu_memory_manager())
953935
assert batch_new.equals(batch)
954936

955937
del cbatch_new
@@ -964,7 +946,7 @@ def test_device_interface_batch_array():
964946
del cbatch
965947
cbatch_new = pa.RecordBatch._import_from_c_device(ptr_array, ptr_schema)
966948
assert cbatch_new.schema == schema
967-
batch_new = _batch_copy_to_host(cbatch_new)
949+
batch_new = cbatch_new.copy_to(pa.default_cpu_memory_manager())
968950
assert batch_new.equals(batch)
969951

970952
del cbatch_new

‎python/pyarrow/tests/test_device.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
import pyarrow as pa
1919

20+
import pytest
21+
2022

2123
def test_device_memory_manager():
2224
mm = pa.default_cpu_memory_manager()
@@ -41,3 +43,27 @@ def test_buffer_device():
4143
assert buf.device.is_cpu
4244
assert buf.device == pa.default_cpu_memory_manager().device
4345
assert buf.memory_manager.is_cpu
46+
47+
48+
def test_copy_to():
49+
mm = pa.default_cpu_memory_manager()
50+
51+
arr = pa.array([0, 1, 2])
52+
batch = pa.record_batch({"col": arr})
53+
54+
for dest in [mm, mm.device]:
55+
arr_copied = arr.copy_to(dest)
56+
assert arr_copied.equals(arr)
57+
assert arr_copied.buffers()[1].device == mm.device
58+
assert arr_copied.buffers()[1].address != arr.buffers()[1].address
59+
60+
batch_copied = batch.copy_to(dest)
61+
assert batch_copied.equals(batch)
62+
assert batch_copied["col"].buffers()[1].device == mm.device
63+
assert batch_copied["col"].buffers()[1].address != arr.buffers()[1].address
64+
65+
with pytest.raises(TypeError, match="Argument 'destination' has incorrect type"):
66+
arr.copy_to(mm.device.device_type)
67+
68+
with pytest.raises(TypeError, match="Argument 'destination' has incorrect type"):
69+
batch.copy_to(mm.device.device_type)

0 commit comments

Comments
 (0)