Skip to content

Commit 942184b

Browse files
apacheGH-42222: [Python] Add bindings for CopyTo on RecordBatch and Array classes
1 parent 89d6354 commit 942184b

7 files changed

Lines changed: 91 additions & 50 deletions

File tree

‎python/pyarrow/array.pxi‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1658,6 +1658,30 @@ cdef class Array(_PandasConvertible):
16581658
_append_array_buffers(self.sp_array.get().data().get(), res)
16591659
return res
16601660

1661+
def copy_to(self, MemoryManager memory_manager):
1662+
"""
1663+
Construct a copy of the array with all buffers on destination
1664+
Memory Manager
1665+
1666+
This method recursively copies the array's buffers and those of its
1667+
children onto the destination MemoryManager device and returns the
1668+
new Array.
1669+
1670+
Parameters
1671+
----------
1672+
memory_manager : pyarrow.MemoryManager
1673+
1674+
Returns
1675+
------
1676+
Array
1677+
"""
1678+
cdef:
1679+
shared_ptr[CArray] c_array
1680+
1681+
with nogil:
1682+
c_array = GetResultValue(self.ap.CopyTo(memory_manager.unwrap()))
1683+
return pyarrow_wrap_array(c_array)
1684+
16611685
def _export_to_c(self, out_ptr, out_schema_ptr=0):
16621686
"""
16631687
Export to a C ArrowArray struct, given its pointer.

‎python/pyarrow/device.pxi‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -130,6 +130,9 @@ cdef class MemoryManager(_Weakrefable):
130130
self.init(mm)
131131
return self
132132

133+
cdef inline shared_ptr[CMemoryManager] unwrap(self) nogil:
134+
return self.memory_manager
135+
133136
def __repr__(self):
134137
return "<pyarrow.MemoryManager device: {}>".format(
135138
frombytes(self.memory_manager.get().device().get().ToString())

‎python/pyarrow/includes/libarrow.pxd‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -234,6 +234,7 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
234234
CStatus Validate() const
235235
CStatus ValidateFull() const
236236
CResult[shared_ptr[CArray]] View(const shared_ptr[CDataType]& type)
237+
CResult[shared_ptr[CArray]] CopyTo(const shared_ptr[CMemoryManager]& to) const
237238

238239
shared_ptr[CArray] MakeArray(const shared_ptr[CArrayData]& data)
239240
CResult[shared_ptr[CArray]] MakeArrayOfNull(
@@ -1024,6 +1025,8 @@ cdef extern from "arrow/api.h" namespace "arrow" nogil:
10241025
shared_ptr[CRecordBatch] Slice(int64_t offset)
10251026
shared_ptr[CRecordBatch] Slice(int64_t offset, int64_t length)
10261027

1028+
CResult[shared_ptr[CRecordBatch]] CopyTo(const shared_ptr[CMemoryManager]& to) const
1029+
10271030
CResult[shared_ptr[CTensor]] ToTensor(c_bool null_to_nan, c_bool row_major,
10281031
CMemoryPool* pool) const
10291032

‎python/pyarrow/lib.pxd‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -543,6 +543,8 @@ cdef class MemoryManager(_Weakrefable):
543543
@staticmethod
544544
cdef wrap(const shared_ptr[CMemoryManager]& mm)
545545

546+
cdef inline shared_ptr[CMemoryManager] unwrap(self) nogil
547+
546548

547549
cdef class Buffer(_Weakrefable):
548550
cdef:

‎python/pyarrow/table.pxi‎

Lines changed: 23 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -3549,6 +3549,29 @@ cdef class RecordBatch(_Tabular):
35493549
row_major, pool))
35503550
return pyarrow_wrap_tensor(c_tensor)
35513551

3552+
def copy_to(self, MemoryManager memory_manager):
3553+
"""
3554+
Copy the entire RecordBatch to destination MemoryManager.
3555+
3556+
This copies each column of the record batch to create
3557+
a new record batch where all underlying buffers for the columns have
3558+
been copied to the destination MemoryManager.
3559+
3560+
Parameters
3561+
----------
3562+
memory_manager : pyarrow.MemoryManager
3563+
3564+
Returns
3565+
------
3566+
RecordBatch
3567+
"""
3568+
cdef:
3569+
shared_ptr[CRecordBatch] c_batch
3570+
3571+
with nogil:
3572+
c_batch = GetResultValue(self.batch.CopyTo(memory_manager.unwrap()))
3573+
return pyarrow_wrap_batch(c_batch)
3574+
35523575
def _export_to_c(self, out_ptr, out_schema_ptr=0):
35533576
"""
35543577
Export to a C ArrowArray struct, given its pointer.

‎python/pyarrow/tests/test_cuda.py‎

Lines changed: 23 additions & 50 deletions
Original file line numberDiff line numberDiff line change
@@ -794,21 +794,20 @@ def test_IPC(size):
794794
assert p.exitcode == 0
795795

796796

797-
def _arr_copy_to_host(carr):
798-
# TODO replace below with copy to device when exposed in python
799-
buffers = []
800-
for cbuf in carr.buffers():
801-
if cbuf is None:
802-
buffers.append(None)
803-
else:
804-
buf = global_context.foreign_buffer(
805-
cbuf.address, cbuf.size, cbuf
806-
).copy_to_host()
807-
buffers.append(buf)
808-
809-
child = pa.Array.from_buffers(carr.type.value_type, 3, buffers[2:])
810-
new = pa.Array.from_buffers(carr.type, 2, buffers[:2], children=[child])
811-
return new
797+
def test_copy_to():
798+
_, buf = make_random_buffer(size=10, target='device')
799+
mm_cuda = buf.memory_manager
800+
801+
arr = pa.array([0, 1, 2])
802+
arr_cuda = arr.copy_to(mm_cuda)
803+
assert not arr_cuda.buffers()[1].is_cpu
804+
assert arr_cuda.buffers()[1].device_type == pa.DeviceAllocationType.CUDA
805+
806+
batch = pa.record_batch({"col": arr})
807+
batch_cuda = batch.copy_to(mm_cuda)
808+
buf_cuda = batch_cuda["col"].buffers()[1]
809+
assert not buf_cuda.is_cpu
810+
assert buf_cuda.device_type == pa.DeviceAllocationType.CUDA
812811

813812

814813
def test_device_interface_array():
@@ -823,19 +822,10 @@ def test_device_interface_array():
823822
typ = pa.list_(pa.int32())
824823
arr = pa.array([[1], [2, 42]], type=typ)
825824

826-
# TODO replace below with copy to device when exposed in python
827-
cbuffers = []
828-
for buf in arr.buffers():
829-
if buf is None:
830-
cbuffers.append(None)
831-
else:
832-
cbuf = global_context.new_buffer(buf.size)
833-
cbuf.copy_from_host(buf, position=0, nbytes=buf.size)
834-
cbuffers.append(cbuf)
835-
836-
carr = pa.Array.from_buffers(typ, 2, cbuffers[:2], children=[
837-
pa.Array.from_buffers(typ.value_type, 3, cbuffers[2:])
838-
])
825+
# copy to device
826+
_, buf = make_random_buffer(size=10, target='device')
827+
mm_cuda = buf.memory_manager
828+
carr = arr.copy_to(mm_cuda)
839829

840830
# Type is known up front
841831
carr._export_to_c_device(ptr_array)
@@ -849,7 +839,7 @@ def test_device_interface_array():
849839
del carr
850840
carr_new = pa.Array._import_from_c_device(ptr_array, typ)
851841
assert carr_new.type == pa.list_(pa.int32())
852-
arr_new = _arr_copy_to_host(carr_new)
842+
arr_new = carr_new.copy_to(pa.default_cpu_memory_manager())
853843
assert arr_new.equals(arr)
854844

855845
del carr_new
@@ -858,15 +848,13 @@ def test_device_interface_array():
858848
pa.Array._import_from_c_device(ptr_array, typ)
859849

860850
# Schema is exported and imported at the same time
861-
carr = pa.Array.from_buffers(typ, 2, cbuffers[:2], children=[
862-
pa.Array.from_buffers(typ.value_type, 3, cbuffers[2:])
863-
])
851+
carr = arr.copy_to(mm_cuda)
864852
carr._export_to_c_device(ptr_array, ptr_schema)
865853
# Delete and recreate C++ objects from exported pointers
866854
del carr
867855
carr_new = pa.Array._import_from_c_device(ptr_array, ptr_schema)
868856
assert carr_new.type == pa.list_(pa.int32())
869-
arr_new = _arr_copy_to_host(carr_new)
857+
arr_new = carr_new.copy_to(pa.default_cpu_memory_manager())
870858
assert arr_new.equals(arr)
871859

872860
del carr_new
@@ -875,21 +863,6 @@ def test_device_interface_array():
875863
pa.Array._import_from_c_device(ptr_array, ptr_schema)
876864

877865

878-
def _batch_copy_to_host(cbatch):
879-
# TODO replace below with copy to device when exposed in python
880-
arrs = []
881-
for col in cbatch.columns:
882-
buffers = [
883-
global_context.foreign_buffer(buf.address, buf.size, buf).copy_to_host()
884-
if buf is not None else None
885-
for buf in col.buffers()
886-
]
887-
new = pa.Array.from_buffers(col.type, len(col), buffers)
888-
arrs.append(new)
889-
890-
return pa.RecordBatch.from_arrays(arrs, schema=cbatch.schema)
891-
892-
893866
def test_device_interface_batch_array():
894867
cffi = pytest.importorskip("pyarrow.cffi")
895868
ffi = cffi.ffi
@@ -916,7 +889,7 @@ def test_device_interface_batch_array():
916889
del cbatch
917890
cbatch_new = pa.RecordBatch._import_from_c_device(ptr_array, schema)
918891
assert cbatch_new.schema == schema
919-
batch_new = _batch_copy_to_host(cbatch_new)
892+
batch_new = cbatch_new.copy_to(pa.default_cpu_memory_manager())
920893
assert batch_new.equals(batch)
921894

922895
del cbatch_new
@@ -931,7 +904,7 @@ def test_device_interface_batch_array():
931904
del cbatch
932905
cbatch_new = pa.RecordBatch._import_from_c_device(ptr_array, ptr_schema)
933906
assert cbatch_new.schema == schema
934-
batch_new = _batch_copy_to_host(cbatch_new)
907+
batch_new = cbatch_new.copy_to(pa.default_cpu_memory_manager())
935908
assert batch_new.equals(batch)
936909

937910
del cbatch_new

‎python/pyarrow/tests/test_device.py‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -41,3 +41,16 @@ def test_buffer_device():
4141
assert buf.device.is_cpu
4242
assert buf.device == pa.default_cpu_memory_manager().device
4343
assert buf.memory_manager.is_cpu
44+
45+
46+
def test_copy_to():
47+
mm = pa.default_cpu_memory_manager()
48+
arr = pa.array([0, 1, 2])
49+
arr_copied = arr.copy_to(mm)
50+
assert arr.equals(arr_copied)
51+
assert arr.buffers()[1].address != arr_copied.buffers()[1].address
52+
53+
batch = pa.record_batch({"col": arr})
54+
batch_copied = batch.copy_to(mm)
55+
assert batch.equals(batch_copied)
56+
assert arr.buffers()[1].address != batch_copied["col"].buffers()[1].address

0 commit comments

Comments
 (0)