Skip to content

Commit 576d20d

Browse files
also accept device, expand testing
1 parent 6707d8e commit 576d20d

6 files changed

Lines changed: 74 additions & 25 deletions

File tree

‎python/pyarrow/array.pxi‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1702,28 +1702,39 @@ cdef class Array(_PandasConvertible):
17021702
_append_array_buffers(self.sp_array.get().data().get(), res)
17031703
return res
17041704

1705-
def copy_to(self, MemoryManager memory_manager):
1705+
def copy_to(self, destination):
17061706
"""
17071707
Construct a copy of the array with all buffers on destination
1708-
Memory Manager
1708+
device.
17091709
17101710
This method recursively copies the array's buffers and those of its
17111711
children onto the destination MemoryManager device and returns the
17121712
new Array.
17131713
17141714
Parameters
17151715
----------
1716-
memory_manager : pyarrow.MemoryManager
1716+
destination : pyarrow.MemoryManager or pyarrow.Device
17171717
17181718
Returns
17191719
------
17201720
Array
17211721
"""
17221722
cdef:
17231723
shared_ptr[CArray] c_array
1724+
shared_ptr[CMemoryManager] c_memory_manager
1725+
1726+
if isinstance(destination, Device):
1727+
c_memory_manager = (<Device>destination).unwrap().get().default_memory_manager()
1728+
elif isinstance(destination, MemoryManager):
1729+
c_memory_manager = (<MemoryManager>destination).unwrap()
1730+
else:
1731+
raise TypeError(
1732+
"Argument 'destination' has incorrect type (expected a "
1733+
f"pyarrow Device or MemoryManager, got {type(destination)})"
1734+
)
17241735

17251736
with nogil:
1726-
c_array = GetResultValue(self.ap.CopyTo(memory_manager.unwrap()))
1737+
c_array = GetResultValue(self.ap.CopyTo(c_memory_manager))
17271738
return pyarrow_wrap_array(c_array)
17281739

17291740
def _export_to_c(self, out_ptr, out_schema_ptr=0):

‎python/pyarrow/device.pxi‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -64,6 +64,9 @@ cdef class Device(_Weakrefable):
6464
self.init(device)
6565
return self
6666

67+
cdef inline shared_ptr[CDevice] unwrap(self) nogil:
68+
return self.device
69+
6770
def __eq__(self, other):
6871
if not isinstance(other, Device):
6972
return False

‎python/pyarrow/lib.pxd‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -534,6 +534,8 @@ cdef class Device(_Weakrefable):
534534
@staticmethod
535535
cdef wrap(const shared_ptr[CDevice]& device)
536536

537+
cdef inline shared_ptr[CDevice] unwrap(self) nogil
538+
537539

538540
cdef class MemoryManager(_Weakrefable):
539541
cdef:

‎python/pyarrow/table.pxi‎

Lines changed: 15 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -3569,27 +3569,38 @@ cdef class RecordBatch(_Tabular):
35693569
row_major, pool))
35703570
return pyarrow_wrap_tensor(c_tensor)
35713571

3572-
def copy_to(self, MemoryManager memory_manager):
3572+
def copy_to(self, destination):
35733573
"""
3574-
Copy the entire RecordBatch to destination MemoryManager.
3574+
Copy the entire RecordBatch to destination device.
35753575
35763576
This copies each column of the record batch to create
35773577
a new record batch where all underlying buffers for the columns have
35783578
been copied to the destination MemoryManager.
35793579
35803580
Parameters
35813581
----------
3582-
memory_manager : pyarrow.MemoryManager
3582+
destination : pyarrow.MemoryManager or pyarrow.Device
35833583
35843584
Returns
35853585
------
35863586
RecordBatch
35873587
"""
35883588
cdef:
35893589
shared_ptr[CRecordBatch] c_batch
3590+
shared_ptr[CMemoryManager] c_memory_manager
3591+
3592+
if isinstance(destination, Device):
3593+
c_memory_manager = (<Device>destination).unwrap().get().default_memory_manager()
3594+
elif isinstance(destination, MemoryManager):
3595+
c_memory_manager = (<MemoryManager>destination).unwrap()
3596+
else:
3597+
raise TypeError(
3598+
"Argument 'destination' has incorrect type (expected a "
3599+
f"pyarrow Device or MemoryManager, got {type(destination)})"
3600+
)
35903601

35913602
with nogil:
3592-
c_batch = GetResultValue(self.batch.CopyTo(memory_manager.unwrap()))
3603+
c_batch = GetResultValue(self.batch.CopyTo(c_memory_manager))
35933604
return pyarrow_wrap_batch(c_batch)
35943605

35953606
def _export_to_c(self, out_ptr, out_schema_ptr=0):

‎python/pyarrow/tests/test_cuda.py‎

Lines changed: 19 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -809,16 +809,25 @@ def test_copy_to():
809809
_, buf = make_random_buffer(size=10, target='device')
810810
mm_cuda = buf.memory_manager
811811

812-
arr = pa.array([0, 1, 2])
813-
arr_cuda = arr.copy_to(mm_cuda)
814-
assert not arr_cuda.buffers()[1].is_cpu
815-
assert arr_cuda.buffers()[1].device_type == pa.DeviceAllocationType.CUDA
816-
817-
batch = pa.record_batch({"col": arr})
818-
batch_cuda = batch.copy_to(mm_cuda)
819-
buf_cuda = batch_cuda["col"].buffers()[1]
820-
assert not buf_cuda.is_cpu
821-
assert buf_cuda.device_type == pa.DeviceAllocationType.CUDA
812+
for dest in [mm_cuda, mm_cuda.device]:
813+
arr = pa.array([0, 1, 2])
814+
arr_cuda = arr.copy_to(dest)
815+
assert not arr_cuda.buffers()[1].is_cpu
816+
assert arr_cuda.buffers()[1].device_type == pa.DeviceAllocationType.CUDA
817+
assert arr_cuda.buffers()[1].device == mm_cuda.device
818+
819+
arr_roundtrip = arr_cuda.copy_to(pa.default_cpu_memory_manager())
820+
assert arr_roundtrip.equals(arr)
821+
822+
batch = pa.record_batch({"col": arr})
823+
batch_cuda = batch.copy_to(dest)
824+
buf_cuda = batch_cuda["col"].buffers()[1]
825+
assert not buf_cuda.is_cpu
826+
assert buf_cuda.device_type == pa.DeviceAllocationType.CUDA
827+
assert buf_cuda.device == mm_cuda.device
828+
829+
batch_roundtrip = batch_cuda.copy_to(pa.default_cpu_memory_manager())
830+
assert batch_roundtrip.equals(batch)
822831

823832

824833
def test_device_interface_array():

‎python/pyarrow/tests/test_device.py‎

Lines changed: 20 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -17,6 +17,8 @@
1717

1818
import pyarrow as pa
1919

20+
import pytest
21+
2022

2123
def test_device_memory_manager():
2224
mm = pa.default_cpu_memory_manager()
@@ -45,12 +47,23 @@ def test_buffer_device():
4547

4648
def test_copy_to():
4749
mm = pa.default_cpu_memory_manager()
48-
arr = pa.array([0, 1, 2])
49-
arr_copied = arr.copy_to(mm)
50-
assert arr.equals(arr_copied)
51-
assert arr.buffers()[1].address != arr_copied.buffers()[1].address
5250

51+
arr = pa.array([0, 1, 2])
5352
batch = pa.record_batch({"col": arr})
54-
batch_copied = batch.copy_to(mm)
55-
assert batch.equals(batch_copied)
56-
assert arr.buffers()[1].address != batch_copied["col"].buffers()[1].address
53+
54+
for dest in [mm, mm.device]:
55+
arr_copied = arr.copy_to(dest)
56+
assert arr_copied.equals(arr)
57+
assert arr_copied.buffers()[1].device == mm.device
58+
assert arr_copied.buffers()[1].address != arr.buffers()[1].address
59+
60+
batch_copied = batch.copy_to(dest)
61+
assert batch_copied.equals(batch)
62+
assert batch_copied["col"].buffers()[1].device == mm.device
63+
assert batch_copied["col"].buffers()[1].address != arr.buffers()[1].address
64+
65+
with pytest.raises(TypeError, match="Argument 'destination' has incorrect type"):
66+
arr.copy_to(mm.device.device_type)
67+
68+
with pytest.raises(TypeError, match="Argument 'destination' has incorrect type"):
69+
batch.copy_to(mm.device.device_type)

0 commit comments

Comments
 (0)