Skip to content

Commit e2b0de2

Browse files
GH-41664: [C++][Python] PrettyPrint non-cpu data by copying to default CPU device (#42010)
### Rationale for this change The various Python reprs or the C++ `PrettyPrint` functions will currently just segfault when passing an object that has its data on a non-CPU device. In python, getting a segfault while displaying the object is very annoying, and so we should make this at least not crash. ### What changes are included in this PR? When we detect data on a non-CPU device passed to `PrettyPrint`, we copy the necessary part (the full Arrays for Array/RecordBatch, or the full chunks that are being printed for ChunkedArray/Table) to the default CPU device, and then use the existing print utilities as is on this copied subset. For large data, this can be potentially costly by copying a lot of data (but you can always avoid that by not printing the data), but for chunked data we will still only copy those chunks of the full dataset needed to print the object. Longer term, we should investigate if we can actually copy sliced arrays to a different device (with actual pruning of the buffers while copying): #43055 ### Are these changes tested? Yes ### Are there any user-facing changes? No * GitHub Issue: #41664 Lead-authored-by: Joris Van den Bossche <[email protected]> Co-authored-by: Felipe Oliveira Carvalho <[email protected]> Signed-off-by: Joris Van den Bossche <[email protected]>
1 parent 577cafb commit e2b0de2

4 files changed

Lines changed: 67 additions & 7 deletions

File tree

‎cpp/src/arrow/pretty_print.cc‎

Lines changed: 8 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -391,7 +391,14 @@ class ArrayPrinter : public PrettyPrinter {
391391
}
392392

393393
Status Print(const Array& array) {
394-
RETURN_NOT_OK(VisitArrayInline(array, this));
394+
if (array.device_type() != DeviceAllocationType::kCPU) {
395+
// GH-43055: ideally we only copy start/end slices from non-CPU memory
396+
// based on the window size that is being printed
397+
ARROW_ASSIGN_OR_RAISE(auto array_cpu, array.CopyTo(default_cpu_memory_manager()));
398+
RETURN_NOT_OK(VisitArrayInline(*array_cpu, this));
399+
} else {
400+
RETURN_NOT_OK(VisitArrayInline(array, this));
401+
}
395402
Flush();
396403
return Status::OK();
397404
}

‎python/pyarrow/array.pxi‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -1241,6 +1241,9 @@ cdef class Array(_PandasConvertible):
12411241
"""
12421242
Render a "pretty-printed" string representation of the Array.
12431243
1244+
Note: for data on a non-CPU device, the full array is copied to CPU
1245+
memory.
1246+
12441247
Parameters
12451248
----------
12461249
indent : int, default 2
@@ -1261,8 +1264,6 @@ cdef class Array(_PandasConvertible):
12611264
If the array should be rendered as a single line of text
12621265
or if each element should be on its own line.
12631266
"""
1264-
self._assert_cpu()
1265-
12661267
cdef:
12671268
c_string result
12681269
PrettyPrintOptions options

‎python/pyarrow/tests/test_array.py‎

Lines changed: 2 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -4035,6 +4035,8 @@ def test_non_cpu_array():
40354035
assert arr.is_cpu is False
40364036
assert len(arr) == 4
40374037
assert arr.slice(2, 2).offset == 2
4038+
assert repr(arr)
4039+
assert str(arr)
40384040

40394041
# TODO support DLPack for CUDA
40404042
with pytest.raises(NotImplementedError):
@@ -4065,10 +4067,6 @@ def test_non_cpu_array():
40654067
arr.get_total_buffer_size()
40664068
with pytest.raises(NotImplementedError):
40674069
[i for i in iter(arr)]
4068-
with pytest.raises(NotImplementedError):
4069-
repr(arr)
4070-
with pytest.raises(NotImplementedError):
4071-
str(arr)
40724070
with pytest.raises(NotImplementedError):
40734071
arr == arr2
40744072
with pytest.raises(NotImplementedError):

‎python/pyarrow/tests/test_cuda.py‎

Lines changed: 54 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -944,3 +944,57 @@ def test_device_interface_batch_array():
944944
with pytest.raises(ValueError,
945945
match="ArrowSchema describes non-struct type"):
946946
pa.RecordBatch._import_from_c_device(ptr_array, ptr_schema)
947+
948+
949+
def test_print_array():
950+
batch = make_recordbatch(10)
951+
cbuf = cuda.serialize_record_batch(batch, global_context)
952+
cbatch = cuda.read_record_batch(cbuf, batch.schema)
953+
arr = batch["f0"]
954+
carr = cbatch["f0"]
955+
assert str(carr) == str(arr)
956+
957+
batch = make_recordbatch(100)
958+
cbuf = cuda.serialize_record_batch(batch, global_context)
959+
cbatch = cuda.read_record_batch(cbuf, batch.schema)
960+
arr = batch["f0"]
961+
carr = cbatch["f0"]
962+
assert str(carr) == str(arr)
963+
964+
965+
def make_chunked_array(n_elements_per_chunk, n_chunks):
966+
arrs = []
967+
carrs = []
968+
for _ in range(n_chunks):
969+
batch = make_recordbatch(n_elements_per_chunk)
970+
cbuf = cuda.serialize_record_batch(batch, global_context)
971+
cbatch = cuda.read_record_batch(cbuf, batch.schema)
972+
arrs.append(batch["f0"])
973+
carrs.append(cbatch["f0"])
974+
975+
return pa.chunked_array(arrs), pa.chunked_array(carrs)
976+
977+
978+
def test_print_chunked_array():
979+
arr, carr = make_chunked_array(10, 3)
980+
assert str(carr) == str(arr)
981+
982+
arr, carr = make_chunked_array(100, 20)
983+
assert str(carr) == str(arr)
984+
985+
986+
def test_print_record_batch():
987+
batch = make_recordbatch(10)
988+
cbuf = cuda.serialize_record_batch(batch, global_context)
989+
cbatch = cuda.read_record_batch(cbuf, batch.schema)
990+
assert str(cbatch) == str(batch)
991+
992+
batch = make_recordbatch(100)
993+
cbuf = cuda.serialize_record_batch(batch, global_context)
994+
cbatch = cuda.read_record_batch(cbuf, batch.schema)
995+
assert str(cbatch) == str(batch)
996+
997+
998+
def test_print_table():
999+
_, table, _, ctable = make_table_cuda()
1000+
assert str(ctable) == str(table)

0 commit comments

Comments
 (0)