Skip to content

Commit 0f79a3a

Browse files
bonpytbonpyt
andauthored
BUG: Fix copy performance regression for structured arrays (#29270)
This adds a new dtype flag used by structured dtypes to indicate that they _cannot_ trivially copy (`NPY_NOT_TRIVIALLY_COPYABLE`). This way, dtypes that do not set this (and _also_ don't have the references flag set) can be assumed to be trivially copyable, which should allow certain default implementations/fast-path improvements elsewhere as well. The current small caveat is that this flag cannot be unpickled yet, so it isn't pickled (safe but slower on round-trip) and it may also make sense to allow setting it explicitly. Co-authored-by: bonpyt <[email protected]>
1 parent 200f11c commit 0f79a3a

6 files changed

Lines changed: 90 additions & 6 deletions

File tree

‎benchmarks/benchmarks/bench_io.py‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -60,6 +60,32 @@ def time_copyto_8_dense(self):
6060
np.copyto(self.d, self.e, where=self.im8)
6161

6262

63+
class CopyStructured(Benchmark):
64+
params = [
65+
[10, 100, 1000],
66+
[
67+
# Contiguous layout (trivially copyable, uses memcpy)
68+
[('a', 'f8'), ('b', 'f8'), ('c', 'f8')],
69+
# Many small fields (trivially copyable, uses memcpy)
70+
[('x', 'i4'), ('y', 'i4'), ('z', 'i4'),
71+
('w', 'i4'), ('v', 'i4')],
72+
],
73+
]
74+
param_names = ['n_kilo_elements', 'dtype']
75+
76+
def setup(self, n_kilo, dtype):
77+
n = n_kilo * 1000
78+
self.dt = np.dtype(dtype)
79+
self.src = np.ones(n, dtype=self.dt)
80+
self.dst = np.empty(n, dtype=self.dt)
81+
82+
def time_copy(self, n_kilo, dtype):
83+
self.src.copy()
84+
85+
def time_assign(self, n_kilo, dtype):
86+
self.dst[...] = self.src
87+
88+
6389
class Savez(Benchmark):
6490
def setup(self):
6591
self.squares = get_squares()
Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
Structured array copies now use ``memcpy`` for contiguous dtypes
2+
----------------------------------------------------------------
3+
Copying structured arrays with identical dtypes now uses ``memcpy`` instead of
4+
field-by-field transfer when the dtype has a contiguous layout (no gaps between
5+
fields). A new ``NPY_NOT_TRIVIALLY_COPYABLE`` dtype flag is set on structured
6+
dtypes that have gaps in their memory layout, such as those created with
7+
explicit ``offsets`` or via multi-field indexing. Only these dtypes continue to
8+
use the slower field-by-field copy.
9+
10+
This means that padding bytes in contiguous structured dtypes (e.g. those
11+
created without explicit ``offsets``) may now be copied as part of the
12+
``memcpy``, whereas previously they were left untouched. Code that relies on
13+
padding bytes being preserved during structured array copies may be affected.

‎numpy/_core/include/numpy/ndarrayobject.h‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -253,6 +253,9 @@ PyArray_ITEMSIZE(const PyArrayObject *arr)
253253
#define PyDataType_REFCHK(dtype) \
254254
PyDataType_FLAGCHK(dtype, NPY_ITEM_REFCOUNT)
255255

256+
#define PyDataType_ISTRIVIALLYCOPYABLE(dtype) \
257+
(!(PyDataType_FLAGS(dtype) & (NPY_NOT_TRIVIALLY_COPYABLE | NPY_ITEM_REFCOUNT)))
258+
256259
#define NPY_BEGIN_THREADS_DESCR(dtype) \
257260
do {if (!(PyDataType_FLAGCHK((dtype), NPY_NEEDS_PYAPI))) \
258261
NPY_BEGIN_THREADS;} while (0);

‎numpy/_core/include/numpy/ndarraytypes.h‎

Lines changed: 4 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -596,13 +596,16 @@ typedef struct {
596596
#define NPY_USE_SETITEM 0x40
597597
/* A sticky flag specifically for structured arrays */
598598
#define NPY_ALIGNED_STRUCT 0x80
599+
/* Structured dtype has non-contiguous field layout */
600+
#define NPY_NOT_TRIVIALLY_COPYABLE 0x100
599601

600602
/*
601603
*These are inherited for global data-type if any data-types in the
602604
* field have them
603605
*/
604606
#define NPY_FROM_FIELDS (NPY_NEEDS_INIT | NPY_LIST_PICKLE | \
605-
NPY_ITEM_REFCOUNT | NPY_NEEDS_PYAPI)
607+
NPY_ITEM_REFCOUNT | NPY_NEEDS_PYAPI | \
608+
NPY_NOT_TRIVIALLY_COPYABLE)
606609

607610
#define NPY_OBJECT_DTYPE_FLAGS (NPY_LIST_PICKLE | NPY_USE_GETITEM | \
608611
NPY_ITEM_IS_POINTER | NPY_ITEM_REFCOUNT | \

‎numpy/_core/src/multiarray/convert_datatype.c‎

Lines changed: 16 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3281,7 +3281,22 @@ void_to_void_get_loop(
32813281
{
32823282
if (PyDataType_NAMES(context->descriptors[0]) != NULL ||
32833283
PyDataType_NAMES(context->descriptors[1]) != NULL) {
3284-
if (get_fields_transfer_function(
3284+
/*
3285+
* Fast path: if dtypes are equivalent and the destination is
3286+
* trivially copyable, use memcpy instead of field-by-field transfer.
3287+
*/
3288+
if ((context->descriptors[0] == context->descriptors[1] ||
3289+
PyArray_EquivTypes(context->descriptors[0], context->descriptors[1])) &&
3290+
PyDataType_ISTRIVIALLYCOPYABLE(context->descriptors[1])) {
3291+
if (PyArray_GetStridedZeroPadCopyFn(
3292+
0, 0, strides[0], strides[1],
3293+
context->descriptors[0]->elsize, context->descriptors[1]->elsize,
3294+
out_loop, out_transferdata) == NPY_FAIL) {
3295+
return -1;
3296+
}
3297+
*flags = PyArrayMethod_MINIMAL_FLAGS;
3298+
}
3299+
else if (get_fields_transfer_function(
32853300
aligned, strides[0], strides[1],
32863301
context->descriptors[0], context->descriptors[1],
32873302
move_references, out_loop, out_transferdata,

‎numpy/_core/src/multiarray/descriptor.c‎

Lines changed: 28 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -408,7 +408,7 @@ _convert_from_array_descr(PyObject *obj, int align)
408408
}
409409

410410
/* Types with fields need the Python C API for field access */
411-
char dtypeflags = NPY_NEEDS_PYAPI;
411+
npy_uint64 dtypeflags = NPY_NEEDS_PYAPI;
412412
int maxalign = 1;
413413
int totalsize = 0;
414414
PyObject *fields = PyDict_New();
@@ -631,7 +631,7 @@ _convert_from_list(PyObject *obj, int align)
631631
}
632632

633633
/* Types with fields need the Python C API for field access */
634-
char dtypeflags = NPY_NEEDS_PYAPI;
634+
npy_uint64 dtypeflags = NPY_NEEDS_PYAPI;
635635
int maxalign = 1;
636636
int totalsize = 0;
637637
for (int i = 0; i < n; i++) {
@@ -1100,7 +1100,7 @@ _convert_from_dict(PyObject *obj, int align)
11001100
}
11011101

11021102
/* Types with fields need the Python C API for field access */
1103-
char dtypeflags = NPY_NEEDS_PYAPI;
1103+
npy_uint64 dtypeflags = NPY_NEEDS_PYAPI;
11041104
int totalsize = 0;
11051105
int maxalign = 1;
11061106
int has_out_of_order_fields = 0;
@@ -1319,6 +1319,16 @@ _convert_from_dict(PyObject *obj, int align)
13191319
new->elsize = itemsize;
13201320
}
13211321

1322+
/*
1323+
* Mark as not trivially copyable only if explicit offsets were provided
1324+
* and the layout has holes. Alignment padding (without explicit offsets)
1325+
* is safe to overwrite with memcpy.
1326+
*/
1327+
if (offsets != NULL &&
1328+
!is_dtype_struct_simple_unaligned_layout((PyArray_Descr *)new)) {
1329+
new->flags |= NPY_NOT_TRIVIALLY_COPYABLE;
1330+
}
1331+
13221332
/* Add the metadata if provided */
13231333
PyObject *metadata = PyMapping_GetItemString(obj, "metadata");
13241334

@@ -2817,7 +2827,8 @@ arraydescr_reduce(PyArray_Descr *self, PyObject *NPY_UNUSED(args))
28172827
}
28182828
PyTuple_SET_ITEM(state, 5, PyLong_FromLong(elsize));
28192829
PyTuple_SET_ITEM(state, 6, PyLong_FromLong(alignment));
2820-
PyTuple_SET_ITEM(state, 7, PyLong_FromUnsignedLongLong(self->flags));
2830+
PyTuple_SET_ITEM(state, 7, PyLong_FromUnsignedLongLong(
2831+
self->flags & ~NPY_NOT_TRIVIALLY_COPYABLE));
28212832

28222833
PyTuple_SET_ITEM(ret, 2, state);
28232834
return ret;
@@ -3187,6 +3198,15 @@ arraydescr_setstate(_PyArray_LegacyDescr *self, PyObject *args)
31873198
self->flags = _descr_find_object((PyArray_Descr *)self);
31883199
}
31893200

3201+
/*
3202+
* Mark as not trivially copyable if layout is not simple (has padding).
3203+
* This flag is always recomputed on unpickle (not stored in pickle).
3204+
*/
3205+
if (PyDataType_HASFIELDS((PyArray_Descr *)self) &&
3206+
!is_dtype_struct_simple_unaligned_layout((PyArray_Descr *)self)) {
3207+
self->flags |= NPY_NOT_TRIVIALLY_COPYABLE;
3208+
}
3209+
31903210
PyObject *old_metadata, *new_metadata;
31913211
if (PyDataType_ISDATETIME(self)) {
31923212
PyArray_DatetimeMetaData temp_dt_data;
@@ -3755,6 +3775,10 @@ arraydescr_field_subset_view(_PyArray_LegacyDescr *self, PyObject *ind)
37553775
view_dtype->names = names;
37563776
view_dtype->fields = fields;
37573777
view_dtype->flags = self->flags;
3778+
/* Mark as not trivially copyable if layout is not simple (has padding) */
3779+
if (!is_dtype_struct_simple_unaligned_layout((PyArray_Descr *)view_dtype)) {
3780+
view_dtype->flags |= NPY_NOT_TRIVIALLY_COPYABLE;
3781+
}
37583782
return (PyArray_Descr *)view_dtype;
37593783

37603784
fail:

0 commit comments

Comments
 (0)