Skip to content

Commit 23b30f4

Browse files
ngoldbaumcharris
authored andcommitted
BUG: avoid segfaults when legacy copyswap slot is not defined (#32151)
1 parent c37ed94 commit 23b30f4

7 files changed

Lines changed: 217 additions & 17 deletions

File tree

‎numpy/_core/src/multiarray/arraytypes.c.src‎

Lines changed: 67 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1013,10 +1013,26 @@ _copy_and_return_void_setitem(_PyArray_LegacyDescr *dstdescr, char *dstdata,
10131013
if (_setup_field(i, dstdescr, dummy_arr, &offset, dstdata)) {
10141014
return -1;
10151015
}
1016-
PyDataType_GetArrFuncs(PyArray_DESCR(dummy_arr))->copyswap(dstdata + offset,
1017-
srcdata + offset, 0, dummy_arr);
1016+
PyArray_CopySwapFunc *copyswap =
1017+
PyDataType_GetArrFuncs(PyArray_DESCR(dummy_arr))->copyswap;
1018+
if (copyswap == NULL) {
1019+
/* a field dtype without the legacy copyswap slot; the
1020+
casting path below handles it correctly */
1021+
break;
1022+
}
1023+
copyswap(dstdata + offset, srcdata + offset, 0, dummy_arr);
1024+
if (PyErr_Occurred()) {
1025+
/*
1026+
* Clear the error and use the casting path, which
1027+
* handles it and overwrites any partially copied fields.
1028+
*/
1029+
PyErr_Clear();
1030+
break;
1031+
}
1032+
}
1033+
if (i == names_size) {
1034+
return 0;
10181035
}
1019-
return 0;
10201036
}
10211037

10221038
/* Slow path */
@@ -2259,6 +2275,17 @@ STRING_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride,
22592275
}
22602276

22612277

2278+
/* See common.h for the contract of this helper. */
2279+
NPY_NO_EXPORT int
2280+
raise_missing_copyswap(PyArray_Descr *dtype, int inplace_swap)
2281+
{
2282+
(void)inplace_swap;
2283+
npy_gil_error(PyExc_TypeError,
2284+
"dtype %S does not implement copyswap",
2285+
(PyObject *)dtype);
2286+
return -1;
2287+
}
2288+
22622289
/* */
22632290
static void
22642291
VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride,
@@ -2289,9 +2316,14 @@ VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride,
22892316
}
22902317

22912318
dummy_fields.descr = new;
2292-
PyDataType_GetArrFuncs(new)->copyswapn(dst+offset, dstride,
2293-
(src != NULL ? src+offset : NULL),
2294-
sstride, n, swap, dummy_arr);
2319+
PyArray_CopySwapNFunc *copyswapn =
2320+
PyDataType_GetArrFuncs(new)->copyswapn;
2321+
if (copyswapn == NULL && raise_missing_copyswap(new, src == NULL) < 0) {
2322+
return;
2323+
}
2324+
copyswapn(dst+offset, dstride,
2325+
(src != NULL ? src+offset : NULL),
2326+
sstride, n, swap, dummy_arr);
22952327
}
22962328
return;
22972329
}
@@ -2330,10 +2362,15 @@ VOID_copyswapn (char *dst, npy_intp dstride, char *src, npy_intp sstride,
23302362
PyArrayObject *dummy_arr = (PyArrayObject *)&dummy_fields;
23312363
((PyArrayObject_fields *)dummy_arr)->descr = new;
23322364

2365+
PyArray_CopySwapNFunc *copyswapn =
2366+
PyDataType_GetArrFuncs(new)->copyswapn;
2367+
if (copyswapn == NULL && raise_missing_copyswap(new, src == NULL) < 0) {
2368+
return;
2369+
}
23332370
num = descr->elsize / subitemsize;
23342371
for (i = 0; i < n; i++) {
2335-
PyDataType_GetArrFuncs(new)->copyswapn(dstptr, subitemsize, srcptr,
2336-
subitemsize, num, swap, dummy_arr);
2372+
copyswapn(dstptr, subitemsize, srcptr,
2373+
subitemsize, num, swap, dummy_arr);
23372374
dstptr += dstride;
23382375
if (srcptr) {
23392376
srcptr += sstride;
@@ -2374,9 +2411,14 @@ VOID_copyswap (char *dst, char *src, int swap, PyArrayObject *arr)
23742411
return;
23752412
}
23762413
dummy_fields.descr = new;
2377-
PyDataType_GetArrFuncs(new)->copyswap(dst+offset,
2378-
(src != NULL ? src+offset : NULL),
2379-
swap, dummy_arr);
2414+
PyArray_CopySwapFunc *copyswap =
2415+
PyDataType_GetArrFuncs(new)->copyswap;
2416+
if (copyswap == NULL && raise_missing_copyswap(new, src == NULL) < 0) {
2417+
return;
2418+
}
2419+
copyswap(dst+offset,
2420+
(src != NULL ? src+offset : NULL),
2421+
swap, dummy_arr);
23802422
}
23812423
return;
23822424
}
@@ -2411,9 +2453,14 @@ VOID_copyswap (char *dst, char *src, int swap, PyArrayObject *arr)
24112453
PyArrayObject *dummy_arr = (PyArrayObject *)&dummy_fields;
24122454
dummy_fields.descr = new;
24132455

2456+
PyArray_CopySwapNFunc *copyswapn =
2457+
PyDataType_GetArrFuncs(new)->copyswapn;
2458+
if (copyswapn == NULL && raise_missing_copyswap(new, src == NULL) < 0) {
2459+
return;
2460+
}
24142461
num = descr->elsize / subitemsize;
2415-
PyDataType_GetArrFuncs(new)->copyswapn(dst, subitemsize, src,
2416-
subitemsize, num, swap, dummy_arr);
2462+
copyswapn(dst, subitemsize, src,
2463+
subitemsize, num, swap, dummy_arr);
24172464
return;
24182465
}
24192466
/* Must be a naive Void type (e.g. a "V8") so simple copy is sufficient. */
@@ -3049,6 +3096,13 @@ VOID_compare(char *ip1, char *ip2, PyArrayObject *ap)
30493096
/* Set the fields needed by compare or copyswap */
30503097
dummy_struct.descr = new;
30513098

3099+
if (PyDataType_GetArrFuncs(new)->compare == NULL) {
3100+
npy_gil_error(PyExc_TypeError,
3101+
"dtype %S does not support comparison",
3102+
(PyObject *)new);
3103+
/* res is 0, so the caller sees equality alongside the error */
3104+
goto finish;
3105+
}
30523106
swap = PyArray_ISBYTESWAPPED(dummy);
30533107
nip1 = ip1 + offset;
30543108
nip2 = ip2 + offset;

‎numpy/_core/src/multiarray/common.h‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -59,6 +59,30 @@ _array_find_python_scalar_type(PyObject *op);
5959
NPY_NO_EXPORT npy_bool
6060
_IsWriteable(PyArrayObject *ap);
6161

62+
/*
63+
* Raise a TypeError for a dtype whose legacy copyswap slot is missing
64+
* (e.g. a dtype written using the new DType API). This never succeeds:
65+
* it always returns -1, so that it can be chained onto the NULL test at
66+
* the call site. `inplace_swap` marks a request that only swaps bytes
67+
* in place (no data copy); it is unused for now, but is passed so that
68+
* such requests can become a no-op (returning 0) when byte order does
69+
* not apply to the dtype (see gh-32150).
70+
*
71+
* WARNING: every call site is written as
72+
*
73+
* if (copyswap == NULL && raise_missing_copyswap(...) < 0) {
74+
* <error return>;
75+
* }
76+
* copyswap(...);
77+
*
78+
* which calls through the NULL slot if this function returns 0. This is
79+
* safe only while it unconditionally returns -1: before changing it to
80+
* return 0, every call site must first be restructured to skip the
81+
* copyswap call on success.
82+
*/
83+
NPY_NO_EXPORT int
84+
raise_missing_copyswap(PyArray_Descr *dtype, int inplace_swap);
85+
6286
NPY_NO_EXPORT PyObject *
6387
convert_shape_to_string(npy_intp n, npy_intp const *vals, char *ending);
6488

‎numpy/_core/src/multiarray/compiled_base.c‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -433,6 +433,7 @@ arr_place(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict)
433433
NPY_cast_info_xfree(&cast_info);
434434
}
435435
else {
436+
int needs_api = PyDataType_FLAGCHK(PyArray_DESCR(array), NPY_NEEDS_PYAPI);
436437
NPY_BEGIN_THREADS_DESCR(PyArray_DESCR(array));
437438
for (i = 0; i < ni; i++) {
438439
if (mask_data[i]) {
@@ -441,10 +442,17 @@ arr_place(PyObject *NPY_UNUSED(self), PyObject *args, PyObject *kwdict)
441442
}
442443

443444
copyswap(dest + i*chunk, src + j*chunk, 0, array);
445+
if (needs_api && PyErr_Occurred()) {
446+
/* e.g. a structured dtype field that does not support copyswap */
447+
break;
448+
}
444449
j++;
445450
}
446451
}
447452
NPY_END_THREADS;
453+
if (PyErr_Occurred()) {
454+
goto fail;
455+
}
448456
}
449457

450458
Py_XDECREF(values);

‎numpy/_core/src/multiarray/getset.c‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -713,6 +713,11 @@ array_flat_set(PyArrayObject *self, PyObject *val, void *NPY_UNUSED(ignored))
713713
swap = PyArray_ISNOTSWAPPED(self) != PyArray_ISNOTSWAPPED(arr);
714714
while(selfit->index < selfit->size) {
715715
copyswap(selfit->dataptr, arrit->dataptr, swap, self);
716+
if (PyErr_Occurred()) {
717+
/* e.g. a structured dtype field that does not support copyswap;
718+
stop writing as soon as the error is visible */
719+
goto exit;
720+
}
716721
PyArray_ITER_NEXT(selfit);
717722
PyArray_ITER_NEXT(arrit);
718723
if (arrit->index == arrit->size) {

‎numpy/_core/src/multiarray/methods.c‎

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -533,10 +533,13 @@ PyArray_Byteswap(PyArrayObject *self, npy_bool inplace)
533533
PyArrayIterObject *it;
534534

535535
copyswapn = PyDataType_GetArrFuncs(PyArray_DESCR(self))->copyswapn;
536+
if (inplace && PyArray_FailUnlessWriteable(self, "array to be byte-swapped") < 0) {
537+
return NULL;
538+
}
539+
if (copyswapn == NULL && raise_missing_copyswap(PyArray_DESCR(self), 1) < 0) {
540+
return NULL;
541+
}
536542
if (inplace) {
537-
if (PyArray_FailUnlessWriteable(self, "array to be byte-swapped") < 0) {
538-
return NULL;
539-
}
540543
size = PyArray_SIZE(self);
541544
if (PyArray_ISONESEGMENT(self)) {
542545
copyswapn(PyArray_DATA(self), PyArray_ITEMSIZE(self), NULL, -1, size, 1, self);
@@ -554,6 +557,10 @@ PyArray_Byteswap(PyArrayObject *self, npy_bool inplace)
554557
}
555558
Py_DECREF(it);
556559
}
560+
if (PyErr_Occurred()) {
561+
/* e.g. a structured dtype field that does not support copyswap */
562+
return NULL;
563+
}
557564

558565
Py_INCREF(self);
559566
return (PyObject *)self;
@@ -564,6 +571,10 @@ PyArray_Byteswap(PyArrayObject *self, npy_bool inplace)
564571
return NULL;
565572
}
566573
new = PyArray_Byteswap(ret, NPY_TRUE);
574+
if (new == NULL) {
575+
Py_DECREF(ret);
576+
return NULL;
577+
}
567578
Py_DECREF(new);
568579
return (PyObject *)ret;
569580
}

‎numpy/_core/src/multiarray/scalartypes.c.src‎

Lines changed: 7 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -2177,13 +2177,19 @@ gentype_byteswap(PyObject *self, PyObject *args, PyObject *kwds)
21772177
descr = PyArray_DescrFromScalar(self);
21782178
data = (void *)scalar_value(self, descr);
21792179

2180+
PyArray_CopySwapFunc *copyswap =
2181+
PyDataType_GetArrFuncs(descr)->copyswap;
2182+
if (copyswap == NULL && raise_missing_copyswap(descr, 1) < 0) {
2183+
Py_DECREF(descr);
2184+
return NULL;
2185+
}
21802186
newmem = PyMem_Malloc(descr->elsize);
21812187
if (newmem == NULL) {
21822188
Py_DECREF(descr);
21832189
return PyErr_NoMemory();
21842190
}
21852191
else {
2186-
PyDataType_GetArrFuncs(descr)->copyswap(newmem, data, 1, NULL);
2192+
copyswap(newmem, data, 1, NULL);
21872193
}
21882194
new = PyArray_Scalar(newmem, descr, NULL);
21892195
PyMem_Free(newmem);

‎numpy/_core/tests/test_custom_dtypes.py‎

Lines changed: 92 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -44,6 +44,98 @@ def test_scaled_float_from_floats(self, scaling):
4444
assert a.dtype.get_scaling() == scaling
4545
assert_array_equal(scaling * a.view(np.float64), [1., 2., 3.])
4646

47+
def test_structured_field_byteswap_raises(self):
48+
# a field (or subarray field) whose dtype lacks the legacy copyswap
49+
# slot cannot be byteswapped; previously this segfaulted via the
50+
# unguarded field copyswap calls in VOID_copyswapn
51+
arr = np.zeros(2, dtype=[("a", "i4"), ("v", SF(1.))])
52+
with pytest.raises(TypeError, match="does not implement copyswap"):
53+
arr.byteswap()
54+
55+
subarr = np.zeros(2, dtype=[("v", SF(1.), (2,))])
56+
with pytest.raises(TypeError, match="does not implement copyswap"):
57+
subarr.byteswap()
58+
59+
def test_byteswap_raises(self):
60+
# a top-level DType without the legacy copyswap slot does not
61+
# support byteswapping
62+
arr = np.zeros(2, dtype=SF(1.))
63+
with pytest.raises(TypeError, match="does not implement copyswap"):
64+
arr.byteswap()
65+
66+
# the in-place writeability check fires before the missing-slot
67+
# check, matching the behavior for dtypes that fill the slot
68+
arr.flags.writeable = False
69+
with pytest.raises(ValueError, match="array to be byte-swapped"):
70+
arr.byteswap(inplace=True)
71+
72+
def test_structured_field_place_and_flat_raise(self):
73+
# requests that need to copy through the missing legacy copyswap
74+
# slot raise instead of crashing
75+
arr = np.zeros(2, dtype=[("v", SF(1.))])
76+
with pytest.raises(TypeError, match="does not implement copyswap"):
77+
np.place(arr, [True, False], arr[:1])
78+
with pytest.raises(TypeError, match="does not implement copyswap"):
79+
arr.flat = arr[:1]
80+
81+
# both stop copying as soon as the error is detected, so elements
82+
# after the first (failing) one keep their values, even for fields
83+
# that could have been copied
84+
marr = np.zeros(3, dtype=[("a", "i4"), ("v", SF(1.))])
85+
marr["a"] = [5, 6, 7]
86+
src = marr[:1].copy()
87+
src["a"] = 1
88+
with pytest.raises(TypeError, match="does not implement copyswap"):
89+
marr.flat = src
90+
assert marr["a"].tolist()[1:] == [6, 7]
91+
with pytest.raises(TypeError, match="does not implement copyswap"):
92+
np.place(marr, [True, True, True], src)
93+
assert marr["a"].tolist()[1:] == [6, 7]
94+
95+
def test_structured_field_sort_raises(self):
96+
# VOID_compare called the field's legacy compare slot, which
97+
# new-style DTypes do not fill
98+
arr = np.zeros(3, dtype=[("v", SF(1.))])
99+
with pytest.raises(TypeError, match="does not support comparison"):
100+
np.sort(arr, order="v")
101+
102+
# packed fields are misaligned; the compare == NULL check fires
103+
# before VOID_compare's alignment handling, so this exits through
104+
# the same guard as the aligned case above and never reaches the
105+
# misaligned buffer path (which would call the missing copyswap
106+
# slot); it checks that the guard keeps covering that layout
107+
packed_dt = np.dtype({"names": ["a", "v"],
108+
"formats": ["u1", SF(1.)],
109+
"offsets": [0, 1], "itemsize": 17})
110+
with pytest.raises(TypeError, match="does not support comparison"):
111+
np.sort(np.zeros(3, dtype=packed_dt), order="v")
112+
113+
def test_structured_setitem_uses_cast_path(self):
114+
# scalar assignment between equivalent structured dtypes used to
115+
# segfault in the copyswap fast path; it now falls back to casting
116+
arr = np.zeros(2, dtype=[("v", SF(1.))])
117+
arr["v"] = np.array([1., 2.]).view(SF(1.))
118+
arr[0] = arr[1]
119+
assert arr["v"].view(np.float64).tolist() == [2., 2.]
120+
121+
def test_structured_setitem_nested_uses_cast_path(self):
122+
# for a nested field the copyswap slot is VOID_copyswap, which
123+
# fails on the inner sfloat field only after it is called; the
124+
# error must not leak out of a successful-looking assignment and
125+
# the copy falls back to the casting path like the flat case
126+
arr = np.zeros(2, dtype=[("a", "i4"), ("nested", [("v", SF(1.))])])
127+
arr["a"] = [1, 2]
128+
arr["nested"]["v"] = np.array([1., 2.]).view(SF(1.))
129+
arr[0] = arr[1]
130+
assert arr["a"].tolist() == [2, 2]
131+
assert arr["nested"]["v"].view(np.float64).tolist() == [2., 2.]
132+
133+
# same for a subarray field of an sfloat dtype
134+
arr = np.zeros(2, dtype=[("sub", SF(1.), (2,))])
135+
arr["sub"] = np.array([[1., 2.], [3., 4.]]).view(SF(1.))
136+
arr[0] = arr[1]
137+
assert arr["sub"].view(np.float64).tolist() == [[3., 4.], [3., 4.]]
138+
47139
def test_repr(self):
48140
# Check the repr, mainly to cover the code paths:
49141
assert repr(SF(scaling=1.)) == "_ScaledFloatTestDType(scaling=1.0)"

0 commit comments

Comments
 (0)