Skip to content

Commit 7412b85

Browse files
committed
MAINT: improve struct assignment, allow multi fields views
This commit attempts to make structure assignment more consistent, and then changes multi-field indices to return a view instead of a copy. Assignment between structures now works "by field position" rather than "by field name". Fixes #2353, fixes #6085, fixes #3351, fixes #6085, fixes #6314, fixes #2346, fixes #7058, fixes #3641, fixes #5994, fixes #7262, fixes #7493
1 parent 2afa142 commit 7412b85

10 files changed

Lines changed: 345 additions & 422 deletions

File tree

‎doc/release/1.14.0-notes.rst‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -209,3 +209,40 @@ Integer scalars are now unaffected by ``np.set_string_function``
209209
Previously the str/repr of integer scalars could be controlled by
210210
``np.set_string_function``, unlike most other numpy scalars. This is no longer
211211
the case.
212+
213+
Multiple-field indexing/assignment of structured arrays
214+
-------------------------------------------------------
215+
The indexing and assignment of structured arrays with multiple fields has
216+
changed in a number of ways:
217+
218+
First, indexing a structured array with multiple fields (eg,
219+
``arr[['f1', 'f3']]``) returns a view into the original array instead of a
220+
copy. The returned view will have extra padding bytes corresponding to
221+
intervening fields in the original array, unlike the copy in 1.13, which will
222+
affect code such as ``arr[['f1', 'f3']].view(newdtype)``.
223+
224+
Second, assignment between structured arrays will now occur "by position"
225+
instead of "by field name". The Nth field of the destination will be set to the
226+
Nth field of the source regardless of field name, unlike in numpy versions 1.6
227+
to 1.13 in which fields in the destination array were set to the
228+
identically-named field in the source array or to 0 if the source did not have
229+
a field.
230+
231+
Correspondingly, the order of fields in a structured dtypes now matters when
232+
computing dtype equality. For example with the dtypes
233+
`x = dtype({'names': ['A', 'B'], 'formats': ['i4', 'f4'], 'offsets': [0, 4]})`
234+
`y = dtype({'names': ['B', 'A'], 'formats': ['f4', 'i4'], 'offsets': [4, 0]})`
235+
now `x == y` will return `False`, unlike before. This makes dictionary-based
236+
dtype specifications like `dtype({'a': ('i4', 0), 'b': ('f4', 4)})` dangerous
237+
in python < 3.6 since dict key-order is not preserved in those versions.
238+
239+
Assignment from a structured array to a boolean array now raises a ValueError,
240+
unlike in 1.13 where it always set the destination elements to `True`.
241+
242+
Assignment from structured array with more than one field to a non-structured
243+
array now raises a ValueError. In 1.13 this copied just the first field of the
244+
source to the destination.
245+
246+
Using field "titles" in multiple-field indexing is now disallowed, as is
247+
repeating a field name in a multiple-field index.
248+

‎numpy/core/src/multiarray/array_assign_array.c‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -293,7 +293,8 @@ PyArray_AssignArray(PyArrayObject *dst, PyArrayObject *src,
293293
if (((PyArray_NDIM(dst) == 1 && PyArray_NDIM(src) >= 1 &&
294294
PyArray_STRIDES(dst)[0] *
295295
PyArray_STRIDES(src)[PyArray_NDIM(src) - 1] < 0) ||
296-
PyArray_NDIM(dst) > 1) && arrays_overlap(src, dst)) {
296+
PyArray_NDIM(dst) > 1 || PyArray_HASFIELDS(dst)) &&
297+
arrays_overlap(src, dst)) {
297298
PyArrayObject *tmp;
298299

299300
/*

‎numpy/core/src/multiarray/arraytypes.c.src‎

Lines changed: 156 additions & 54 deletions
Original file line numberDiff line numberDiff line change
@@ -774,66 +774,173 @@ VOID_getitem(void *input, void *vap)
774774

775775
NPY_NO_EXPORT int PyArray_CopyObject(PyArrayObject *, PyObject *);
776776

777+
/* Given a structured PyArrayObject arr, index i and structured datatype descr,
778+
* modify the dtype of arr to contain a single field corresponding to the ith
779+
* field of descr, recompute the alignment flag, and return the offset of the
780+
* field (in offset_p). This is useful in preparation for calling copyswap on
781+
* individual fields of a numpy structure, in VOID_setitem. Compare to inner
782+
* loops in VOID_getitem and VOID_nonzero.
783+
*
784+
* WARNING: Clobbers arr's dtype and alignment flag.
785+
*/
786+
NPY_NO_EXPORT int
787+
_setup_field(int i, PyArray_Descr *descr, PyArrayObject *arr,
788+
npy_intp *offset_p)
789+
{
790+
PyObject *key;
791+
PyObject *tup;
792+
PyArray_Descr *new;
793+
npy_intp offset;
794+
795+
key = PyTuple_GET_ITEM(descr->names, i);
796+
tup = PyDict_GetItem(descr->fields, key);
797+
if (_unpack_field(tup, &new, &offset) < 0) {
798+
return -1;
799+
}
800+
801+
((PyArrayObject_fields *)(arr))->descr = new;
802+
if ((new->alignment > 1) && ((offset % new->alignment) != 0)) {
803+
PyArray_CLEARFLAGS(arr, NPY_ARRAY_ALIGNED);
804+
}
805+
else {
806+
PyArray_ENABLEFLAGS(arr, NPY_ARRAY_ALIGNED);
807+
}
808+
809+
*offset_p = offset;
810+
return 0;
811+
}
812+
813+
/* Helper function for VOID_setitem, which uses the copyswap or casting code to
814+
* copy structured datatypes between numpy arrays or scalars.
815+
*/
816+
static int
817+
_copy_and_return_void_setitem(PyArray_Descr *dstdescr, char *dstdata,
818+
PyArray_Descr *srcdescr, char *srcdata){
819+
PyArrayObject_fields dummy_struct;
820+
PyArrayObject *dummy = (PyArrayObject *)&dummy_struct;
821+
npy_int names_size = PyTuple_GET_SIZE(dstdescr->names);
822+
npy_intp offset;
823+
npy_int i;
824+
int ret;
825+
826+
/* Fast path if dtypes are equal */
827+
if (PyArray_EquivTypes(srcdescr, dstdescr)) {
828+
for (i = 0; i < names_size; i++) {
829+
/* neither line can ever fail, in principle */
830+
if (_setup_field(i, dstdescr, dummy, &offset)) {
831+
return -1;
832+
}
833+
PyArray_DESCR(dummy)->f->copyswap(dstdata + offset,
834+
srcdata + offset, 0, dummy);
835+
}
836+
return 0;
837+
}
838+
839+
/* Slow path */
840+
ret = PyArray_CastRawArrays(1, srcdata, dstdata, 0, 0,
841+
srcdescr, dstdescr, 0);
842+
if (ret != NPY_SUCCEED) {
843+
return -1;
844+
}
845+
return 0;
846+
}
847+
777848
static int
778849
VOID_setitem(PyObject *op, void *input, void *vap)
779850
{
780851
char *ip = input;
781852
PyArrayObject *ap = vap;
782853
PyArray_Descr *descr;
854+
int flags;
783855
int itemsize=PyArray_DESCR(ap)->elsize;
784856
int res;
785857

786858
descr = PyArray_DESCR(ap);
787-
if (descr->names && PyTuple_Check(op)) {
788-
PyObject *key;
789-
PyObject *names;
790-
int i, n;
791-
PyObject *tup;
792-
int savedflags;
793-
794-
res = 0;
795-
/* get the names from the fields dictionary*/
796-
names = descr->names;
797-
n = PyTuple_GET_SIZE(names);
798-
if (PyTuple_GET_SIZE(op) != n) {
799-
PyErr_SetString(PyExc_ValueError,
800-
"size of tuple must match number of fields.");
801-
return -1;
802-
}
803-
savedflags = PyArray_FLAGS(ap);
804-
for (i = 0; i < n; i++) {
805-
PyArray_Descr *new;
806-
npy_intp offset;
807-
key = PyTuple_GET_ITEM(names, i);
808-
tup = PyDict_GetItem(descr->fields, key);
809-
if (_unpack_field(tup, &new, &offset) < 0) {
810-
((PyArrayObject_fields *)ap)->descr = descr;
859+
flags = PyArray_FLAGS(ap);
860+
if (PyDataType_HASFIELDS(descr)) {
861+
PyObject *errmsg;
862+
npy_int i;
863+
npy_intp offset;
864+
int failed = 0;
865+
866+
/* If op is 0d-ndarray or numpy scalar, directly get dtype & data ptr */
867+
if (PyArray_Check(op)) {
868+
PyArrayObject *oparr = (PyArrayObject *)op;
869+
if (PyArray_SIZE(oparr) != 1) {
870+
PyErr_SetString(PyExc_ValueError,
871+
"setting an array element with a sequence.");
811872
return -1;
812873
}
813-
/*
814-
* TODO: temporarily modifying the array like this
815-
* is bad coding style, should be changed.
816-
*/
817-
((PyArrayObject_fields *)ap)->descr = new;
818-
/* remember to update alignment flags */
819-
if ((new->alignment > 1)
820-
&& ((((npy_intp)(ip+offset)) % new->alignment) != 0)) {
821-
PyArray_CLEARFLAGS(ap, NPY_ARRAY_ALIGNED);
874+
return _copy_and_return_void_setitem(descr, ip,
875+
PyArray_DESCR(oparr), PyArray_DATA(oparr));
876+
}
877+
else if (PyArray_IsScalar(op, Void)) {
878+
PyArray_Descr *srcdescr = ((PyVoidScalarObject *)op)->descr;
879+
char *srcdata = ((PyVoidScalarObject *)op)->obval;
880+
return _copy_and_return_void_setitem(descr, ip, srcdescr, srcdata);
881+
}
882+
else if (PyTuple_Check(op)) {
883+
/* if it's a tuple, copy field-by-field to ap, */
884+
npy_intp names_size = PyTuple_GET_SIZE(descr->names);
885+
886+
if (names_size != PyTuple_Size(op)) {
887+
errmsg = PyUString_FromFormat(
888+
"could not assign tuple of length %zd to structure "
889+
"with %" NPY_INTP_FMT " fields.",
890+
PyTuple_Size(op), names_size);
891+
PyErr_SetObject(PyExc_ValueError, errmsg);
892+
Py_DECREF(errmsg);
893+
return -1;
822894
}
823-
else {
824-
PyArray_ENABLEFLAGS(ap, NPY_ARRAY_ALIGNED);
895+
896+
for (i = 0; i < names_size; i++) {
897+
PyObject *item;
898+
899+
/* temporarily make ap have only this field */
900+
if (_setup_field(i, descr, ap, &offset) == -1) {
901+
failed = 1;
902+
break;
903+
}
904+
item = PyTuple_GetItem(op, i);
905+
if (item == NULL) {
906+
failed = 1;
907+
break;
908+
}
909+
/* use setitem to set this field */
910+
if (PyArray_DESCR(ap)->f->setitem(item, ip + offset, ap) < 0) {
911+
failed = 1;
912+
break;
913+
}
825914
}
826-
res = new->f->setitem(PyTuple_GET_ITEM(op, i), ip+offset, ap);
827-
((PyArrayObject_fields *)ap)->flags = savedflags;
828-
if (res < 0) {
829-
break;
915+
}
916+
else {
917+
/* Otherwise must be non-void scalar. Try to assign to each field */
918+
npy_intp names_size = PyTuple_GET_SIZE(descr->names);
919+
920+
for (i = 0; i < names_size; i++) {
921+
/* temporarily make ap have only this field */
922+
if (_setup_field(i, descr, ap, &offset) == -1) {
923+
failed = 1;
924+
break;
925+
}
926+
/* use setitem to set this field */
927+
if (PyArray_DESCR(ap)->f->setitem(op, ip + offset, ap) < 0) {
928+
failed = 1;
929+
break;
930+
}
830931
}
831932
}
832-
((PyArrayObject_fields *)ap)->descr = descr;
833-
return res;
834-
}
835933

836-
if (descr->subarray) {
934+
/* reset clobbered attributes */
935+
((PyArrayObject_fields *)(ap))->descr = descr;
936+
((PyArrayObject_fields *)(ap))->flags = flags;
937+
938+
if (failed) {
939+
return -1;
940+
}
941+
return 0;
942+
}
943+
else if (PyDataType_HASSUBARRAY(descr)) {
837944
/* copy into an array of the same basic type */
838945
PyArray_Dims shape = {NULL, -1};
839946
PyArrayObject *ret;
@@ -862,29 +969,24 @@ VOID_setitem(PyObject *op, void *input, void *vap)
862969
return res;
863970
}
864971

865-
/* Default is to use buffer interface to set item */
972+
/*
973+
* Fall through case - non-structured void datatype. This is a very
974+
* undiscerning case: It interprets any object as a buffer
975+
* and reads as many bytes as possible, padding with 0.
976+
*/
866977
{
867978
const void *buffer;
868979
Py_ssize_t buflen;
869-
if (PyDataType_FLAGCHK(descr, NPY_ITEM_HASOBJECT)
870-
|| PyDataType_FLAGCHK(descr, NPY_ITEM_IS_POINTER)) {
871-
PyErr_SetString(PyExc_ValueError,
872-
"Setting void-array with object members using buffer.");
873-
return -1;
874-
}
875980
res = PyObject_AsReadBuffer(op, &buffer, &buflen);
876981
if (res == -1) {
877-
goto fail;
982+
return -1;
878983
}
879984
memcpy(ip, buffer, PyArray_MIN(buflen, itemsize));
880985
if (itemsize > buflen) {
881986
memset(ip + buflen, 0, itemsize - buflen);
882987
}
883988
}
884989
return 0;
885-
886-
fail:
887-
return -1;
888990
}
889991

890992
static PyObject *

‎numpy/core/src/multiarray/descriptor.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3119,7 +3119,7 @@ static PyMethodDef arraydescr_methods[] = {
31193119
*
31203120
* Returns 1 if it has a simple layout, 0 otherwise.
31213121
*/
3122-
static int
3122+
NPY_NO_EXPORT int
31233123
is_dtype_struct_simple_unaligned_layout(PyArray_Descr *dtype)
31243124
{
31253125
PyObject *names, *fields, *key, *tup, *title;

‎numpy/core/src/multiarray/descriptor.h‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -10,6 +10,10 @@ array_set_typeDict(PyObject *NPY_UNUSED(ignored), PyObject *args);
1010
NPY_NO_EXPORT PyArray_Descr *
1111
_arraydescr_fromobj(PyObject *obj);
1212

13+
14+
NPY_NO_EXPORT int
15+
is_dtype_struct_simple_unaligned_layout(PyArray_Descr *dtype);
16+
1317
/*
1418
* Creates a string repr of the dtype, excluding the 'dtype()' part
1519
* surrounding the object. This object may be a string, a list, or

0 commit comments

Comments
 (0)