Skip to content

Commit 0bf907a

Browse files
committed
BUG: Fix numerous bugs related to zero-width string arrays (numpy#473, numpy#1901, numpy#2196, numpy#2585, numpy#4955)
PyArray_NewFromDescr normally does not allow zero-width string dtypes (or rather, it automatically converts them to 1-width strings). This affects any code that uses PyArray_NewFromDescr, which is a lot. So we extend PyArray_NewFromDescr_int to allow disabling this functionality in a couple narrow cases where it's appropriate--one is when extracting a field from a structured array that has a zero-width string dtype (which, intentionally or not, has been allowed). The other, which is related, is returning a view of an array that has a zero-width string dtype. This shouldn't otherwise break or change any existing behavior. Remove roadblocks to creating structured dtypes with zero-width fields using dict-based constructors (this was possible by other means such as the list-based constructor--with the previous fix in particular it should not be necessary to block this anymore). Adds tests based on the tests cases given in the issues this is fixing. Fix a bug with array to zero-width array assignment revealed by the tests. I am slightly concerned that the blunt-force check for this in raw_array_assign_array may be masking a bug somewhere else though.
1 parent e6593fb commit 0bf907a

12 files changed

Lines changed: 96 additions & 38 deletions

File tree

‎numpy/core/_internal.py‎

Lines changed: 0 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -33,8 +33,6 @@ def _makenames_list(adict, align):
3333
if (num < 0):
3434
raise ValueError("invalid offset.")
3535
format = dtype(obj[0], align=align)
36-
if (format.itemsize == 0):
37-
raise ValueError("all itemsizes must be fixed.")
3836
if (n > 2):
3937
title = obj[2]
4038
else:

‎numpy/core/records.py‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -424,7 +424,7 @@ def __new__(subtype, shape, dtype=None, buf=None, offset=0, strides=None,
424424
return self
425425

426426
def __array_finalize__(self, obj):
427-
if self.dtype.type is not record:
427+
if self.dtype.type is not record and self.dtype.fields:
428428
# if self.dtype is not np.record, invoke __setattr__ which will
429429
# convert it to a record if it is a void dtype.
430430
self.dtype = self.dtype

‎numpy/core/src/multiarray/convert.c‎

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "npy_pycompat.h"
1515

1616
#include "arrayobject.h"
17+
#include "ctors.h"
1718
#include "mapping.h"
1819
#include "lowlevel_strided_loops.h"
1920
#include "scalartypes.h"
@@ -600,13 +601,13 @@ PyArray_View(PyArrayObject *self, PyArray_Descr *type, PyTypeObject *pytype)
600601

601602
dtype = PyArray_DESCR(self);
602603
Py_INCREF(dtype);
603-
ret = (PyArrayObject *)PyArray_NewFromDescr(subtype,
604+
ret = (PyArrayObject *)PyArray_NewFromDescr_int(subtype,
604605
dtype,
605606
PyArray_NDIM(self), PyArray_DIMS(self),
606607
PyArray_STRIDES(self),
607608
PyArray_DATA(self),
608609
flags,
609-
(PyObject *)self);
610+
(PyObject *)self, 0, 1);
610611
if (ret == NULL) {
611612
Py_XDECREF(type);
612613
return NULL;

‎numpy/core/src/multiarray/ctors.c‎

Lines changed: 18 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -894,10 +894,11 @@ discover_dimensions(PyObject *obj, int *maxndim, npy_intp *d, int check_it,
894894
*
895895
* steals a reference to descr (even on failure)
896896
*/
897-
static PyObject *
897+
NPY_NO_EXPORT PyObject *
898898
PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
899899
npy_intp *dims, npy_intp *strides, void *data,
900-
int flags, PyObject *obj, int zeroed)
900+
int flags, PyObject *obj, int zeroed,
901+
int allow_emptystring)
901902
{
902903
PyArrayObject_fields *fa;
903904
int i;
@@ -916,7 +917,8 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
916917
newstrides, nd);
917918
ret = PyArray_NewFromDescr_int(subtype, descr, nd, newdims,
918919
newstrides,
919-
data, flags, obj, zeroed);
920+
data, flags, obj, zeroed,
921+
allow_emptystring);
920922
return ret;
921923
}
922924

@@ -935,16 +937,17 @@ PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
935937
PyErr_SetString(PyExc_TypeError, "Empty data-type");
936938
Py_DECREF(descr);
937939
return NULL;
938-
}
939-
PyArray_DESCR_REPLACE(descr);
940-
if (descr == NULL) {
941-
return NULL;
942-
}
943-
if (descr->type_num == NPY_STRING) {
944-
nbytes = descr->elsize = 1;
945-
}
946-
else {
947-
nbytes = descr->elsize = sizeof(npy_ucs4);
940+
} else if (!allow_emptystring) {
941+
PyArray_DESCR_REPLACE(descr);
942+
if (descr == NULL) {
943+
return NULL;
944+
}
945+
if (descr->type_num == NPY_STRING) {
946+
nbytes = descr->elsize = 1;
947+
}
948+
else {
949+
nbytes = descr->elsize = sizeof(npy_ucs4);
950+
}
948951
}
949952
}
950953

@@ -1134,7 +1137,7 @@ PyArray_NewFromDescr(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
11341137
{
11351138
return PyArray_NewFromDescr_int(subtype, descr, nd,
11361139
dims, strides, data,
1137-
flags, obj, 0);
1140+
flags, obj, 0, 0);
11381141
}
11391142

11401143
/*NUMPY_API
@@ -2855,7 +2858,7 @@ PyArray_Zeros(int nd, npy_intp *dims, PyArray_Descr *type, int is_f_order)
28552858
type,
28562859
nd, dims,
28572860
NULL, NULL,
2858-
is_f_order, NULL, 1);
2861+
is_f_order, NULL, 1, 0);
28592862

28602863
if (ret == NULL) {
28612864
return NULL;

‎numpy/core/src/multiarray/ctors.h‎

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,12 @@ PyArray_NewFromDescr(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
66
npy_intp *dims, npy_intp *strides, void *data,
77
int flags, PyObject *obj);
88

9+
NPY_NO_EXPORT PyObject *
10+
PyArray_NewFromDescr_int(PyTypeObject *subtype, PyArray_Descr *descr, int nd,
11+
npy_intp *dims, npy_intp *strides, void *data,
12+
int flags, PyObject *obj, int zeroed,
13+
int allow_emptystring);
14+
915
NPY_NO_EXPORT PyObject *PyArray_New(PyTypeObject *, int nd, npy_intp *,
1016
int, npy_intp *, void *, int, int, PyObject *);
1117

‎numpy/core/src/multiarray/descriptor.c‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1138,7 +1138,7 @@ _convert_from_dict(PyObject *obj, int align)
11381138
}
11391139
}
11401140
Py_DECREF(tup);
1141-
if ((ret == NPY_FAIL) || (newdescr->elsize == 0)) {
1141+
if (ret == NPY_FAIL) {
11421142
goto fail;
11431143
}
11441144
dtypeflags |= (newdescr->flags & NPY_FROM_FIELDS);

‎numpy/core/src/multiarray/dtype_transfer.c‎

Lines changed: 7 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -22,6 +22,7 @@
2222
#include "npy_pycompat.h"
2323

2424
#include "convert_datatype.h"
25+
#include "ctors.h"
2526
#include "_datetime.h"
2627
#include "datetime_strings.h"
2728

@@ -549,8 +550,8 @@ wrap_copy_swap_function(int aligned,
549550
* The copyswap functions shouldn't need that.
550551
*/
551552
Py_INCREF(dtype);
552-
data->arr = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, dtype,
553-
1, &shape, NULL, NULL, 0, NULL);
553+
data->arr = (PyArrayObject *)PyArray_NewFromDescr_int(&PyArray_Type, dtype,
554+
1, &shape, NULL, NULL, 0, NULL, 0, 1);
554555
if (data->arr == NULL) {
555556
PyArray_free(data);
556557
return NPY_FAIL;
@@ -1405,8 +1406,8 @@ get_nbo_cast_transfer_function(int aligned,
14051406
return NPY_FAIL;
14061407
}
14071408
}
1408-
data->aip = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, tmp_dtype,
1409-
1, &shape, NULL, NULL, 0, NULL);
1409+
data->aip = (PyArrayObject *)PyArray_NewFromDescr_int(&PyArray_Type,
1410+
tmp_dtype, 1, &shape, NULL, NULL, 0, NULL, 0, 1);
14101411
if (data->aip == NULL) {
14111412
PyArray_free(data);
14121413
return NPY_FAIL;
@@ -1429,8 +1430,8 @@ get_nbo_cast_transfer_function(int aligned,
14291430
return NPY_FAIL;
14301431
}
14311432
}
1432-
data->aop = (PyArrayObject *)PyArray_NewFromDescr(&PyArray_Type, tmp_dtype,
1433-
1, &shape, NULL, NULL, 0, NULL);
1433+
data->aop = (PyArrayObject *)PyArray_NewFromDescr_int(&PyArray_Type,
1434+
tmp_dtype, 1, &shape, NULL, NULL, 0, NULL, 0, 1);
14341435
if (data->aop == NULL) {
14351436
Py_DECREF(data->aip);
14361437
PyArray_free(data);

‎numpy/core/src/multiarray/mapping.c‎

Lines changed: 5 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -14,6 +14,7 @@
1414
#include "npy_import.h"
1515

1616
#include "common.h"
17+
#include "ctors.h"
1718
#include "iterators.h"
1819
#include "mapping.h"
1920
#include "lowlevel_strided_loops.h"
@@ -1291,15 +1292,15 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view)
12911292

12921293
/* view the array at the new offset+dtype */
12931294
Py_INCREF(fieldtype);
1294-
*view = (PyArrayObject*)PyArray_NewFromDescr(
1295+
*view = (PyArrayObject*)PyArray_NewFromDescr_int(
12951296
Py_TYPE(arr),
12961297
fieldtype,
12971298
PyArray_NDIM(arr),
12981299
PyArray_SHAPE(arr),
12991300
PyArray_STRIDES(arr),
13001301
PyArray_BYTES(arr) + offset,
13011302
PyArray_FLAGS(arr),
1302-
(PyObject *)arr);
1303+
(PyObject *)arr, 0, 1);
13031304
if (*view == NULL) {
13041305
return 0;
13051306
}
@@ -1397,15 +1398,15 @@ _get_field_view(PyArrayObject *arr, PyObject *ind, PyArrayObject **view)
13971398
view_dtype->fields = fields;
13981399
view_dtype->flags = PyArray_DESCR(arr)->flags;
13991400

1400-
*view = (PyArrayObject*)PyArray_NewFromDescr(
1401+
*view = (PyArrayObject*)PyArray_NewFromDescr_int(
14011402
Py_TYPE(arr),
14021403
view_dtype,
14031404
PyArray_NDIM(arr),
14041405
PyArray_SHAPE(arr),
14051406
PyArray_STRIDES(arr),
14061407
PyArray_DATA(arr),
14071408
PyArray_FLAGS(arr),
1408-
(PyObject *)arr);
1409+
(PyObject *)arr, 0, 1);
14091410
if (*view == NULL) {
14101411
return 0;
14111412
}

‎numpy/core/src/multiarray/methods.c‎

Lines changed: 7 additions & 7 deletions
Original file line numberDiff line numberDiff line change
@@ -379,13 +379,13 @@ PyArray_GetField(PyArrayObject *self, PyArray_Descr *typed, int offset)
379379
Py_DECREF(safe);
380380
}
381381

382-
ret = PyArray_NewFromDescr(Py_TYPE(self),
383-
typed,
384-
PyArray_NDIM(self), PyArray_DIMS(self),
385-
PyArray_STRIDES(self),
386-
PyArray_BYTES(self) + offset,
387-
PyArray_FLAGS(self)&(~NPY_ARRAY_F_CONTIGUOUS),
388-
(PyObject *)self);
382+
ret = PyArray_NewFromDescr_int(Py_TYPE(self),
383+
typed,
384+
PyArray_NDIM(self), PyArray_DIMS(self),
385+
PyArray_STRIDES(self),
386+
PyArray_BYTES(self) + offset,
387+
PyArray_FLAGS(self)&(~NPY_ARRAY_F_CONTIGUOUS),
388+
(PyObject *)self, 0, 1);
389389
if (ret == NULL) {
390390
return NULL;
391391
}

‎numpy/core/tests/test_dtype.py‎

Lines changed: 10 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -256,6 +256,16 @@ def test_from_dictproxy(self):
256256
dt2 = np.dtype((np.void, dt.fields))
257257
assert_equal(dt2.fields, dt.fields)
258258

259+
def test_from_dict_with_zero_width_field(self):
260+
# Regression test for #6430 / #2196
261+
dt = np.dtype([('val1', np.float32, (0,)), ('val2', int)])
262+
dt2 = np.dtype({'names': ['val1', 'val2'],
263+
'formats': [(np.float32, (0,)), int]})
264+
265+
assert_dtype_equal(dt, dt2)
266+
assert_equal(dt.fields['val1'][0].itemsize, 0)
267+
assert_equal(dt.itemsize, dt.fields['val2'][0].itemsize)
268+
259269
def test_bool_commastring(self):
260270
d = np.dtype('?,?,?') # raises?
261271
assert_equal(len(d.names), 3)

0 commit comments

Comments
 (0)