Skip to content

Commit 6c0ad59

Browse files
mwiebecharris
authored andcommitted
ENH: missingdata: Add maskna= and ownmaskna= parameters to np.asarray and friends
Also fix some array() NA mask construction issues and make sure the base object doesn't collapse past the owner of the NA mask being viewed in addition to the data.
1 parent f9c1d41 commit 6c0ad59

6 files changed

Lines changed: 182 additions & 38 deletions

File tree

‎numpy/add_newdocs.py‎

Lines changed: 14 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -631,7 +631,7 @@ def luf(lamdaexpr, *args, **kwargs):
631631

632632
add_newdoc('numpy.core.multiarray', 'array',
633633
"""
634-
array(object, dtype=None, copy=True, order=None, subok=False, ndmin=0)
634+
array(object, dtype=None, copy=True, order=None, subok=False, ndmin=0, maskna=None, ownmaskna=False)
635635
636636
Create an array.
637637
@@ -667,6 +667,19 @@ def luf(lamdaexpr, *args, **kwargs):
667667
Specifies the minimum number of dimensions that the resulting
668668
array should have. Ones will be pre-pended to the shape as
669669
needed to meet this requirement.
670+
maskna : bool or None, optional
671+
If this is set to True, it forces the array to have an NA mask.
672+
If the input is an array without a mask, this means a view with
673+
an NA mask is created. If the input is an array with a mask, the
674+
mask is preserved as-is.
675+
676+
If this is set to False, it forces the array to not have an NA
677+
mask. If the input is an array with a mask, and has no NA values,
678+
it will create a copy of the input without an NA mask.
679+
ownmaskna : bool, optional
680+
If this is set to True, forces the array to have a mask which
681+
it owns. It may still return a view of the data from the input,
682+
but the result will always own its own mask.
670683
671684
Returns
672685
-------

‎numpy/core/numeric.py‎

Lines changed: 40 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -254,7 +254,7 @@ def extend_all(module):
254254
einsum = multiarray.einsum
255255
isna = multiarray.isna
256256

257-
def asarray(a, dtype=None, order=None):
257+
def asarray(a, dtype=None, order=None, maskna=None, ownmaskna=False):
258258
"""
259259
Convert the input to an array.
260260
@@ -269,6 +269,13 @@ def asarray(a, dtype=None, order=None):
269269
order : {'C', 'F'}, optional
270270
Whether to use row-major ('C') or column-major ('F' for FORTRAN)
271271
memory representation. Defaults to 'C'.
272+
maskna : bool or None, optional
273+
If this is set to True, it forces the array to have an NA mask.
274+
If this is set to False, it forces the array to not have an NA
275+
mask.
276+
ownmaskna : bool, optional
277+
If this is set to True, forces the array to have a mask which
278+
it owns.
272279
273280
Returns
274281
-------
@@ -322,9 +329,10 @@ class ndarray is returned.
322329
True
323330
324331
"""
325-
return array(a, dtype, copy=False, order=order)
332+
return array(a, dtype, copy=False, order=order,
333+
maskna=maskna, ownmaskna=ownmaskna)
326334

327-
def asanyarray(a, dtype=None, order=None):
335+
def asanyarray(a, dtype=None, order=None, maskna=None, ownmaskna=False):
328336
"""
329337
Convert the input to an ndarray, but pass ndarray subclasses through.
330338
@@ -339,6 +347,13 @@ def asanyarray(a, dtype=None, order=None):
339347
order : {'C', 'F'}, optional
340348
Whether to use row-major ('C') or column-major ('F') memory
341349
representation. Defaults to 'C'.
350+
maskna : bool or None, optional
351+
If this is set to True, it forces the array to have an NA mask.
352+
If this is set to False, it forces the array to not have an NA
353+
mask.
354+
ownmaskna : bool, optional
355+
If this is set to True, forces the array to have a mask which
356+
it owns.
342357
343358
Returns
344359
-------
@@ -374,9 +389,10 @@ def asanyarray(a, dtype=None, order=None):
374389
True
375390
376391
"""
377-
return array(a, dtype, copy=False, order=order, subok=True)
392+
return array(a, dtype, copy=False, order=order, subok=True,
393+
maskna=maskna, ownmaskna=ownmaskna)
378394

379-
def ascontiguousarray(a, dtype=None):
395+
def ascontiguousarray(a, dtype=None, maskna=None, ownmaskna=False):
380396
"""
381397
Return a contiguous array in memory (C order).
382398
@@ -386,6 +402,13 @@ def ascontiguousarray(a, dtype=None):
386402
Input array.
387403
dtype : str or dtype object, optional
388404
Data-type of returned array.
405+
maskna : bool or None, optional
406+
If this is set to True, it forces the array to have an NA mask.
407+
If this is set to False, it forces the array to not have an NA
408+
mask.
409+
ownmaskna : bool, optional
410+
If this is set to True, forces the array to have a mask which
411+
it owns.
389412
390413
Returns
391414
-------
@@ -410,9 +433,10 @@ def ascontiguousarray(a, dtype=None):
410433
True
411434
412435
"""
413-
return array(a, dtype, copy=False, order='C', ndmin=1)
436+
return array(a, dtype, copy=False, order='C', ndmin=1,
437+
maskna=maskna, ownmaskna=ownmaskna)
414438

415-
def asfortranarray(a, dtype=None):
439+
def asfortranarray(a, dtype=None, maskna=None, ownmaskna=False):
416440
"""
417441
Return an array laid out in Fortran order in memory.
418442
@@ -422,6 +446,13 @@ def asfortranarray(a, dtype=None):
422446
Input array.
423447
dtype : str or dtype object, optional
424448
By default, the data-type is inferred from the input data.
449+
maskna : bool or None, optional
450+
If this is set to True, it forces the array to have an NA mask.
451+
If this is set to False, it forces the array to not have an NA
452+
mask.
453+
ownmaskna : bool, optional
454+
If this is set to True, forces the array to have a mask which
455+
it owns.
425456
426457
Returns
427458
-------
@@ -446,7 +477,8 @@ def asfortranarray(a, dtype=None):
446477
True
447478
448479
"""
449-
return array(a, dtype, copy=False, order='F', ndmin=1)
480+
return array(a, dtype, copy=False, order='F', ndmin=1,
481+
maskna=maskna, ownmaskna=ownmaskna)
450482

451483
def require(a, dtype=None, requirements=None):
452484
"""

‎numpy/core/src/multiarray/arrayobject.c‎

Lines changed: 29 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -93,25 +93,49 @@ PyArray_SetBaseObject(PyArrayObject *arr, PyObject *obj)
9393
"dependency more than once");
9494
return -1;
9595
}
96+
9697
/*
9798
* Don't allow chains of views, always set the base
98-
* to the owner of the data
99+
* to the owner of the data. That is, either the first object
100+
* which isn't an array, the first object with an NA mask
101+
* which owns that NA mask, or the first object which owns
102+
* its own data.
99103
*/
100-
while (PyArray_Check(obj) &&
101-
(PyObject *)arr != obj &&
102-
PyArray_BASE((PyArrayObject *)obj) != NULL) {
103-
PyObject *tmp = PyArray_BASE((PyArrayObject *)obj);
104+
while (PyArray_Check(obj) && (PyObject *)arr != obj) {
105+
PyArrayObject *obj_arr = (PyArrayObject *)arr;
106+
PyObject *tmp;
107+
108+
/* If this array owns its own data, stop collapsing */
109+
if (PyArray_CHKFLAGS(obj_arr, NPY_ARRAY_OWNDATA)) {
110+
break;
111+
}
112+
/*
113+
* If 'arr' doesn't own its NA mask, then if
114+
* 'obj' is NA masked and owns the mask, stop collapsing
115+
*/
116+
if (!PyArray_CHKFLAGS(arr, NPY_ARRAY_OWNMASKNA) &&
117+
PyArray_CHKFLAGS(obj_arr, NPY_ARRAY_OWNMASKNA)) {
118+
break;
119+
}
120+
/* If there's no base, stop collapsing */
121+
tmp = PyArray_BASE(obj_arr);
122+
if (tmp == NULL) {
123+
break;
124+
}
125+
104126
Py_INCREF(tmp);
105127
Py_DECREF(obj);
106128
obj = tmp;
107129
}
130+
108131
/* Disallow circular references */
109132
if ((PyObject *)arr == obj) {
110133
Py_DECREF(obj);
111134
PyErr_SetString(PyExc_ValueError,
112135
"Cannot create a circular NumPy array 'base' dependency");
113136
return -1;
114137
}
138+
115139
((PyArrayObject_fieldaccess *)arr)->base = obj;
116140

117141
return 0;

‎numpy/core/src/multiarray/convert.c‎

Lines changed: 8 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -571,12 +571,6 @@ PyArray_View(PyArrayObject *self, PyArray_Descr *type, PyTypeObject *pytype)
571571
if (ret == NULL) {
572572
return NULL;
573573
}
574-
Py_INCREF(self);
575-
if (PyArray_SetBaseObject(ret, (PyObject *)self) < 0) {
576-
Py_DECREF(ret);
577-
Py_DECREF(type);
578-
return NULL;
579-
}
580574

581575
/* Take a view of the mask if it exists */
582576
if (PyArray_HASMASKNA(self)) {
@@ -600,6 +594,14 @@ PyArray_View(PyArrayObject *self, PyArray_Descr *type, PyTypeObject *pytype)
600594
fa->flags |= NPY_ARRAY_MASKNA;
601595
}
602596

597+
/* Set the base object */
598+
Py_INCREF(self);
599+
if (PyArray_SetBaseObject(ret, (PyObject *)self) < 0) {
600+
Py_DECREF(ret);
601+
Py_DECREF(type);
602+
return NULL;
603+
}
604+
603605
if (type != NULL) {
604606
if (PyObject_SetAttrString((PyObject *)ret, "dtype",
605607
(PyObject *)type) < 0) {

‎numpy/core/src/multiarray/ctors.c‎

Lines changed: 46 additions & 17 deletions
Original file line numberDiff line numberDiff line change
@@ -1999,16 +1999,26 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags)
19991999
if (PyArray_NDIM(arr) <= 1 && (flags & NPY_ARRAY_F_CONTIGUOUS)) {
20002000
flags |= NPY_ARRAY_C_CONTIGUOUS;
20012001
}
2002+
/* If a guaranteed copy was requested */
20022003
copy = (flags & NPY_ARRAY_ENSURECOPY) ||
2003-
((flags & NPY_ARRAY_C_CONTIGUOUS) &&
2004-
(!(arrflags & NPY_ARRAY_C_CONTIGUOUS)))
2005-
|| ((flags & NPY_ARRAY_ALIGNED) &&
2006-
(!(arrflags & NPY_ARRAY_ALIGNED)))
2007-
|| (PyArray_NDIM(arr) > 1 &&
2008-
((flags & NPY_ARRAY_F_CONTIGUOUS) &&
2009-
(!(arrflags & NPY_ARRAY_F_CONTIGUOUS))))
2010-
|| ((flags & NPY_ARRAY_WRITEABLE) &&
2011-
(!(arrflags & NPY_ARRAY_WRITEABLE))) ||
2004+
/* If C contiguous was requested, and arr is not */
2005+
((flags & NPY_ARRAY_C_CONTIGUOUS) &&
2006+
(!(arrflags & NPY_ARRAY_C_CONTIGUOUS))) ||
2007+
/* If an aligned array was requested, and arr is not */
2008+
((flags & NPY_ARRAY_ALIGNED) &&
2009+
(!(arrflags & NPY_ARRAY_ALIGNED))) ||
2010+
/* If a Fortran contiguous array was requested, and arr is not */
2011+
(PyArray_NDIM(arr) > 1 &&
2012+
((flags & NPY_ARRAY_F_CONTIGUOUS) &&
2013+
(!(arrflags & NPY_ARRAY_F_CONTIGUOUS)))) ||
2014+
/* If a writeable array was requested, and arr is not */
2015+
((flags & NPY_ARRAY_WRITEABLE) &&
2016+
(!(arrflags & NPY_ARRAY_WRITEABLE))) ||
2017+
/* If an array with no NA mask was requested, and arr has one */
2018+
((flags & (NPY_ARRAY_ALLOWNA |
2019+
NPY_ARRAY_MASKNA |
2020+
NPY_ARRAY_OWNMASKNA)) == 0 &&
2021+
(arrflags & NPY_ARRAY_MASKNA)) ||
20122022
!PyArray_EquivTypes(oldtype, newtype);
20132023

20142024
if (copy) {
@@ -2043,7 +2053,7 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags)
20432053
* Allocate an NA mask if necessary from the input,
20442054
* is NAs are being allowed.
20452055
*/
2046-
if (PyArray_HASMASKNA(arr) && (flags & NPY_ARRAY_ALLOWNA)) {
2056+
if ((arrflags & NPY_ARRAY_MASKNA) && (flags & NPY_ARRAY_ALLOWNA)) {
20472057
if (PyArray_AllocateMaskNA(ret, 1, 0, 1) < 0) {
20482058
Py_DECREF(ret);
20492059
return NULL;
@@ -2080,25 +2090,44 @@ PyArray_FromArray(PyArrayObject *arr, PyArray_Descr *newtype, int flags)
20802090
}
20812091
}
20822092
/*
2083-
* If no copy then just increase the reference
2084-
* count and return the input
2093+
* If no copy then take an appropriate view if necessary, or
2094+
* just return a reference to ret itself.
20852095
*/
20862096
else {
2097+
int needview = ((flags & NPY_ARRAY_ENSUREARRAY) &&
2098+
!PyArray_CheckExact(arr)) ||
2099+
((flags & NPY_ARRAY_MASKNA) &&
2100+
!(arrflags & NPY_ARRAY_MASKNA)) ||
2101+
((flags & NPY_ARRAY_OWNMASKNA) &&
2102+
!(arrflags & NPY_ARRAY_OWNMASKNA));
2103+
20872104
Py_DECREF(newtype);
2088-
if ((flags & NPY_ARRAY_ENSUREARRAY) &&
2089-
!PyArray_CheckExact(arr)) {
2105+
if (needview) {
20902106
PyArray_Descr *dtype = PyArray_DESCR(arr);
2091-
Py_INCREF(dtype);
2107+
PyTypeObject *subtype = NULL;
2108+
2109+
if (flags & NPY_ARRAY_ENSUREARRAY) {
2110+
subtype = &PyArray_Type;
2111+
}
20922112

2093-
ret = (PyArrayObject *)PyArray_View(arr, NULL, &PyArray_Type);
2113+
Py_INCREF(dtype);
2114+
ret = (PyArrayObject *)PyArray_View(arr, NULL, subtype);
20942115
if (ret == NULL) {
20952116
return NULL;
20962117
}
2118+
2119+
if (flags & (NPY_ARRAY_MASKNA | NPY_ARRAY_OWNMASKNA)) {
2120+
int ownmaskna = (flags & NPY_ARRAY_OWNMASKNA) != 0;
2121+
if (PyArray_AllocateMaskNA(ret, ownmaskna, 0, 1) < 0) {
2122+
Py_DECREF(ret);
2123+
return NULL;
2124+
}
2125+
}
20972126
}
20982127
else {
2128+
Py_INCREF(arr);
20992129
ret = arr;
21002130
}
2101-
Py_INCREF(arr);
21022131
}
21032132

21042133
return (PyObject *)ret;

‎numpy/core/tests/test_maskna.py‎

Lines changed: 45 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,6 +83,43 @@ def test_array_maskna_construction():
8383
assert_(a.flags.maskna)
8484
assert_equal(np.isna(a), True)
8585

86+
def test_array_maskna_asarray():
87+
a = np.arange(6).reshape(2,3)
88+
89+
# Should not add an NA mask by default
90+
res = np.asarray(a)
91+
assert_(res is a)
92+
assert_(not res.flags.maskna)
93+
94+
# Should add an NA mask if requested
95+
res = np.asarray(a, maskna=True)
96+
assert_(res.flags.maskna)
97+
assert_(res.flags.ownmaskna)
98+
res = np.asarray(a, ownmaskna=True)
99+
assert_(res.flags.maskna)
100+
assert_(res.flags.ownmaskna)
101+
102+
a.flags.maskna = True
103+
104+
# Should view or create a copy of the NA mask
105+
res = np.asarray(a)
106+
assert_(res is a)
107+
res = np.asarray(a, maskna=True)
108+
assert_(res is a)
109+
res = np.asarray(a, ownmaskna=True)
110+
assert_(res is a)
111+
112+
b = a.view()
113+
assert_(not b.flags.ownmaskna)
114+
115+
res = np.asarray(b)
116+
assert_(res is b)
117+
res = np.asarray(b, maskna=True)
118+
assert_(res is b)
119+
res = np.asarray(b, ownmaskna=True)
120+
assert_(not (res is b))
121+
assert_(res.flags.ownmaskna)
122+
86123
def test_array_maskna_copy():
87124
a = np.array([1,2,3])
88125
b = np.array([2,3,4], maskna=True)
@@ -444,10 +481,17 @@ def test_array_maskna_array_function_1D():
444481

445482
# Should produce a view with an owned mask with 'ownmaskna=True'
446483
c = np.array(b_view, copy=False, ownmaskna=True)
447-
assert_(c.base is a)
484+
assert_(c.base is b_view)
448485
assert_(c.flags.ownmaskna)
449486
assert_(not (c is b_view))
450487

488+
# Should produce a view whose base is 'c', because 'c' owns
489+
# the data for its mask
490+
d = c.view()
491+
assert_(d.base is c)
492+
assert_(d.flags.maskna)
493+
assert_(not d.flags.ownmaskna)
494+
451495
def test_array_maskna_setasflat():
452496
# Copy from a C to a F array with some NAs
453497
a_orig = np.empty((2,3), order='C')

0 commit comments

Comments
 (0)