Skip to content

Commit 949ee07

Browse files
sebergcharris
authored andcommitted
BUG: Make mmap handling safer in frombuffer
`frombuffer` always used the old buffer protocol, but this is not safe in some cases when combined with the "new" one. The main/only use-case that was ever found for this is that enabling the new buffer protocol (by going through a memoryview) protects users from accidentally closing memorymaps while they are still used. Closes numpygh-9537
1 parent ceb750a commit 949ee07

3 files changed

Lines changed: 54 additions & 6 deletions

File tree

‎numpy/core/_add_newdocs.py‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1548,6 +1548,10 @@
15481548
The data of the resulting array will not be byteswapped, but will be
15491549
interpreted correctly.
15501550
1551+
This function creates a view into the original object. This should be safe
1552+
in general, but it may make sense to copy the result when the original
1553+
object is mutable or untrusted.
1554+
15511555
Examples
15521556
--------
15531557
>>> s = b'hello world'

‎numpy/core/src/multiarray/ctors.c‎

Lines changed: 26 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -3568,28 +3568,44 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type,
35683568
return NULL;
35693569
}
35703570

3571+
/*
3572+
* The array check is probably unnecessary. It preserves the base for
3573+
* arrays. This is the "old" buffer protocol, which had no release logic.
3574+
* (It was assumed that the result is always a view.)
3575+
*
3576+
* NOTE: We could also check if `bf_releasebuffer` is defined which should
3577+
* be the most precise and safe thing to do. But that should only be
3578+
* necessary if unexpected backcompat issues are found downstream.
3579+
*/
3580+
if (!PyArray_Check(buf)) {
3581+
buf = PyMemoryView_FromObject(buf);
3582+
if (buf == NULL) {
3583+
return NULL;
3584+
}
3585+
}
3586+
else {
3587+
Py_INCREF(buf);
3588+
}
3589+
35713590
if (PyObject_GetBuffer(buf, &view, PyBUF_WRITABLE|PyBUF_SIMPLE) < 0) {
35723591
writeable = 0;
35733592
PyErr_Clear();
35743593
if (PyObject_GetBuffer(buf, &view, PyBUF_SIMPLE) < 0) {
3594+
Py_DECREF(buf);
35753595
Py_DECREF(type);
35763596
return NULL;
35773597
}
35783598
}
35793599
data = (char *)view.buf;
35803600
ts = view.len;
3581-
/*
3582-
* In Python 3 both of the deprecated functions PyObject_AsWriteBuffer and
3583-
* PyObject_AsReadBuffer that this code replaces release the buffer. It is
3584-
* up to the object that supplies the buffer to guarantee that the buffer
3585-
* sticks around after the release.
3586-
*/
3601+
/* `buf` is an array or a memoryview; so we know `view` does not own data */
35873602
PyBuffer_Release(&view);
35883603

35893604
if ((offset < 0) || (offset > ts)) {
35903605
PyErr_Format(PyExc_ValueError,
35913606
"offset must be non-negative and no greater than buffer "\
35923607
"length (%" NPY_INTP_FMT ")", (npy_intp)ts);
3608+
Py_DECREF(buf);
35933609
Py_DECREF(type);
35943610
return NULL;
35953611
}
@@ -3602,13 +3618,15 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type,
36023618
if (itemsize == 0) {
36033619
PyErr_SetString(PyExc_ValueError,
36043620
"cannot determine count if itemsize is 0");
3621+
Py_DECREF(buf);
36053622
Py_DECREF(type);
36063623
return NULL;
36073624
}
36083625
if (s % itemsize != 0) {
36093626
PyErr_SetString(PyExc_ValueError,
36103627
"buffer size must be a multiple"\
36113628
" of element size");
3629+
Py_DECREF(buf);
36123630
Py_DECREF(type);
36133631
return NULL;
36143632
}
@@ -3619,6 +3637,7 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type,
36193637
PyErr_SetString(PyExc_ValueError,
36203638
"buffer is smaller than requested"\
36213639
" size");
3640+
Py_DECREF(buf);
36223641
Py_DECREF(type);
36233642
return NULL;
36243643
}
@@ -3628,6 +3647,7 @@ PyArray_FromBuffer(PyObject *buf, PyArray_Descr *type,
36283647
&PyArray_Type, type,
36293648
1, &n, NULL, data,
36303649
NPY_ARRAY_DEFAULT, NULL, buf);
3650+
Py_DECREF(buf);
36313651
if (ret == NULL) {
36323652
return NULL;
36333653
}

‎numpy/core/tests/test_multiarray.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -18,6 +18,7 @@
1818
import pathlib
1919
import builtins
2020
from decimal import Decimal
21+
import mmap
2122

2223
import numpy as np
2324
import numpy.core._multiarray_tests as _multiarray_tests
@@ -5289,9 +5290,32 @@ def test_basic(self, byteorder, dtype):
52895290
buf = x.tobytes()
52905291
assert_array_equal(np.frombuffer(buf, dtype=dt), x.flat)
52915292

5293+
def test_array_base(self):
5294+
arr = np.arange(10)
5295+
new = np.frombuffer(arr)
5296+
# We currently special case arrays to ensure they are used as a base.
5297+
# This could probably be changed (removing the test).
5298+
assert new.base is arr
5299+
52925300
def test_empty(self):
52935301
assert_array_equal(np.frombuffer(b''), np.array([]))
52945302

5303+
@pytest.mark.skipif(IS_PYPY,
5304+
reason="PyPy's memoryview currently does not track exports. See: "
5305+
"https://foss.heptapod.net/pypy/pypy/-/issues/3724")
5306+
def test_mmap_close(self):
5307+
# The old buffer protocol was not safe for some things that the new
5308+
# one is. But `frombuffer` always used the old one for a long time.
5309+
# Checks that it is safe with the new one (using memoryviews)
5310+
with tempfile.TemporaryFile(mode='wb') as tmp:
5311+
tmp.write(b"asdf")
5312+
tmp.flush()
5313+
mm = mmap.mmap(tmp.fileno(), 0)
5314+
arr = np.frombuffer(mm, dtype=np.uint8)
5315+
with pytest.raises(BufferError):
5316+
mm.close() # cannot close while array uses the buffer
5317+
del arr
5318+
mm.close()
52955319

52965320
class TestFlat:
52975321
def setup(self):

0 commit comments

Comments
 (0)