gh-129005: Add bytearray._detach, use in readall

cmaloney · cmaloney · commit 98f8483ae007 · 2025-02-25T20:45:04.000-08:00
This removes the memory overhead of `_pyio.FileIO.readall` relative to
`_io.FileIO.readall`, and significantly improves performance:

```bash
 # _io.FileIO.readall of a large file
./python -m test -M8g -uall test_largefile -m test.test_largefile.CLargeFileTest.test_large_read

 # _pyio.FileIO.readall of a large file
./python -m test -M8g -uall test_largefile -m test.test_largefile.PyLargeFileTest.test_large_read
```

`_io` takes: ~0.791s and uses ~2GB of RAM
`_pyio` current: ~1.073s and uses ~4GB of RAM
`_pyio` w/ bytearray._detach: ~0.887s and uses ~2GB of RAM
diff --git a/Lib/_pyio.py b/Lib/_pyio.py
@@ -1697,7 +1697,7 @@ def readall(self):
         assert len(result) - bytes_read >= 1, \
             "os.readinto buffer size 0 will result in erroneous EOF / returns 0"
         result.resize(bytes_read)
-        return bytes(result)
+        return result._detach()
 
     def readinto(self, buffer):
         """Same as RawIOBase.readinto()."""
diff --git a/Lib/test/test_largefile.py b/Lib/test/test_largefile.py
@@ -56,9 +56,7 @@ class TestFileMethods(LargeFileTest):
     (i.e. > 2 GiB) files.
     """
 
-    # _pyio.FileIO.readall() uses a temporary bytearray then casted to bytes,
-    # so memuse=2 is needed
-    @bigmemtest(size=size, memuse=2, dry_run=False)
+    @bigmemtest(size=size, memuse=1, dry_run=False)
     def test_large_read(self, _size):
         # bpo-24658: Test that a read greater than 2GB does not fail.
         with self.open(TESTFN, "rb") as f:
diff --git a/Objects/bytearrayobject.c b/Objects/bytearrayobject.c
@@ -2455,6 +2455,47 @@ bytearray_decode_impl(PyByteArrayObject *self, const char *encoding,
     return PyUnicode_FromEncodedObject((PyObject*)self, encoding, errors);
 }
 
+/*[clinic input]
+@critical_section
+bytearray._detach
+
+Return existing storage without copying as a bytes and clear bytearray storage.
+
+On error, bytearray will be left in a valid state, but buffer may be cleared. If
+there are exports or the bytearray data is offset will throw rather than copy.
+[clinic start generated code]*/
+
+static PyObject *
+bytearray__detach_impl(PyByteArrayObject *self)
+/*[clinic end generated code: output=eac03c9d6f8d6230 input=5bb91fd1ce9b77dc]*/
+{
+    _Py_CRITICAL_SECTION_ASSERT_OBJECT_LOCKED(self);
+    if (self->ob_exports > 0) {
+        PyErr_SetString(PyExc_BufferError,
+                "Existing exports of data: buffer cannot be detached");
+        return NULL;
+    }
+    if (self->ob_start != self->ob_bytes) {
+        PyErr_SetString(PyExc_BufferError,
+                "Buffer start offset from bytes: buffer cannot be detached; was the object sliced?");
+        return NULL;
+    }
+
+    /* buffer may be overallocated, ensure exact size */
+    if (_PyBytes_Resize(&self->ob_bytes_head, PyByteArray_GET_SIZE(self))) {
+        return NULL;
+    }
+
+    /* FIXME: Take buffer size? Bytes to copy from? */
+    PyObject *new_buffer = PyBytes_FromStringAndSize(NULL, 0);
+    if (!new_buffer) {
+        return NULL;
+    }
+    PyObject *old_buffer = self->ob_bytes_head;
+    bytearray_set_bytes(self, new_buffer, 0);
+    return old_buffer;
+}
+
 PyDoc_STRVAR(alloc_doc,
 "B.__alloc__() -> int\n\
 \n\
@@ -2709,6 +2750,7 @@ static PyMethodDef bytearray_methods[] = {
     BYTEARRAY_COPY_METHODDEF
     BYTEARRAY_COUNT_METHODDEF
     BYTEARRAY_DECODE_METHODDEF
+    BYTEARRAY__DETACH_METHODDEF
     BYTEARRAY_ENDSWITH_METHODDEF
     {"expandtabs", _PyCFunction_CAST(bytearray_expandtabs),
     METH_FASTCALL|METH_KEYWORDS, stringlib_expandtabs__doc__},
diff --git a/Objects/clinic/bytearrayobject.c.h b/Objects/clinic/bytearrayobject.c.h
