Skip to content

Commit f089739

Browse files
committed
Use a bytearray for streaming output, avoid copy
On one hand, we avoid a memory copy at the end of streaming. On the other hand, we need to lock the GIL during the while loop
1 parent 1f5670f commit f089739

1 file changed

Lines changed: 15 additions & 10 deletions

File tree

‎numcodecs/zstd.pyx‎

Lines changed: 15 additions & 10 deletions
Original file line numberDiff line numberDiff line change
@@ -6,6 +6,7 @@
66

77

88
from cpython.bytes cimport PyBytes_AS_STRING, PyBytes_FromStringAndSize
9+
from cpython.bytearray cimport PyByteArray_FromStringAndSize, PyByteArray_Resize, PyByteArray_AsString
910
from cpython.memoryview cimport PyMemoryView_GET_BUFFER
1011

1112
from .compat_ext cimport PyBytes_RESIZE, ensure_continguous_memoryview
@@ -284,6 +285,7 @@ cdef stream_decompress(const Py_buffer* source_pb):
284285
ZSTD_inBuffer input
285286
ZSTD_outBuffer output
286287
ZSTD_DStream *zds
288+
bytearray dest
287289

288290
# Recommended size for output buffer, guaranteed to flush at least
289291
# one completely block in all circumstances
@@ -299,7 +301,9 @@ cdef stream_decompress(const Py_buffer* source_pb):
299301
# minimum dest_size is DEST_GROWTH_SIZE
300302
dest_size = DEST_GROWTH_SIZE
301303

302-
dest_ptr = <char *>malloc(dest_size)
304+
dest = PyByteArray_FromStringAndSize(NULL, dest_size)
305+
dest_ptr = PyByteArray_AsString(dest)
306+
303307
zds = ZSTD_createDStream()
304308

305309
try:
@@ -335,21 +339,22 @@ cdef stream_decompress(const Py_buffer* source_pb):
335339
if new_size < output.size or new_size < DEST_GROWTH_SIZE:
336340
raise RuntimeError('Zstd stream decompression error: output buffer overflow')
337341

338-
new_dst = realloc(output.dst, new_size)
342+
with gil:
343+
status = PyByteArray_Resize(dest, new_size)
344+
345+
if status < 0:
346+
raise RuntimeError('Zstd stream decompression error on realloc: could not expand output buffer')
339347

340-
if new_dst == NULL:
341-
# output.dst freed in finally block
342-
raise RuntimeError('Zstd stream decompression error on realloc: could not expand output buffer')
348+
output.dst = PyByteArray_AsString(dest)
349+
output.size = new_size
343350

344-
output.dst = new_dst
345-
output.size = new_size
351+
status = PyByteArray_Resize(dest, output.pos)
346352

347-
# Copy the output to a bytes object
348-
dest = PyBytes_FromStringAndSize(<char *>output.dst, output.pos)
353+
if status < 0:
354+
raise RuntimeError('Zstd stream decompression error: could not shrink output to size')
349355

350356
finally:
351357
ZSTD_freeDStream(zds)
352-
free(output.dst)
353358

354359
return dest
355360

0 commit comments

Comments
 (0)