Skip to content

Commit 4700f5d

Browse files
Backport PR #71442 on branch 3.1.x (BUG: parallel read_csv raising released memoryview on free-threaded Python (GH#71418)) (#71463)
Co-authored-by: jbrockmendel <[email protected]>
1 parent 08efb14 commit 4700f5d

2 files changed

Lines changed: 25 additions & 5 deletions

File tree

‎pandas/io/parsers/readers.py‎

Lines changed: 11 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -398,8 +398,9 @@ def _read(
398398
)
399399

400400
# For large local uncompressed files with the C engine, attempt parallel reading.
401-
# Each worker gets its own file handle and TextReader so the GIL-free tokenisation
402-
# and type-conversion code in parsers.pyx runs truly in parallel.
401+
# Each worker gets its own view of the mmapped file and its own TextReader so
402+
# the GIL-free tokenisation and type-conversion code in parsers.pyx runs truly
403+
# in parallel.
403404
if not iterator and chunksize is None and nrows is None:
404405
_n_workers = _default_n_workers()
405406
if _n_workers > 1 and _can_parallelize_csv(filepath_or_buffer, kwds):
@@ -1007,7 +1008,7 @@ def _read_csv_chunks(
10071008
chunk_failed = threading.Event()
10081009
workers_readers: list = []
10091010

1010-
def _worker() -> None:
1011+
def _worker(mv: memoryview) -> None:
10111012
reader = TextFileReader(io.BytesIO(b""), **chunk_kwds)
10121013
assert isinstance(reader._engine, CParserWrapper)
10131014
# Buffers are reused across the queued chunks and freed at close;
@@ -1091,10 +1092,15 @@ def _piece_ndarray(piece) -> np.ndarray | None:
10911092
mm = mmap.mmap(fh.fileno(), 0, access=mmap.ACCESS_READ)
10921093
try:
10931094
with (
1094-
memoryview(mm) as mv,
1095+
contextlib.ExitStack() as views,
10951096
ThreadPoolExecutor(max_workers=n_workers) as pool,
10961097
):
1097-
for fut in [pool.submit(_worker) for _ in range(n_workers)]:
1098+
# One view per worker: slicing a shared memoryview from several
1099+
# threads races on free-threaded builds (GH#71418).
1100+
worker_views = [
1101+
views.enter_context(memoryview(mm)) for _ in range(n_workers)
1102+
]
1103+
for fut in [pool.submit(_worker, mv) for mv in worker_views]:
10981104
fut.result()
10991105
if chunk_failed.is_set():
11001106
# Let the serial read raise: a chunk's error can be an artifact

‎pandas/tests/io/parser/test_parallel_read.py‎

Lines changed: 14 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1994,6 +1994,20 @@ def test_parallel_chunk_count_follows_file_size(tmp_path, monkeypatch):
19941994
assert fat_n > lean_n
19951995

19961996

1997+
@pytest.mark.skipif(WASM, reason="WASM cannot spawn threads, so no split happens")
1998+
def test_parallel_read_many_chunks_per_worker(tmp_path, monkeypatch):
1999+
# Workers slicing one shared memoryview raised "released memoryview" on
2000+
# free-threaded builds (GH#71418). The race is narrow, so split a single
2001+
# column (which gets the whole piece budget) into tiny chunks and repeat.
2002+
path = tmp_path / "data.csv"
2003+
_write_grid(path, 20_000, 1)
2004+
for _ in range(20):
2005+
n_chunks = _planned_chunk_count(
2006+
path, monkeypatch, chunk_bytes=16, n_workers=8, min_chunk_rows=1
2007+
)
2008+
assert n_chunks >= 1000
2009+
2010+
19972011
@pytest.mark.skipif(WASM, reason="WASM cannot spawn threads, so no split happens")
19982012
def test_parallel_chunk_count_bounded_by_column_pieces(tmp_path, monkeypatch):
19992013
# Every chunk repeats a per-column cost, so the byte term stays bounded by

0 commit comments

Comments
 (0)