Skip to content

Commit bca0b77

Browse files
[backport 3.0.x] BUG: read_csv leaked the string-intern table when a column failed to decode (#68822)
Co-authored-by: Joris Van den Bossche <[email protected]>
1 parent bca1e5d commit bca0b77

4 files changed

Lines changed: 95 additions & 49 deletions

File tree

‎doc/source/whatsnew/v3.0.6.rst‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -26,7 +26,7 @@ Fixed regressions
2626
Bug fixes
2727
~~~~~~~~~
2828
- Bug in :class:`RangeIndex` where missing Copy-on-Write reference tracking allowed external modification via a :class:`Series` to silently corrupt the index (:issue:`67060`)
29-
29+
- Fixed memory leak in :func:`read_csv` with ``engine="c"`` when a string or ``category`` column failed to decode, for example invalid UTF-8 under ``encoding="utf-8"`` and the default ``encoding_errors="strict"`` (:issue:`67931`)
3030

3131

3232
.. ---------------------------------------------------------------------------

‎pandas/_libs/hashtable.pyi‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -270,5 +270,6 @@ def ismember(
270270
arr: np.ndarray,
271271
values: np.ndarray,
272272
) -> npt.NDArray[np.bool_]: ...
273+
def get_hashtable_trace_domain() -> int: ...
273274
def object_hash(obj) -> int: ...
274275
def objects_are_equal(a, b) -> bool: ...

‎pandas/_libs/parsers.pyx‎

Lines changed: 54 additions & 48 deletions
Original file line numberDiff line numberDiff line change
@@ -1502,36 +1502,39 @@ cdef _string_box_utf8(parser_t *parser, int64_t col,
15021502
khiter_t k
15031503

15041504
table = kh_init_strbox()
1505-
lines = line_end - line_start
1506-
result = np.empty(lines, dtype=np.object_)
1507-
coliter_setup(&it, parser, col, line_start)
1508-
1509-
for i in range(lines):
1510-
COLITER_NEXT(it, word)
1505+
try:
1506+
lines = line_end - line_start
1507+
result = np.empty(lines, dtype=np.object_)
1508+
coliter_setup(&it, parser, col, line_start)
15111509

1512-
if na_filter:
1513-
if kh_get_str_starts_item(na_hashset, word):
1514-
# in the hash table
1515-
na_count += 1
1516-
result[i] = NA
1517-
continue
1510+
for i in range(lines):
1511+
COLITER_NEXT(it, word)
15181512

1519-
k = kh_get_strbox(table, word)
1513+
if na_filter:
1514+
if kh_get_str_starts_item(na_hashset, word):
1515+
# in the hash table
1516+
na_count += 1
1517+
result[i] = NA
1518+
continue
15201519

1521-
# in the hash table
1522-
if k != table.n_buckets:
1523-
# this increments the refcount, but need to test
1524-
pyval = <object>table.vals[k]
1525-
else:
1526-
# box it. new ref?
1527-
pyval = PyUnicode_Decode(word, strlen(word), "utf-8", encoding_errors)
1520+
k = kh_get_strbox(table, word)
15281521

1529-
k = kh_put_strbox(table, word, &ret)
1530-
table.vals[k] = <PyObject *>pyval
1522+
# in the hash table
1523+
if k != table.n_buckets:
1524+
# this increments the refcount, but need to test
1525+
pyval = <object>table.vals[k]
1526+
else:
1527+
# box it. new ref?
1528+
# can raise for invalid UTF-8 under encoding_errors="strict"
1529+
# (GH#67931)
1530+
pyval = PyUnicode_Decode(word, strlen(word), "utf-8", encoding_errors)
15311531

1532-
result[i] = pyval
1532+
k = kh_put_strbox(table, word, &ret)
1533+
table.vals[k] = <PyObject *>pyval
15331534

1534-
kh_destroy_strbox(table)
1535+
result[i] = pyval
1536+
finally:
1537+
kh_destroy_strbox(table)
15351538

15361539
return result, na_count
15371540

@@ -1560,36 +1563,39 @@ cdef _categorical_convert(parser_t *parser, int64_t col,
15601563

15611564
# factorize parsed values, creating a hash table
15621565
# bytes -> category code
1563-
with nogil:
1564-
table = kh_init_str()
1565-
coliter_setup(&it, parser, col, line_start)
1566+
table = kh_init_str()
1567+
try:
1568+
with nogil:
1569+
coliter_setup(&it, parser, col, line_start)
15661570

1567-
for i in range(lines):
1568-
COLITER_NEXT(it, word)
1571+
for i in range(lines):
1572+
COLITER_NEXT(it, word)
15691573

1570-
if na_filter:
1571-
if kh_get_str_starts_item(na_hashset, word):
1572-
# is in NA values
1573-
na_count += 1
1574-
codes[i] = NA
1575-
continue
1574+
if na_filter:
1575+
if kh_get_str_starts_item(na_hashset, word):
1576+
# is in NA values
1577+
na_count += 1
1578+
codes[i] = NA
1579+
continue
15761580

1577-
k = kh_get_str(table, word)
1578-
# not in the hash table
1579-
if k == table.n_buckets:
1580-
k = kh_put_str(table, word, &ret)
1581-
table.vals[k] = current_category
1582-
current_category += 1
1581+
k = kh_get_str(table, word)
1582+
# not in the hash table
1583+
if k == table.n_buckets:
1584+
k = kh_put_str(table, word, &ret)
1585+
table.vals[k] = current_category
1586+
current_category += 1
15831587

1584-
codes[i] = table.vals[k]
1588+
codes[i] = table.vals[k]
15851589

1586-
# parse and box categories to python strings
1587-
result = np.empty(table.n_occupied, dtype=np.object_)
1588-
for k in range(table.n_buckets):
1589-
if kh_exist_str(table, k):
1590-
result[table.vals[k]] = PyUnicode_FromString(table.keys[k])
1590+
# parse and box categories to python strings
1591+
result = np.empty(table.n_occupied, dtype=np.object_)
1592+
for k in range(table.n_buckets):
1593+
if kh_exist_str(table, k):
1594+
# can raise; see _string_box_utf8 (GH#67931)
1595+
result[table.vals[k]] = PyUnicode_FromString(table.keys[k])
1596+
finally:
1597+
kh_destroy_str(table)
15911598

1592-
kh_destroy_str(table)
15931599
return np.asarray(codes), result, na_count
15941600

15951601

‎pandas/tests/io/parser/test_c_parser_only.py‎

Lines changed: 39 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,13 @@
1414
import mmap
1515
import os
1616
import tarfile
17+
import tracemalloc
1718

1819
import numpy as np
1920
import pytest
2021

2122
from pandas._libs import parsers as libparsers
23+
from pandas._libs.hashtable import get_hashtable_trace_domain
2224
from pandas.compat import WASM
2325
from pandas.errors import (
2426
DtypeWarning,
@@ -646,6 +648,43 @@ def test_float_precision_options(c_parser_only):
646648
parser.read_csv(StringIO(s), float_precision="junk")
647649

648650

651+
@pytest.mark.parametrize("dtype", [None, "category"])
652+
def test_decode_failure_frees_string_table(c_parser_only, dtype, tmp_path):
653+
# GH#67931
654+
# the string columns are interned through a khash table that was freed only
655+
# on the success path, so a decode failure part-way through a column leaked
656+
# the whole table -- ~100 KiB per failed parse here, so ~5 MiB over the loop
657+
parser = c_parser_only
658+
rows = "\n".join(f"s{i}" for i in range(2000)).encode()
659+
path = tmp_path / "invalid_utf8.csv"
660+
path.write_bytes(b"a\n" + rows + b"\n\xff\n")
661+
662+
def read():
663+
# a path plus an explicit encoding is what hands the tokenizer
664+
# undecoded bytes, so the column is the first thing to decode them
665+
with pytest.raises(UnicodeDecodeError):
666+
parser.read_csv(
667+
path, dtype=dtype, encoding="utf-8", encoding_errors="strict"
668+
)
669+
670+
# measure only the khash domain, so unrelated allocations cannot mask or
671+
# fake the leak
672+
khash_only = (tracemalloc.DomainFilter(True, get_hashtable_trace_domain()),)
673+
674+
tracemalloc.start()
675+
try:
676+
# warm up, so first-call caching lands outside the measured window
677+
read()
678+
before = tracemalloc.take_snapshot().filter_traces(khash_only)
679+
for _ in range(50):
680+
read()
681+
after = tracemalloc.take_snapshot().filter_traces(khash_only)
682+
finally:
683+
tracemalloc.stop()
684+
685+
assert sum(stat.size_diff for stat in after.compare_to(before, "filename")) == 0
686+
687+
649688
# small enough that DEFAULT_BUFFER_HEURISTIC // table_width leaves one line per
650689
# buffer, so the rows below land in separate low_memory chunks
651690
_MIXED_DTYPE_HEURISTIC = 2**3

0 commit comments

Comments
 (0)