Skip to content

Commit d7eea49

Browse files
committed
fix: cache dynamic adapters created in register_array()
If the base class is a C extension, the subclasses cannot be GC'd. This results in a leak if register_array() is called in a loop. Close #647
1 parent 00b6e05 commit d7eea49

3 files changed

Lines changed: 63 additions & 23 deletions

File tree

‎docs/news.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,13 @@
77
``psycopg`` release notes
88
=========================
99

10+
Psycopg 3.1.12 (unreleased)
11+
^^^^^^^^^^^^^^^^^^^^^^^^^^^
12+
13+
- Fix memory leak when `~register_*()` functions are called repeatedly
14+
(:ticket:`#647`).
15+
16+
1017
Current release
1118
---------------
1219

‎psycopg/psycopg/types/array.py‎

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -311,41 +311,50 @@ def register_array(info: TypeInfo, context: Optional[AdaptContext] = None) -> No
311311
if not info.array_oid:
312312
raise ValueError(f"the type info {info} doesn't describe an array")
313313

314-
base: Type[Any]
315314
adapters = context.adapters if context else postgres.adapters
316315

317-
base = getattr(_psycopg, "ArrayLoader", ArrayLoader)
318-
name = f"{info.name.title()}{base.__name__}"
319-
attribs = {
320-
"base_oid": info.oid,
321-
"delimiter": info.delimiter.encode(),
322-
}
323-
loader = type(name, (base,), attribs)
316+
loader = _make_loader(info.name, info.oid, info.delimiter)
324317
adapters.register_loader(info.array_oid, loader)
325318

319+
# No need to make a new loader because the binary datum has all the info.
326320
loader = getattr(_psycopg, "ArrayBinaryLoader", ArrayBinaryLoader)
327321
adapters.register_loader(info.array_oid, loader)
328322

329-
base = ListDumper
330-
name = f"{info.name.title()}{base.__name__}"
331-
attribs = {
332-
"oid": info.array_oid,
333-
"element_oid": info.oid,
334-
"delimiter": info.delimiter.encode(),
335-
}
336-
dumper = type(name, (base,), attribs)
323+
dumper = _make_dumper(info.name, info.oid, info.array_oid, info.delimiter)
337324
adapters.register_dumper(None, dumper)
338325

339-
base = ListBinaryDumper
340-
name = f"{info.name.title()}{base.__name__}"
341-
attribs = {
342-
"oid": info.array_oid,
343-
"element_oid": info.oid,
344-
}
345-
dumper = type(name, (base,), attribs)
326+
dumper = _make_binary_dumper(info.name, info.oid, info.array_oid)
346327
adapters.register_dumper(None, dumper)
347328

348329

330+
# Cache all dynamically-generated types to avoid leaks in case the types
331+
# cannot be GC'd.
332+
333+
334+
@cache
335+
def _make_loader(name: str, oid: int, delimiter: str) -> Type[Loader]:
336+
# Note: caching this function is really needed because, if the C extension
337+
# is available, the resulting type cannot be GC'd, so calling
338+
# register_array() in a loop results in a leak. See #647.
339+
base = getattr(_psycopg, "ArrayLoader", ArrayLoader)
340+
attribs = {"base_oid": oid, "delimiter": delimiter.encode()}
341+
return type(f"{name.title()}{base.__name__}", (base,), attribs)
342+
343+
344+
@cache
345+
def _make_dumper(
346+
name: str, oid: int, array_oid: int, delimiter: str
347+
) -> Type[BaseListDumper]:
348+
attribs = {"oid": array_oid, "element_oid": oid, "delimiter": delimiter.encode()}
349+
return type(f"{name.title()}ListDumper", (ListDumper,), attribs)
350+
351+
352+
@cache
353+
def _make_binary_dumper(name: str, oid: int, array_oid: int) -> Type[BaseListDumper]:
354+
attribs = {"oid": array_oid, "element_oid": oid}
355+
return type(f"{name.title()}ListBinaryDumper", (ListBinaryDumper,), attribs)
356+
357+
349358
def register_default_adapters(context: AdaptContext) -> None:
350359
# The text dumper is more flexible as it can handle lists of mixed type,
351360
# so register it later.

‎tests/types/test_array.py‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
import gc
12
from typing import List, Any
23
from decimal import Decimal
34

@@ -10,6 +11,9 @@
1011
from psycopg.types import TypeInfo
1112
from psycopg._compat import prod
1213
from psycopg.postgres import types as builtins
14+
from psycopg.types.array import register_array
15+
16+
from ..utils import gc_collect
1317

1418

1519
tests_str = [
@@ -335,3 +339,23 @@ def test_all_chars_with_bounds(conn, fmt_out):
335339
s = "".join(a)
336340
cur.execute("select '[0:1]={a,b}'::text[] || %s::text[]", ([s],))
337341
assert cur.fetchone()[0] == ["a", "b", s]
342+
343+
344+
def test_register_array_leak(conn):
345+
info = TypeInfo.fetch(conn, "date")
346+
ntypes = []
347+
for i in range(2):
348+
cur = conn.cursor()
349+
register_array(info, cur)
350+
cur.close()
351+
del cur
352+
gc_collect()
353+
354+
objs = gc.get_objects()
355+
n = 0
356+
for obj in objs:
357+
if isinstance(obj, type):
358+
n += 1
359+
ntypes.append(n)
360+
361+
assert ntypes[0] == ntypes[1]

0 commit comments

Comments
 (0)