Skip to content

Commit dbddfc5

Browse files
committed
Merge branch 'fix-647' into maint-3.1
2 parents 00b6e05 + 4137ded commit dbddfc5

10 files changed

Lines changed: 301 additions & 115 deletions

File tree

‎docs/news.rst‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -7,6 +7,13 @@
77
``psycopg`` release notes
88
=========================
99

10+
Psycopg 3.1.12 (unreleased)
11+
^^^^^^^^^^^^^^^^^^^^^^^^^^^
12+
13+
- Fix memory leak when `~register_*()` functions are called repeatedly
14+
(:ticket:`#647`).
15+
16+
1017
Current release
1118
---------------
1219

‎psycopg/psycopg/types/array.py‎

Lines changed: 32 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -311,41 +311,50 @@ def register_array(info: TypeInfo, context: Optional[AdaptContext] = None) -> No
311311
if not info.array_oid:
312312
raise ValueError(f"the type info {info} doesn't describe an array")
313313

314-
base: Type[Any]
315314
adapters = context.adapters if context else postgres.adapters
316315

317-
base = getattr(_psycopg, "ArrayLoader", ArrayLoader)
318-
name = f"{info.name.title()}{base.__name__}"
319-
attribs = {
320-
"base_oid": info.oid,
321-
"delimiter": info.delimiter.encode(),
322-
}
323-
loader = type(name, (base,), attribs)
316+
loader = _make_loader(info.name, info.oid, info.delimiter)
324317
adapters.register_loader(info.array_oid, loader)
325318

319+
# No need to make a new loader because the binary datum has all the info.
326320
loader = getattr(_psycopg, "ArrayBinaryLoader", ArrayBinaryLoader)
327321
adapters.register_loader(info.array_oid, loader)
328322

329-
base = ListDumper
330-
name = f"{info.name.title()}{base.__name__}"
331-
attribs = {
332-
"oid": info.array_oid,
333-
"element_oid": info.oid,
334-
"delimiter": info.delimiter.encode(),
335-
}
336-
dumper = type(name, (base,), attribs)
323+
dumper = _make_dumper(info.name, info.oid, info.array_oid, info.delimiter)
337324
adapters.register_dumper(None, dumper)
338325

339-
base = ListBinaryDumper
340-
name = f"{info.name.title()}{base.__name__}"
341-
attribs = {
342-
"oid": info.array_oid,
343-
"element_oid": info.oid,
344-
}
345-
dumper = type(name, (base,), attribs)
326+
dumper = _make_binary_dumper(info.name, info.oid, info.array_oid)
346327
adapters.register_dumper(None, dumper)
347328

348329

330+
# Cache all dynamically-generated types to avoid leaks in case the types
331+
# cannot be GC'd.
332+
333+
334+
@cache
335+
def _make_loader(name: str, oid: int, delimiter: str) -> Type[Loader]:
336+
# Note: caching this function is really needed because, if the C extension
337+
# is available, the resulting type cannot be GC'd, so calling
338+
# register_array() in a loop results in a leak. See #647.
339+
base = getattr(_psycopg, "ArrayLoader", ArrayLoader)
340+
attribs = {"base_oid": oid, "delimiter": delimiter.encode()}
341+
return type(f"{name.title()}{base.__name__}", (base,), attribs)
342+
343+
344+
@cache
345+
def _make_dumper(
346+
name: str, oid: int, array_oid: int, delimiter: str
347+
) -> Type[BaseListDumper]:
348+
attribs = {"oid": array_oid, "element_oid": oid, "delimiter": delimiter.encode()}
349+
return type(f"{name.title()}ListDumper", (ListDumper,), attribs)
350+
351+
352+
@cache
353+
def _make_binary_dumper(name: str, oid: int, array_oid: int) -> Type[BaseListDumper]:
354+
attribs = {"oid": array_oid, "element_oid": oid}
355+
return type(f"{name.title()}ListBinaryDumper", (ListBinaryDumper,), attribs)
356+
357+
349358
def register_default_adapters(context: AdaptContext) -> None:
350359
# The text dumper is more flexible as it can handle lists of mixed type,
351360
# so register it later.

‎psycopg/psycopg/types/composite.py‎

Lines changed: 65 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@
88
import struct
99
from collections import namedtuple
1010
from typing import Any, Callable, cast, Dict, Iterator, List, Optional
11-
from typing import Sequence, Tuple, Type
11+
from typing import NamedTuple, Sequence, Tuple, Type
1212

1313
from .. import pq
1414
from .. import abc
1515
from .. import postgres
1616
from ..adapt import Transformer, PyFormat, RecursiveDumper, Loader, Dumper
17+
from .._compat import cache
1718
from .._struct import pack_len, unpack_len
1819
from ..postgres import TEXT_OID
1920
from .._typeinfo import CompositeInfo as CompositeInfo # exported here
@@ -69,8 +70,8 @@ def dump(self, obj: Tuple[Any, ...]) -> bytes:
6970
class TupleBinaryDumper(Dumper):
7071
format = pq.Format.BINARY
7172

72-
# Subclasses must set an info
73-
info: CompositeInfo
73+
# Subclasses must set this info
74+
_field_types: Tuple[int, ...]
7475

7576
def __init__(self, cls: type, context: Optional[abc.AdaptContext] = None):
7677
super().__init__(cls, context)
@@ -80,17 +81,17 @@ def __init__(self, cls: type, context: Optional[abc.AdaptContext] = None):
8081
# in case the composite contains another composite. Make sure to use
8182
# a separate Transformer instance instead.
8283
self._tx = Transformer(context)
83-
self._tx.set_dumper_types(self.info.field_types, self.format)
84+
self._tx.set_dumper_types(self._field_types, self.format)
8485

85-
nfields = len(self.info.field_types)
86+
nfields = len(self._field_types)
8687
self._formats = (PyFormat.from_pq(self.format),) * nfields
8788

8889
def dump(self, obj: Tuple[Any, ...]) -> bytearray:
8990
out = bytearray(pack_len(len(obj)))
9091
adapted = self._tx.dump_sequence(obj, self._formats)
9192
for i in range(len(obj)):
9293
b = adapted[i]
93-
oid = self.info.field_types[i]
94+
oid = self._field_types[i]
9495
if b is not None:
9596
out += _pack_oidlen(oid, len(b))
9697
out += b
@@ -243,43 +244,27 @@ def register_composite(
243244
info.register(context)
244245

245246
if not factory:
246-
factory = namedtuple( # type: ignore
247-
_as_python_identifier(info.name),
248-
[_as_python_identifier(n) for n in info.field_names],
249-
)
247+
factory = _nt_from_info(info)
250248

251249
adapters = context.adapters if context else postgres.adapters
252250

253251
# generate and register a customized text loader
254-
loader: Type[BaseCompositeLoader] = type(
255-
f"{info.name.title()}Loader",
256-
(CompositeLoader,),
257-
{
258-
"factory": factory,
259-
"fields_types": info.field_types,
260-
},
261-
)
252+
loader: Type[BaseCompositeLoader]
253+
loader = _make_loader(info.name, tuple(info.field_types), factory)
262254
adapters.register_loader(info.oid, loader)
263255

264256
# generate and register a customized binary loader
265-
loader = type(
266-
f"{info.name.title()}BinaryLoader",
267-
(CompositeBinaryLoader,),
268-
{"factory": factory},
269-
)
257+
loader = _make_binary_loader(info.name, factory)
270258
adapters.register_loader(info.oid, loader)
271259

272260
# If the factory is a type, create and register dumpers for it
273261
if isinstance(factory, type):
274-
dumper = type(
275-
f"{info.name.title()}BinaryDumper",
276-
(TupleBinaryDumper,),
277-
{"oid": info.oid, "info": info},
278-
)
262+
dumper: Type[Dumper]
263+
dumper = _make_binary_dumper(info.name, info.oid, tuple(info.field_types))
279264
adapters.register_dumper(factory, dumper)
280265

281266
# Default to the text dumper because it is more flexible
282-
dumper = type(f"{info.name.title()}Dumper", (TupleDumper,), {"oid": info.oid})
267+
dumper = _make_dumper(info.name, info.oid)
283268
adapters.register_dumper(factory, dumper)
284269

285270
info.python_type = factory
@@ -290,3 +275,54 @@ def register_default_adapters(context: abc.AdaptContext) -> None:
290275
adapters.register_dumper(tuple, TupleDumper)
291276
adapters.register_loader("record", RecordLoader)
292277
adapters.register_loader("record", RecordBinaryLoader)
278+
279+
280+
def _nt_from_info(info: CompositeInfo) -> Type[NamedTuple]:
281+
name = _as_python_identifier(info.name)
282+
fields = tuple(_as_python_identifier(n) for n in info.field_names)
283+
return _make_nt(name, fields)
284+
285+
286+
# Cache all dynamically-generated types to avoid leaks in case the types
287+
# cannot be GC'd.
288+
289+
290+
@cache
291+
def _make_nt(name: str, fields: Tuple[str, ...]) -> Type[NamedTuple]:
292+
return namedtuple(name, fields) # type: ignore[return-value]
293+
294+
295+
@cache
296+
def _make_loader(
297+
name: str, types: Tuple[int, ...], factory: Callable[..., Any]
298+
) -> Type[BaseCompositeLoader]:
299+
return type(
300+
f"{name.title()}Loader",
301+
(CompositeLoader,),
302+
{"factory": factory, "fields_types": list(types)},
303+
)
304+
305+
306+
@cache
307+
def _make_binary_loader(
308+
name: str, factory: Callable[..., Any]
309+
) -> Type[BaseCompositeLoader]:
310+
return type(
311+
f"{name.title()}BinaryLoader", (CompositeBinaryLoader,), {"factory": factory}
312+
)
313+
314+
315+
@cache
316+
def _make_dumper(name: str, oid: int) -> Type[TupleDumper]:
317+
return type(f"{name.title()}Dumper", (TupleDumper,), {"oid": oid})
318+
319+
320+
@cache
321+
def _make_binary_dumper(
322+
name: str, oid: int, field_types: Tuple[int, ...]
323+
) -> Type[TupleBinaryDumper]:
324+
return type(
325+
f"{name.title()}BinaryDumper",
326+
(TupleBinaryDumper,),
327+
{"oid": oid, "_field_types": field_types},
328+
)

0 commit comments

Comments
 (0)