Skip to content

Commit 4137ded

Browse files
committed
fix: cache all dynamically generated adapter types
These are not currently a leak in cPython, but I don't think it's guaranteed anywhere, and it might well not be the case in other Python implementations. So, as a matter of hygiene, make sure that calling register_*() in a loop doesn't create an unbound number of new types.
1 parent d7eea49 commit 4137ded

7 files changed

Lines changed: 238 additions & 92 deletions

File tree

‎psycopg/psycopg/types/composite.py‎

Lines changed: 65 additions & 29 deletions
Original file line numberDiff line numberDiff line change
@@ -8,12 +8,13 @@
88
import struct
99
from collections import namedtuple
1010
from typing import Any, Callable, cast, Dict, Iterator, List, Optional
11-
from typing import Sequence, Tuple, Type
11+
from typing import NamedTuple, Sequence, Tuple, Type
1212

1313
from .. import pq
1414
from .. import abc
1515
from .. import postgres
1616
from ..adapt import Transformer, PyFormat, RecursiveDumper, Loader, Dumper
17+
from .._compat import cache
1718
from .._struct import pack_len, unpack_len
1819
from ..postgres import TEXT_OID
1920
from .._typeinfo import CompositeInfo as CompositeInfo # exported here
@@ -69,8 +70,8 @@ def dump(self, obj: Tuple[Any, ...]) -> bytes:
6970
class TupleBinaryDumper(Dumper):
7071
format = pq.Format.BINARY
7172

72-
# Subclasses must set an info
73-
info: CompositeInfo
73+
# Subclasses must set this info
74+
_field_types: Tuple[int, ...]
7475

7576
def __init__(self, cls: type, context: Optional[abc.AdaptContext] = None):
7677
super().__init__(cls, context)
@@ -80,17 +81,17 @@ def __init__(self, cls: type, context: Optional[abc.AdaptContext] = None):
8081
# in case the composite contains another composite. Make sure to use
8182
# a separate Transformer instance instead.
8283
self._tx = Transformer(context)
83-
self._tx.set_dumper_types(self.info.field_types, self.format)
84+
self._tx.set_dumper_types(self._field_types, self.format)
8485

85-
nfields = len(self.info.field_types)
86+
nfields = len(self._field_types)
8687
self._formats = (PyFormat.from_pq(self.format),) * nfields
8788

8889
def dump(self, obj: Tuple[Any, ...]) -> bytearray:
8990
out = bytearray(pack_len(len(obj)))
9091
adapted = self._tx.dump_sequence(obj, self._formats)
9192
for i in range(len(obj)):
9293
b = adapted[i]
93-
oid = self.info.field_types[i]
94+
oid = self._field_types[i]
9495
if b is not None:
9596
out += _pack_oidlen(oid, len(b))
9697
out += b
@@ -243,43 +244,27 @@ def register_composite(
243244
info.register(context)
244245

245246
if not factory:
246-
factory = namedtuple( # type: ignore
247-
_as_python_identifier(info.name),
248-
[_as_python_identifier(n) for n in info.field_names],
249-
)
247+
factory = _nt_from_info(info)
250248

251249
adapters = context.adapters if context else postgres.adapters
252250

253251
# generate and register a customized text loader
254-
loader: Type[BaseCompositeLoader] = type(
255-
f"{info.name.title()}Loader",
256-
(CompositeLoader,),
257-
{
258-
"factory": factory,
259-
"fields_types": info.field_types,
260-
},
261-
)
252+
loader: Type[BaseCompositeLoader]
253+
loader = _make_loader(info.name, tuple(info.field_types), factory)
262254
adapters.register_loader(info.oid, loader)
263255

264256
# generate and register a customized binary loader
265-
loader = type(
266-
f"{info.name.title()}BinaryLoader",
267-
(CompositeBinaryLoader,),
268-
{"factory": factory},
269-
)
257+
loader = _make_binary_loader(info.name, factory)
270258
adapters.register_loader(info.oid, loader)
271259

272260
# If the factory is a type, create and register dumpers for it
273261
if isinstance(factory, type):
274-
dumper = type(
275-
f"{info.name.title()}BinaryDumper",
276-
(TupleBinaryDumper,),
277-
{"oid": info.oid, "info": info},
278-
)
262+
dumper: Type[Dumper]
263+
dumper = _make_binary_dumper(info.name, info.oid, tuple(info.field_types))
279264
adapters.register_dumper(factory, dumper)
280265

281266
# Default to the text dumper because it is more flexible
282-
dumper = type(f"{info.name.title()}Dumper", (TupleDumper,), {"oid": info.oid})
267+
dumper = _make_dumper(info.name, info.oid)
283268
adapters.register_dumper(factory, dumper)
284269

285270
info.python_type = factory
@@ -290,3 +275,54 @@ def register_default_adapters(context: abc.AdaptContext) -> None:
290275
adapters.register_dumper(tuple, TupleDumper)
291276
adapters.register_loader("record", RecordLoader)
292277
adapters.register_loader("record", RecordBinaryLoader)
278+
279+
280+
def _nt_from_info(info: CompositeInfo) -> Type[NamedTuple]:
281+
name = _as_python_identifier(info.name)
282+
fields = tuple(_as_python_identifier(n) for n in info.field_names)
283+
return _make_nt(name, fields)
284+
285+
286+
# Cache all dynamically-generated types to avoid leaks in case the types
287+
# cannot be GC'd.
288+
289+
290+
@cache
291+
def _make_nt(name: str, fields: Tuple[str, ...]) -> Type[NamedTuple]:
292+
return namedtuple(name, fields) # type: ignore[return-value]
293+
294+
295+
@cache
296+
def _make_loader(
297+
name: str, types: Tuple[int, ...], factory: Callable[..., Any]
298+
) -> Type[BaseCompositeLoader]:
299+
return type(
300+
f"{name.title()}Loader",
301+
(CompositeLoader,),
302+
{"factory": factory, "fields_types": list(types)},
303+
)
304+
305+
306+
@cache
307+
def _make_binary_loader(
308+
name: str, factory: Callable[..., Any]
309+
) -> Type[BaseCompositeLoader]:
310+
return type(
311+
f"{name.title()}BinaryLoader", (CompositeBinaryLoader,), {"factory": factory}
312+
)
313+
314+
315+
@cache
316+
def _make_dumper(name: str, oid: int) -> Type[TupleDumper]:
317+
return type(f"{name.title()}Dumper", (TupleDumper,), {"oid": oid})
318+
319+
320+
@cache
321+
def _make_binary_dumper(
322+
name: str, oid: int, field_types: Tuple[int, ...]
323+
) -> Type[TupleBinaryDumper]:
324+
return type(
325+
f"{name.title()}BinaryDumper",
326+
(TupleBinaryDumper,),
327+
{"oid": oid, "_field_types": field_types},
328+
)

‎psycopg/psycopg/types/enum.py‎

Lines changed: 66 additions & 23 deletions
Original file line numberDiff line numberDiff line change
@@ -2,7 +2,7 @@
22
Adapters for the enum type.
33
"""
44
from enum import Enum
5-
from typing import Any, Dict, Generic, Optional, Mapping, Sequence
5+
from typing import Dict, Generic, Optional, Mapping, Sequence
66
from typing import Tuple, Type, TypeVar, Union, cast
77
from typing_extensions import TypeAlias
88

@@ -11,6 +11,7 @@
1111
from ..pq import Format
1212
from ..abc import AdaptContext
1313
from ..adapt import Buffer, Dumper, Loader
14+
from .._compat import cache
1415
from .._encodings import conn_encoding
1516
from .._typeinfo import EnumInfo as EnumInfo # exported here
1617

@@ -20,6 +21,13 @@
2021
EnumLoadMap: TypeAlias = Dict[bytes, E]
2122
EnumMapping: TypeAlias = Union[Mapping[E, str], Sequence[Tuple[E, str]], None]
2223

24+
# Hashable versions
25+
_HEnumDumpMap: TypeAlias = Tuple[Tuple[E, bytes], ...]
26+
_HEnumLoadMap: TypeAlias = Tuple[Tuple[bytes, E], ...]
27+
28+
TEXT = Format.TEXT
29+
BINARY = Format.BINARY
30+
2331

2432
class _BaseEnumLoader(Loader, Generic[E]):
2533
"""
@@ -69,7 +77,7 @@ def dump(self, value: E) -> Buffer:
6977

7078

7179
class EnumBinaryDumper(EnumDumper):
72-
format = Format.BINARY
80+
format = BINARY
7381

7482

7583
def register_enum(
@@ -94,43 +102,78 @@ def register_enum(
94102
raise TypeError("no info passed. Is the requested enum available?")
95103

96104
if enum is None:
97-
enum = cast(Type[E], Enum(info.name.title(), info.labels, module=__name__))
105+
enum = cast(Type[E], _make_enum(info.name, tuple(info.labels)))
98106

99107
info.enum = enum
100108
adapters = context.adapters if context else postgres.adapters
101109
info.register(context)
102110

103111
load_map = _make_load_map(info, enum, mapping, context)
104-
attribs: Dict[str, Any] = {"enum": info.enum, "_load_map": load_map}
105112

106-
name = f"{info.name.title()}Loader"
107-
loader = type(name, (_BaseEnumLoader,), attribs)
113+
loader = _make_loader(info.name, info.enum, load_map)
108114
adapters.register_loader(info.oid, loader)
109115

110-
name = f"{info.name.title()}BinaryLoader"
111-
loader = type(name, (_BaseEnumLoader,), {**attribs, "format": Format.BINARY})
116+
loader = _make_binary_loader(info.name, info.enum, load_map)
112117
adapters.register_loader(info.oid, loader)
113118

114119
dump_map = _make_dump_map(info, enum, mapping, context)
115-
attribs = {"oid": info.oid, "enum": info.enum, "_dump_map": dump_map}
116120

117-
name = f"{enum.__name__}Dumper"
118-
dumper = type(name, (_BaseEnumDumper,), attribs)
121+
dumper = _make_dumper(info.enum, info.oid, dump_map)
119122
adapters.register_dumper(info.enum, dumper)
120123

121-
name = f"{enum.__name__}BinaryDumper"
122-
dumper = type(name, (_BaseEnumDumper,), {**attribs, "format": Format.BINARY})
124+
dumper = _make_binary_dumper(info.enum, info.oid, dump_map)
123125
adapters.register_dumper(info.enum, dumper)
124126

125127

128+
# Cache all dynamically-generated types to avoid leaks in case the types
129+
# cannot be GC'd.
130+
131+
132+
@cache
133+
def _make_enum(name: str, labels: Tuple[str, ...]) -> Enum:
134+
return Enum(name.title(), labels, module=__name__)
135+
136+
137+
@cache
138+
def _make_loader(
139+
name: str, enum: Type[Enum], load_map: _HEnumLoadMap[E]
140+
) -> Type[_BaseEnumLoader[E]]:
141+
attribs = {"enum": enum, "_load_map": dict(load_map)}
142+
return type(f"{name.title()}Loader", (_BaseEnumLoader,), attribs)
143+
144+
145+
@cache
146+
def _make_binary_loader(
147+
name: str, enum: Type[Enum], load_map: _HEnumLoadMap[E]
148+
) -> Type[_BaseEnumLoader[E]]:
149+
attribs = {"enum": enum, "_load_map": dict(load_map), "format": BINARY}
150+
return type(f"{name.title()}BinaryLoader", (_BaseEnumLoader,), attribs)
151+
152+
153+
@cache
154+
def _make_dumper(
155+
enum: Type[Enum], oid: int, dump_map: _HEnumDumpMap[E]
156+
) -> Type[_BaseEnumDumper[E]]:
157+
attribs = {"enum": enum, "oid": oid, "_dump_map": dict(dump_map)}
158+
return type(f"{enum.__name__}Dumper", (_BaseEnumDumper,), attribs)
159+
160+
161+
@cache
162+
def _make_binary_dumper(
163+
enum: Type[Enum], oid: int, dump_map: _HEnumDumpMap[E]
164+
) -> Type[_BaseEnumDumper[E]]:
165+
attribs = {"enum": enum, "oid": oid, "_dump_map": dict(dump_map), "format": BINARY}
166+
return type(f"{enum.__name__}BinaryDumper", (_BaseEnumDumper,), attribs)
167+
168+
126169
def _make_load_map(
127170
info: EnumInfo,
128171
enum: Type[E],
129172
mapping: EnumMapping[E],
130173
context: Optional[AdaptContext],
131-
) -> EnumLoadMap[E]:
174+
) -> _HEnumLoadMap[E]:
132175
enc = conn_encoding(context.connection if context else None)
133-
rv: EnumLoadMap[E] = {}
176+
rv = []
134177
for label in info.labels:
135178
try:
136179
member = enum[label]
@@ -139,37 +182,37 @@ def _make_load_map(
139182
# will get a DataError on fetch.
140183
pass
141184
else:
142-
rv[label.encode(enc)] = member
185+
rv.append((label.encode(enc), member))
143186

144187
if mapping:
145188
if isinstance(mapping, Mapping):
146189
mapping = list(mapping.items())
147190

148191
for member, label in mapping:
149-
rv[label.encode(enc)] = member
192+
rv.append((label.encode(enc), member))
150193

151-
return rv
194+
return tuple(rv)
152195

153196

154197
def _make_dump_map(
155198
info: EnumInfo,
156199
enum: Type[E],
157200
mapping: EnumMapping[E],
158201
context: Optional[AdaptContext],
159-
) -> EnumDumpMap[E]:
202+
) -> _HEnumDumpMap[E]:
160203
enc = conn_encoding(context.connection if context else None)
161-
rv: EnumDumpMap[E] = {}
204+
rv = []
162205
for member in enum:
163-
rv[member] = member.name.encode(enc)
206+
rv.append((member, member.name.encode(enc)))
164207

165208
if mapping:
166209
if isinstance(mapping, Mapping):
167210
mapping = list(mapping.items())
168211

169212
for member, label in mapping:
170-
rv[member] = label.encode(enc)
213+
rv.append((member, label.encode(enc)))
171214

172-
return rv
215+
return tuple(rv)
173216

174217

175218
def register_default_adapters(context: AdaptContext) -> None:

‎psycopg/psycopg/types/hstore.py‎

Lines changed: 21 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -5,13 +5,14 @@
55
# Copyright (C) 2021 The Psycopg Team
66

77
import re
8-
from typing import Dict, List, Optional
8+
from typing import Dict, List, Optional, Type
99
from typing_extensions import TypeAlias
1010

1111
from .. import errors as e
1212
from .. import postgres
1313
from ..abc import Buffer, AdaptContext
1414
from ..adapt import PyFormat, RecursiveDumper, RecursiveLoader
15+
from .._compat import cache
1516
from ..postgres import TEXT_OID
1617
from .._typeinfo import TypeInfo
1718

@@ -121,10 +122,25 @@ def register_hstore(info: TypeInfo, context: Optional[AdaptContext] = None) -> N
121122
adapters = context.adapters if context else postgres.adapters
122123

123124
# Generate and register a customized text dumper
124-
class HstoreDumper(BaseHstoreDumper):
125-
oid = info.oid
126-
127-
adapters.register_dumper(dict, HstoreDumper)
125+
adapters.register_dumper(dict, _make_hstore_dumper(info.oid))
128126

129127
# register the text loader on the oid
130128
adapters.register_loader(info.oid, HstoreLoader)
129+
130+
131+
# Cache all dynamically-generated types to avoid leaks in case the types
132+
# cannot be GC'd.
133+
134+
135+
@cache
136+
def _make_hstore_dumper(oid_in: int) -> Type[BaseHstoreDumper]:
137+
"""
138+
Return an hstore dumper class configured using `oid_in`.
139+
140+
Avoid to create new classes if the oid configured is the same.
141+
"""
142+
143+
class HstoreDumper(BaseHstoreDumper):
144+
oid = oid_in
145+
146+
return HstoreDumper

0 commit comments

Comments
 (0)