Skip to content

Commit 143386f

Browse files
Python: fix(core): restrict unpickler module-prefix allowlist to types only (#5923)
* fix(core): harden restricted pickle attribute resolution * fix(core): validate nested pickle types against allowlist --------- Co-authored-by: White-Mouse <[email protected]> Co-authored-by: Evan Mattson <[email protected]> Co-authored-by: Evan Mattson <[email protected]>
1 parent 47c8e29 commit 143386f

2 files changed

Lines changed: 224 additions & 8 deletions

File tree

‎python/packages/core/agent_framework/_workflows/_checkpoint_encoding.py‎

Lines changed: 60 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -20,9 +20,9 @@
2020
format uses Python's ``pickle`` module which can execute arbitrary code during
2121
deserialization. The ``RestrictedUnpickler`` provides a defense-in-depth
2222
allowlist that limits instantiable classes, but it is **not** a security
23-
boundary — certain allowlisted builtins (e.g. ``getattr``) are required for
24-
legitimate object reconstruction (enums, named tuples) and cannot be removed
25-
without breaking compatibility.
23+
boundary — certain reconstruction helpers are required for legitimate object
24+
reconstruction and application-defined types can provide custom pickle
25+
behavior.
2626
2727
Developers **must** ensure that:
2828
@@ -50,6 +50,7 @@
5050
import io
5151
import logging
5252
import pickle # nosec # ruff:ignore[suspicious-pickle-import]
53+
from enum import EnumMeta
5354
from typing import Any, cast
5455

5556
from ..exceptions import WorkflowCheckpointException
@@ -81,9 +82,9 @@
8182
"agent_framework._workflows._checkpoint_encoding:encode_checkpoint_value",
8283
})
8384

84-
# Built-in types considered safe for checkpoint deserialization.
85+
# Built-in globals considered safe for checkpoint deserialization.
8586
# Each entry is a ``module:qualname`` string matching the format produced by
86-
# :func:`_type_to_key`. These are the classes for which pickle's
87+
# :func:`_type_to_key`. These are the globals for which pickle's
8788
# ``find_class`` will be called when unpickling common Python value types.
8889
_BUILTIN_ALLOWED_TYPE_KEYS: frozenset[str] = frozenset({
8990
# builtins
@@ -103,7 +104,8 @@
103104
"builtins:dict",
104105
"builtins:tuple",
105106
"builtins:type",
106-
# getattr is used by pickle to reconstruct enum members
107+
# getattr is used by pickle to reconstruct nested types. find_class
108+
# substitutes a restricted implementation during unpickling.
107109
"builtins:getattr",
108110
# copyreg helpers used by pickle for object reconstruction
109111
"copyreg:_reconstructor",
@@ -123,6 +125,12 @@
123125
"collections:deque",
124126
})
125127

128+
_GETATTR_GLOBAL_KEYS: frozenset[str] = frozenset({
129+
"builtins:getattr",
130+
# Protocol 2 pickles use Python 2's module name for the same global.
131+
"__builtin__:getattr",
132+
})
133+
126134

127135
class _RestrictedUnpickler(pickle.Unpickler): # ruff:ignore[suspicious-pickle-usage]
128136
"""Unpickler that restricts which classes may be instantiated.
@@ -137,15 +145,58 @@ def __init__(self, data: bytes, allowed_types: frozenset[str]) -> None:
137145
super().__init__(io.BytesIO(data))
138146
self._allowed_types = allowed_types
139147

140-
def find_class(self, module: str, name: str) -> type:
148+
def _is_allowed_type(self, resolved: type) -> bool:
149+
type_key = _type_to_key(resolved)
150+
return (
151+
type_key in _BUILTIN_ALLOWED_TYPE_KEYS
152+
or type_key in self._allowed_types
153+
or resolved.__module__.startswith(_FRAMEWORK_MODULE_PREFIX)
154+
or resolved.__module__.startswith(_OPENAI_MODULE_PREFIX)
155+
)
156+
157+
def _restricted_getattr(self, obj: Any, name: str) -> Any:
158+
"""Resolve an allowlisted nested type or named enum member."""
159+
if not isinstance(obj, type) or not isinstance(name, str):
160+
raise pickle.UnpicklingError("Checkpoint deserialization blocked for unsafe attribute traversal.")
161+
162+
resolved = getattr(obj, name)
163+
if isinstance(resolved, type):
164+
if not self._is_allowed_type(resolved):
165+
type_key = _type_to_key(resolved)
166+
raise pickle.UnpicklingError(
167+
f"Checkpoint deserialization blocked for nested type '{type_key}'. "
168+
f"Include the nested type in 'allowed_types' before loading the checkpoint."
169+
)
170+
return resolved
171+
172+
# enum.pickle_by_enum_name reconstructs a member with getattr(EnumClass, member_name).
173+
if isinstance(obj, EnumMeta) and self._is_allowed_type(obj):
174+
members = obj.__members__
175+
if name in members and members[name] is resolved:
176+
return resolved
177+
178+
raise pickle.UnpicklingError(
179+
f"Checkpoint deserialization blocked for non-type attribute '{obj.__module__}:{obj.__qualname__}.{name}'."
180+
)
181+
182+
def find_class(self, module: str, name: str) -> Any:
141183
type_key = f"{module}:{name}"
142184

143185
if type_key in _BLOCKED_FRAMEWORK_GLOBAL_KEYS:
144186
raise pickle.UnpicklingError(f"Checkpoint deserialization blocked for type '{type_key}'.")
145187

146-
if type_key in _BUILTIN_ALLOWED_TYPE_KEYS or type_key in self._allowed_types:
188+
if type_key in _GETATTR_GLOBAL_KEYS:
189+
return self._restricted_getattr
190+
191+
if type_key in _BUILTIN_ALLOWED_TYPE_KEYS:
147192
return super().find_class(module, name) # nosec
148193

194+
if type_key in self._allowed_types:
195+
resolved = super().find_class(module, name) # nosec
196+
if isinstance(resolved, type):
197+
return resolved
198+
raise pickle.UnpicklingError(f"Checkpoint deserialization blocked for non-type global '{type_key}'.")
199+
149200
if module.startswith(_FRAMEWORK_MODULE_PREFIX) or module.startswith(_OPENAI_MODULE_PREFIX):
150201
# Pickle dotted names traverse attributes on an allowed module; keep the prefix allowlist to concrete
151202
# top-level classes rather than helper callables reachable through module attributes.
@@ -154,6 +205,7 @@ def find_class(self, module: str, name: str) -> type:
154205
resolved = super().find_class(module, name) # nosec
155206
if isinstance(resolved, type):
156207
return resolved
208+
raise pickle.UnpicklingError(f"Checkpoint deserialization blocked for non-type global '{type_key}'.")
157209

158210
raise pickle.UnpicklingError(
159211
f"Checkpoint deserialization blocked for type '{type_key}'. "

‎python/packages/core/tests/workflow/test_checkpoint_unrestricted_pickle.py‎

Lines changed: 164 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,7 @@
1212
"""
1313

1414
import base64
15+
import enum
1516
import os
1617
import pickle
1718
import tempfile
@@ -49,6 +50,39 @@ def __reduce__(self) -> tuple[Any, tuple[str]]:
4950
return (_base64_to_unpickle, (self.nested_payload,))
5051

5152

53+
class _NestedTypeContainer:
54+
@dataclass
55+
class NestedType:
56+
value: int
57+
58+
59+
class _NestedSetStateContainer:
60+
class Evil:
61+
setstate_called = False
62+
63+
def __init__(self) -> None:
64+
self.value = 42
65+
66+
def __setstate__(self, state: dict[str, object]) -> None:
67+
type(self).setstate_called = True
68+
self.__dict__.update(state)
69+
70+
71+
class _NamedPickleColor(enum.Enum):
72+
RED = 1
73+
BLUE = 2
74+
CRIMSON = 1
75+
76+
def __reduce_ex__(self, proto: object) -> tuple[Any, tuple[type[enum.Enum], str]]:
77+
del proto
78+
return (getattr, (type(self), self.name))
79+
80+
81+
class _EnumNonMemberAttributePayload:
82+
def __reduce__(self):
83+
return (getattr, (_NamedPickleColor, "__members__"))
84+
85+
5286
def test_restricted_decode_blocks_arbitrary_callable():
5387
"""Restricted decoding blocks arbitrary module-level callables."""
5488
pickled = pickle.dumps(os.getpid, protocol=pickle.HIGHEST_PROTOCOL)
@@ -300,6 +334,136 @@ def test_restricted_unpickler_raises_pickle_error():
300334
unpickler.load()
301335

302336

337+
def test_restricted_decode_rejects_non_type_global_in_allowed_types():
338+
"""Explicit allowed_types entries must resolve to types."""
339+
from agent_framework._workflows._checkpoint_encoding import _RestrictedUnpickler
340+
341+
unpickler = _RestrictedUnpickler(pickle.dumps(object), frozenset({"os:getpid"}))
342+
with pytest.raises(pickle.UnpicklingError, match="non-type global"):
343+
unpickler.find_class("os", "getpid")
344+
345+
346+
def test_restricted_decode_rejects_non_type_global_under_prefix():
347+
"""Allowed package prefixes must not expose arbitrary module globals."""
348+
from agent_framework._workflows._checkpoint_encoding import _RestrictedUnpickler
349+
350+
unpickler = _RestrictedUnpickler(pickle.dumps(object), frozenset())
351+
with pytest.raises(pickle.UnpicklingError, match="non-type global"):
352+
unpickler.find_class(
353+
"agent_framework._workflows._checkpoint_encoding",
354+
"_value_type_to_key",
355+
)
356+
357+
358+
@pytest.mark.parametrize("protocol", [2, 3])
359+
def test_restricted_decode_allows_explicitly_listed_nested_type(protocol: int):
360+
"""Protocol 2/3 nested types load when both the container and nested type are allowed."""
361+
original = _NestedTypeContainer.NestedType(42)
362+
checkpoint_value = {
363+
_PICKLE_MARKER: base64.b64encode(pickle.dumps(original, protocol=protocol)).decode("ascii"),
364+
_TYPE_MARKER: f"{type(original).__module__}:{type(original).__qualname__}",
365+
}
366+
allowed_types = frozenset({
367+
f"{_NestedTypeContainer.__module__}:{_NestedTypeContainer.__qualname__}",
368+
f"{_NestedTypeContainer.NestedType.__module__}:{_NestedTypeContainer.NestedType.__qualname__}",
369+
})
370+
371+
decoded = decode_checkpoint_value(checkpoint_value, allowed_types=allowed_types)
372+
373+
assert isinstance(decoded, _NestedTypeContainer.NestedType)
374+
assert decoded.value == 42
375+
376+
377+
@pytest.mark.parametrize("protocol", [2, 3])
378+
def test_restricted_decode_blocks_unlisted_nested_type_before_setstate(protocol: int):
379+
"""Allowing only a container must not allow its nested types or run their state hooks."""
380+
original = _NestedSetStateContainer.Evil()
381+
checkpoint_value = {
382+
_PICKLE_MARKER: base64.b64encode(pickle.dumps(original, protocol=protocol)).decode("ascii"),
383+
_TYPE_MARKER: f"{type(original).__module__}:{type(original).__qualname__}",
384+
}
385+
outer_type_key = f"{_NestedSetStateContainer.__module__}:{_NestedSetStateContainer.__qualname__}"
386+
_NestedSetStateContainer.Evil.setstate_called = False
387+
388+
with pytest.raises(WorkflowCheckpointException, match="nested type"):
389+
decode_checkpoint_value(checkpoint_value, allowed_types=frozenset({outer_type_key}))
390+
391+
assert not _NestedSetStateContainer.Evil.setstate_called
392+
393+
394+
@pytest.mark.parametrize("protocol", [2, 3, 4, 5])
395+
def test_restricted_decode_allows_allowlisted_enum_member_by_name(protocol: int):
396+
"""Named enum members load only through their allowlisted enum class."""
397+
original = _NamedPickleColor.RED
398+
checkpoint_value = {
399+
_PICKLE_MARKER: base64.b64encode(pickle.dumps(original, protocol=protocol)).decode("ascii"),
400+
_TYPE_MARKER: f"{type(original).__module__}:{type(original).__qualname__}",
401+
}
402+
enum_type_key = f"{_NamedPickleColor.__module__}:{_NamedPickleColor.__qualname__}"
403+
404+
decoded = decode_checkpoint_value(checkpoint_value, allowed_types=frozenset({enum_type_key}))
405+
406+
assert decoded is _NamedPickleColor.RED
407+
408+
409+
def test_restricted_encoder_round_trips_allowlisted_enum_member_by_name():
410+
"""The current checkpoint encoder's protocol preserves named enum members."""
411+
enum_type_key = f"{_NamedPickleColor.__module__}:{_NamedPickleColor.__qualname__}"
412+
413+
encoded = encode_checkpoint_value(_NamedPickleColor.RED)
414+
decoded = decode_checkpoint_value(encoded, allowed_types=frozenset({enum_type_key}))
415+
416+
assert decoded is _NamedPickleColor.RED
417+
418+
419+
def test_restricted_decode_blocks_non_member_attribute_on_allowlisted_enum():
420+
"""Allowing an enum class does not expose its other attributes through getattr."""
421+
payload = pickle.dumps(_EnumNonMemberAttributePayload(), protocol=pickle.HIGHEST_PROTOCOL)
422+
checkpoint_value = {
423+
_PICKLE_MARKER: base64.b64encode(payload).decode("ascii"),
424+
_TYPE_MARKER: "builtins:dict",
425+
}
426+
enum_type_key = f"{_NamedPickleColor.__module__}:{_NamedPickleColor.__qualname__}"
427+
428+
with pytest.raises(WorkflowCheckpointException, match="non-type attribute"):
429+
decode_checkpoint_value(checkpoint_value, allowed_types=frozenset({enum_type_key}))
430+
431+
432+
def test_restricted_decode_blocks_member_of_unlisted_enum():
433+
"""Enum member reconstruction still requires the enum class in allowed_types."""
434+
original = _NamedPickleColor.RED
435+
checkpoint_value = {
436+
_PICKLE_MARKER: base64.b64encode(pickle.dumps(original, protocol=pickle.HIGHEST_PROTOCOL)).decode("ascii"),
437+
_TYPE_MARKER: f"{type(original).__module__}:{type(original).__qualname__}",
438+
}
439+
440+
with pytest.raises(WorkflowCheckpointException, match="deserialization blocked"):
441+
decode_checkpoint_value(checkpoint_value, allowed_types=frozenset())
442+
443+
444+
def test_restricted_decode_blocks_getattr_globals_pickle_loads_chain():
445+
"""Restricted decoding blocks attribute traversal to an inner unrestricted pickle load."""
446+
inner_pickle = b"cbuiltins\neval\n(V40 + 2\ntR."
447+
escaped_inner_pickle = inner_pickle.decode("ascii").replace("\\", "\\u005c").replace("\n", "\\u000a")
448+
# The outer pickle memoizes getattr and walks __init__.__globals__["pickle"].loads.
449+
payload = (
450+
b"cbuiltins\ngetattr\np0\n0"
451+
b"g0\n(cagent_framework._workflows._checkpoint_encoding\n_RestrictedUnpickler\nV__init__\ntRp1\n0"
452+
b"g0\n(g1\nV__globals__\ntRp2\n0"
453+
b"g0\n(cbuiltins\ndict\nV__getitem__\ntRp3\n0"
454+
b"g3\n(g2\nVpickle\ntRp4\n0"
455+
b"g0\n(g4\nVloads\ntRp5\n0"
456+
b"g5\n(cbuiltins\nbytearray\n(V" + escaped_inner_pickle.encode("ascii") + b"\nVascii\ntRtR."
457+
)
458+
checkpoint_value = {
459+
_PICKLE_MARKER: base64.b64encode(payload).decode("ascii"),
460+
_TYPE_MARKER: "builtins:int",
461+
}
462+
463+
with pytest.raises(WorkflowCheckpointException, match="deserialization blocked"):
464+
decode_checkpoint_value(checkpoint_value, allowed_types=frozenset())
465+
466+
303467
def test_restricted_decode_allows_openai_types():
304468
"""OpenAI SDK types are always allowed during restricted deserialization."""
305469
from openai.types.chat.chat_completion import ChatCompletion, Choice

0 commit comments

Comments
 (0)