Classes:

Pickler

Unpickler

Functions:

dump(object, file)

dumps(object) -> string

load(file) -> object

Misc variables:

format_version

compatible_formats

"""

from copyreg import dispatch_table

from copyreg import _extension_registry, _inverted_registry, _extension_cache

from sys import maxsize

from struct import pack, unpack

"Unpickler", "dump", "dumps", "load", "loads"]

try:

from _pickle import PickleBuffer

__all__.append("PickleBuffer")

_HAVE_PICKLE_BUFFER = True

except ImportError:

_HAVE_PICKLE_BUFFER = False

"1.2", # Original protocol 1

"1.3", # Protocol 1 with BINFLOAT added

"2.0", # Protocol 2

# Only bump this if the oldest still supported version of Python already

# includes it.

DEFAULT_PROTOCOL = 4

pass

class PicklingError(PickleError):

"""This exception is raised when an unpicklable object is passed to the

dump() method.

"""

pass

class UnpicklingError(PickleError):

"""This exception is raised when there is a problem unpickling an object,

such as a security violation.

Note that other exceptions may also be raised during unpickling, including

(but not necessarily limited to) AttributeError, EOFError, ImportError,

and IndexError.

"""

pass

# An instance of _Stop is raised by Unpickler.load_stop() in response to

# the STOP opcode, passing the object that is the result of unpickling.

class _Stop(Exception):

def __init__(self, value):

self.value = value

try:

from org.python.core import PyStringMap

PyStringMap = None

# Pickle opcodes. See pickletools.py for extensive docs. The listing

# here is in kind-of alphabetical order of 1-character pickle code.

# pickletools groups them by purpose.

MARK = b'(' # push special markobject on stack

STOP = b'.' # every pickle ends with STOP

POP = b'0' # discard topmost stack item

POP_MARK = b'1' # discard stack top through topmost markobject

DUP = b'2' # duplicate top stack item

FLOAT = b'F' # push float object; decimal string argument

INT = b'I' # push integer or bool; decimal string argument

BININT = b'J' # push four-byte signed int

BININT1 = b'K' # push 1-byte unsigned int

LONG = b'L' # push long; decimal string argument

BININT2 = b'M' # push 2-byte unsigned int

NONE = b'N' # push None

PERSID = b'P' # push persistent object; id is taken from string arg

BINPERSID = b'Q' # " " " ; " " " " stack

REDUCE = b'R' # apply callable to argtuple, both on stack

STRING = b'S' # push string; NL-terminated string argument

BINSTRING = b'T' # push string; counted binary string argument

SHORT_BINSTRING= b'U' # " " ; " " " " < 256 bytes

UNICODE = b'V' # push Unicode string; raw-unicode-escaped'd argument

BINUNICODE = b'X' # " " " ; counted UTF-8 string argument

APPEND = b'a' # append stack top to list below it

BUILD = b'b' # call __setstate__ or __dict__.update()

GLOBAL = b'c' # push self.find_class(modname, name); 2 string args

DICT = b'd' # build a dict from stack items

EMPTY_DICT = b'}' # push empty dict

APPENDS = b'e' # extend list on stack by topmost stack slice

GET = b'g' # push item from memo on stack; index is string arg

BINGET = b'h' # " " " " " " ; " " 1-byte arg

INST = b'i' # build & push class instance

LONG_BINGET = b'j' # push item from memo on stack; index is 4-byte arg

LIST = b'l' # build list from topmost stack items

EMPTY_LIST = b']' # push empty list

OBJ = b'o' # build & push class instance

PUT = b'p' # store stack top in memo; index is string arg

BINPUT = b'q' # " " " " " ; " " 1-byte arg

LONG_BINPUT = b'r' # " " " " " ; " " 4-byte arg

SETITEM = b's' # add key+value pair to dict

TUPLE = b't' # build tuple from topmost stack items

EMPTY_TUPLE = b')' # push empty tuple

SETITEMS = b'u' # modify dict by adding topmost key+value pairs

BINFLOAT = b'G' # push float; arg is 8-byte float encoding

TRUE = b'I01\n' # not an opcode; see INT docs in pickletools.py

FALSE = b'I00\n' # not an opcode; see INT docs in pickletools.py

PROTO = b'\x80' # identify pickle protocol

NEWOBJ = b'\x81' # build object by applying cls.__new__ to argtuple

EXT1 = b'\x82' # push object from extension registry; 1-byte index

EXT2 = b'\x83' # ditto, but 2-byte index

EXT4 = b'\x84' # ditto, but 4-byte index

TUPLE1 = b'\x85' # build 1-tuple from stack top

TUPLE2 = b'\x86' # build 2-tuple from two topmost stack items

TUPLE3 = b'\x87' # build 3-tuple from three topmost stack items

NEWTRUE = b'\x88' # push True

NEWFALSE = b'\x89' # push False

LONG1 = b'\x8a' # push long from < 256 bytes

LONG4 = b'\x8b' # push really big long

_tuplesize2code = [EMPTY_TUPLE, TUPLE1, TUPLE2, TUPLE3]

# Protocol 3 (Python 3.x)

BINBYTES = b'B' # push bytes; counted binary string argument

SHORT_BINBYTES = b'C' # " " ; " " " " < 256 bytes

SHORT_BINUNICODE = b'\x8c' # push short string; UTF-8 length < 256 bytes

BINUNICODE8 = b'\x8d' # push very long string

BINBYTES8 = b'\x8e' # push very long bytes string

EMPTY_SET = b'\x8f' # push empty set on the stack

ADDITEMS = b'\x90' # modify set by adding topmost stack items

FROZENSET = b'\x91' # build frozenset from topmost stack items

NEWOBJ_EX = b'\x92' # like NEWOBJ but work with keyword only arguments

STACK_GLOBAL = b'\x93' # same as GLOBAL but using names on the stacks

MEMOIZE = b'\x94' # store top of the stack in memo

FRAME = b'\x95' # indicate the beginning of a new frame

# Protocol 5

BYTEARRAY8 = b'\x96' # push bytearray

NEXT_BUFFER = b'\x97' # push next out-of-band buffer

READONLY_BUFFER = b'\x98' # make top of stack readonly

__all__.extend([x for x in dir() if re.match("[A-Z][A-Z0-9_]+$", x)])

class _Framer:

_FRAME_SIZE_TARGET = 64 * 1024

def __init__(self, file_write):

self.file_write = file_write

self.current_frame = None

def start_framing(self):

self.current_frame = io.BytesIO()

def end_framing(self):

if self.current_frame and self.current_frame.tell() > 0:

self.commit_frame(force=True)

self.current_frame = None

def commit_frame(self, force=False):

if self.current_frame:

f = self.current_frame

if f.tell() >= self._FRAME_SIZE_TARGET or force:

data = f.getbuffer()

write = self.file_write

if len(data) >= self._FRAME_SIZE_MIN:

# Issue a single call to the write method of the underlying

# file object for the frame opcode with the size of the

# frame. The concatenation is expected to be less expensive

# than issuing an additional call to write.

write(FRAME + pack("<Q", len(data)))

# Issue a separate call to write to append the frame

# contents without concatenation to the above to avoid a

# memory copy.

write(data)

# Start the new frame with a new io.BytesIO instance so that

# the file object can have delayed access to the previous frame

# contents via an unreleased memoryview of the previous

# io.BytesIO instance.

self.current_frame = io.BytesIO()

if self.current_frame:

return self.current_frame.write(data)

return self.file_write(data)

def write_large_bytes(self, header, payload):

write = self.file_write

if self.current_frame:

# Terminate the current frame and flush it to the file.

self.commit_frame(force=True)

# Perform direct write of the header and payload of the large binary

# object. Be careful not to concatenate the header and the payload

# prior to calling 'write' as we do not want to allocate a large

# temporary bytes object.

# We intentionally do not insert a protocol 4 frame opcode to make

# it possible to optimize file.read calls in the loader.

write(header)

write(payload)

class _Unframer:

def __init__(self, file_read, file_readline, file_tell=None):

self.file_read = file_read

self.file_readline = file_readline

self.current_frame = None

def readinto(self, buf):

if self.current_frame:

n = self.current_frame.readinto(buf)

if n == 0 and len(buf) != 0:

self.current_frame = None

n = len(buf)

buf[:] = self.file_read(n)

return n

if n < len(buf):

raise UnpicklingError(

"pickle exhausted before end of frame")

return n

else:

n = len(buf)

buf[:] = self.file_read(n)

return n

if self.current_frame:

data = self.current_frame.read(n)

if not data and n != 0:

self.current_frame = None

return self.file_read(n)

if len(data) < n:

raise UnpicklingError(

"pickle exhausted before end of frame")

return data

else:

return self.file_read(n)

def readline(self):

if self.current_frame:

data = self.current_frame.readline()

if not data:

self.current_frame = None

return self.file_readline()

raise UnpicklingError(

"pickle exhausted before end of frame")

return data

def load_frame(self, frame_size):

if self.current_frame and self.current_frame.read() != b'':

raise UnpicklingError(

"beginning of a new frame before end of current frame")

self.current_frame = io.BytesIO(self.file_read(frame_size))

# Tools used for pickling.

def _getattribute(obj, name):

for subpath in name.split('.'):

if subpath == '<locals>':

raise AttributeError("Can't get local attribute {!r} on {!r}"

.format(name, obj))

try:

obj = getattr(obj, subpath)

except AttributeError:

raise AttributeError("Can't get attribute {!r} on {!r}"

"""Find the module an object belong to."""

module_name = getattr(obj, '__module__', None)

if module_name is not None:

return module_name

# Protect the iteration by using a list copy of sys.modules against dynamic

# modules that trigger imports of other modules upon calls to getattr.

if (module_name == '__main__'

or module_name == '__mp_main__' # bpo-42406

or module is None):

continue

try:

return module_name

except AttributeError:

pass

return '__main__'

def encode_long(x):

r"""Encode a long to a two's complement little-endian binary string.

Note that 0 is a special case, returning an empty string, to save a

byte in the LONG1 pickling context.

>>> encode_long(0)

b''

>>> encode_long(255)

b'\xff\x00'

>>> encode_long(32767)

b'\xff\x7f'

>>> encode_long(-256)

b'\x00\xff'

>>> encode_long(-32768)

b'\x00\x80'

>>> encode_long(-128)

b'\x80'

>>> encode_long(127)

b'\x7f'

>>>

"""

if x == 0:

return b''

nbytes = (x.bit_length() >> 3) + 1

result = x.to_bytes(nbytes, byteorder='little', signed=True)

if x < 0 and nbytes > 1:

if result[-1] == 0xff and (result[-2] & 0x80) != 0:

result = result[:-1]

return result

def decode_long(data):

r"""Decode a long from a two's complement little-endian binary string.

>>> decode_long(b'')

0

>>> decode_long(b"\xff\x00")

255

>>> decode_long(b"\xff\x7f")

32767

>>> decode_long(b"\x00\xff")

-256

>>> decode_long(b"\x00\x80")

-32768

>>> decode_long(b"\x80")

-128

>>> decode_long(b"\x7f")

127

"""

return int.from_bytes(data, byteorder='little', signed=True)

# Pickling machinery

def __init__(self, file, protocol=None, *, fix_imports=True,

buffer_callback=None):

"""This takes a binary file for writing a pickle data stream.

given protocol; supported protocols are 0, 1, 2, 3, 4 and 5.

The default protocol is 4. It was introduced in Python 3.4, and

is incompatible with previous versions.

protocol version supported. The higher the protocol used, the

more recent the version of Python needed to read the pickle

produced.

The *file* argument must have a write() method that accepts a

single bytes argument. It can thus be a file object opened for

If *fix_imports* is True and *protocol* is less than 3, pickle

will try to map the new Python 3 names to the old module names

used in Python 2, so that the pickle data stream is readable

with Python 2.

If *buffer_callback* is None (the default), buffer views are

serialized into *file* as part of the pickle stream.

If *buffer_callback* is not None, then it can be called any number

of times with a buffer view. If the callback returns a false value

(such as None), the given buffer is out-of-band; otherwise the

buffer is serialized in-band, i.e. inside the pickle stream.

It is an error if *buffer_callback* is not None and *protocol*

is None or smaller than 5.

protocol = HIGHEST_PROTOCOL

elif not 0 <= protocol <= HIGHEST_PROTOCOL:

raise ValueError("pickle protocol must be <= %d" % HIGHEST_PROTOCOL)

if buffer_callback is not None and protocol < 5:

raise ValueError("buffer_callback needs protocol >= 5")

self._buffer_callback = buffer_callback

except AttributeError:

raise TypeError("file must have a 'write' attribute")

self.framer = _Framer(self._file_write)

self.write = self.framer.write

self.proto = int(protocol)

self.bin = protocol >= 1

"""Clears the pickler's "memo".

The memo is the data structure that remembers which objects the

pickler has already seen, so that shared or recursive objects

are pickled by reference and not by value. This method is

useful when re-using picklers.

self.memo.clear()

# Check whether Pickler was initialized correctly. This is

# only needed to mimic the behavior of _pickle.Pickler.dump().

raise PicklingError("Pickler.__init__() was not called by "

"%s.__init__()" % (self.__class__.__name__,))

def memoize(self, obj):

"""Store an object in the memo."""

# The Pickler memo is a dictionary mapping object ids to 2-tuples

# that contain the Unpickler memo key and the object being memoized.

# The memo key is written to the pickle and will become

# Pickler memo so that transient objects are kept alive during

# pickling.

# The use of the Unpickler memo length as the memo key is just a

# convention. The only requirement is that the memo values be unique.

# But there appears no advantage to any other scheme, and this

if self.fast:

return

idx = len(self.memo)

self.write(self.put(idx))

self.memo[id(obj)] = idx, obj

def put(self, idx):

if self.proto >= 4:

return MEMOIZE

elif self.bin:

if idx < 256:

return BINPUT + pack("<B", idx)

return LONG_BINPUT + pack("<I", idx)

else:

return PUT + repr(idx).encode("ascii") + b'\n'

if self.bin:

if i < 256: