Hello community,
here is the log from the commit of package python-cloudpickle for openSUSE:Factory checked in at 2019-04-02 09:21:32
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Comparing /work/SRC/openSUSE:Factory/python-cloudpickle (Old)
and /work/SRC/openSUSE:Factory/.python-cloudpickle.new.25356 (New)
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Package is "python-cloudpickle"
Tue Apr 2 09:21:32 2019 rev:5 rq:689381 version:0.8.1
Changes:
--------
--- /work/SRC/openSUSE:Factory/python-cloudpickle/python-cloudpickle.changes 2019-02-06 15:48:19.751226078 +0100
+++ /work/SRC/openSUSE:Factory/.python-cloudpickle.new.25356/python-cloudpickle.changes 2019-04-02 09:21:46.028681327 +0200
@@ -1,0 +2,13 @@
+Thu Mar 28 14:20:54 UTC 2019 - Tomáš Chvátal
+
+- Update to 0.8.1:
+ * Fix a bug (already present before 0.5.3 and re-introduced in 0.8.0) affecting relative import instructions inside depickled functions (issue #254)
+
+-------------------------------------------------------------------
+Thu Mar 7 13:00:07 UTC 2019 - Tomáš Chvátal
+
+- Update to 0.8.0:
+ * Add support for pickling interactively defined dataclasses. (issue #245)
+ * Global variables referenced by functions pickled by cloudpickle are now unpickled in a new and isolated namespace scoped by the CloudPickler instance. This restores the (previously untested) behavior of cloudpickle prior to changes done in 0.5.4 for functions defined in the __main__ module, and 0.6.0/1 for other dynamic functions.
+
+-------------------------------------------------------------------
Old:
----
cloudpickle-0.7.0.tar.gz
New:
----
cloudpickle-0.8.1.tar.gz
++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++
Other differences:
------------------
++++++ python-cloudpickle.spec ++++++
--- /var/tmp/diff_new_pack.87w4wu/_old 2019-04-02 09:21:47.204682428 +0200
+++ /var/tmp/diff_new_pack.87w4wu/_new 2019-04-02 09:21:47.208682432 +0200
@@ -18,7 +18,7 @@
%{?!python_module:%define python_module() python-%{**} python3-%{**}}
Name: python-cloudpickle
-Version: 0.7.0
+Version: 0.8.1
Release: 0
Summary: Extended pickling support for Python objects
License: BSD-3-Clause
@@ -27,11 +27,13 @@
Source: https://files.pythonhosted.org/packages/source/c/cloudpickle/cloudpickle-%{version}.tar.gz
BuildRequires: %{python_module setuptools}
BuildRequires: fdupes
+BuildRequires: python-futures
BuildRequires: python-rpm-macros
BuildArch: noarch
BuildRequires: %{python_module curses}
BuildRequires: %{python_module mock}
BuildRequires: %{python_module numpy >= 1.8.2}
+BuildRequires: %{python_module psutil}
BuildRequires: %{python_module pytest-cov}
BuildRequires: %{python_module pytest}
BuildRequires: %{python_module scipy}
@@ -65,7 +67,6 @@
%python_expand %fdupes %{buildroot}%{$python_sitelib}
%check
-# Tests require very specific paths and py.test arguments
export PYTHONPATH='.:tests'
%python_expand py.test-%{$python_bin_suffix} -s
++++++ cloudpickle-0.7.0.tar.gz -> cloudpickle-0.8.1.tar.gz ++++++
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/PKG-INFO new/cloudpickle-0.8.1/PKG-INFO
--- old/cloudpickle-0.7.0/PKG-INFO 2019-01-23 17:36:06.000000000 +0100
+++ new/cloudpickle-0.8.1/PKG-INFO 2019-03-25 10:07:23.000000000 +0100
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cloudpickle
-Version: 0.7.0
+Version: 0.8.1
Summary: Extended pickling support for Python objects
Home-page: https://github.com/cloudpipe/cloudpickle
Author: Cloudpipe
@@ -15,16 +15,20 @@
`cloudpickle` makes it possible to serialize Python constructs not supported
by the default `pickle` module from the Python standard library.
- `cloudpickle` is especially useful for cluster computing where Python
- expressions are shipped over the network to execute on remote hosts, possibly
- close to the data.
-
- Among other things, `cloudpickle` supports pickling for lambda expressions,
- functions and classes defined interactively in the `__main__` module.
-
- `cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default: it is meant to
- send objects between processes running the same version of Python. It is
- discouraged to use `cloudpickle` for long-term storage.
+ `cloudpickle` is especially useful for **cluster computing** where Python
+ code is shipped over the network to execute on remote hosts, possibly close
+ to the data.
+
+ Among other things, `cloudpickle` supports pickling for **lambda functions**
+ along with **functions and classes defined interactively** in the
+ `__main__` module (for instance in a script, a shell or a Jupyter notebook).
+
+ **`cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default**: it is meant to
+ send objects between processes running the **same version of Python**.
+
+ Using `cloudpickle` for **long-term object storage is not supported and
+ discouraged.**
+
Installation
------------
@@ -75,7 +79,7 @@
or alternatively for a specific environment:
- tox -e py27
+ tox -e py37
- With `py.test` to only run the tests for your current version of
@@ -108,9 +112,9 @@
Classifier: Operating System :: Microsoft :: Windows
Classifier: Operating System :: MacOS :: MacOS X
Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Topic :: Software Development :: Libraries :: Python Modules
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/README.md new/cloudpickle-0.8.1/README.md
--- old/cloudpickle-0.7.0/README.md 2017-11-15 17:06:41.000000000 +0100
+++ new/cloudpickle-0.8.1/README.md 2019-02-19 14:29:43.000000000 +0100
@@ -7,16 +7,20 @@
`cloudpickle` makes it possible to serialize Python constructs not supported
by the default `pickle` module from the Python standard library.
-`cloudpickle` is especially useful for cluster computing where Python
-expressions are shipped over the network to execute on remote hosts, possibly
-close to the data.
-
-Among other things, `cloudpickle` supports pickling for lambda expressions,
-functions and classes defined interactively in the `__main__` module.
-
-`cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default: it is meant to
-send objects between processes running the same version of Python. It is
-discouraged to use `cloudpickle` for long-term storage.
+`cloudpickle` is especially useful for **cluster computing** where Python
+code is shipped over the network to execute on remote hosts, possibly close
+to the data.
+
+Among other things, `cloudpickle` supports pickling for **lambda functions**
+along with **functions and classes defined interactively** in the
+`__main__` module (for instance in a script, a shell or a Jupyter notebook).
+
+**`cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default**: it is meant to
+send objects between processes running the **same version of Python**.
+
+Using `cloudpickle` for **long-term object storage is not supported and
+discouraged.**
+
Installation
------------
@@ -67,7 +71,7 @@
or alternatively for a specific environment:
- tox -e py27
+ tox -e py37
- With `py.test` to only run the tests for your current version of
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/cloudpickle/__init__.py new/cloudpickle-0.8.1/cloudpickle/__init__.py
--- old/cloudpickle-0.7.0/cloudpickle/__init__.py 2019-01-23 17:34:22.000000000 +0100
+++ new/cloudpickle-0.8.1/cloudpickle/__init__.py 2019-03-25 10:07:01.000000000 +0100
@@ -2,4 +2,4 @@
from cloudpickle.cloudpickle import *
-__version__ = '0.7.0'
+__version__ = '0.8.1'
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/cloudpickle/cloudpickle.py new/cloudpickle-0.8.1/cloudpickle/cloudpickle.py
--- old/cloudpickle-0.7.0/cloudpickle/cloudpickle.py 2019-01-23 17:32:14.000000000 +0100
+++ new/cloudpickle-0.8.1/cloudpickle/cloudpickle.py 2019-03-25 09:45:31.000000000 +0100
@@ -63,7 +63,7 @@
DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL
-if sys.version < '3':
+if sys.version_info[0] < 3: # pragma: no branch
from pickle import Pickler
try:
from cStringIO import StringIO
@@ -79,22 +79,6 @@
PY3 = True
-# Container for the global namespace to ensure consistent unpickling of
-# functions defined in dynamic modules (modules not registed in sys.modules).
-_dynamic_modules_globals = weakref.WeakValueDictionary()
-
-
-class _DynamicModuleFuncGlobals(dict):
- """Global variables referenced by a function defined in a dynamic module
-
- To avoid leaking references we store such context in a WeakValueDictionary
- instance. However instances of python builtin types such as dict cannot
- be used directly as values in such a construct, hence the need for a
- derived class.
- """
- pass
-
-
def _make_cell_set_template_code():
"""Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF
@@ -128,7 +112,7 @@
# NOTE: we are marking the cell variable as a free variable intentionally
# so that we simulate an inner function instead of the outer function. This
# is what gives us the ``nonlocal`` behavior in a Python 2 compatible way.
- if not PY3:
+ if not PY3: # pragma: no branch
return types.CodeType(
co.co_argcount,
co.co_nlocals,
@@ -229,14 +213,14 @@
}
-if sys.version_info < (3, 4):
+if sys.version_info < (3, 4): # pragma: no branch
def _walk_global_ops(code):
"""
Yield (opcode, argument number) tuples for all
global-referencing instructions in *code*.
"""
code = getattr(code, 'co_code', b'')
- if not PY3:
+ if not PY3: # pragma: no branch
code = map(ord, code)
n = len(code)
@@ -293,7 +277,7 @@
dispatch[memoryview] = save_memoryview
- if not PY3:
+ if not PY3: # pragma: no branch
def save_buffer(self, obj):
self.save(str(obj))
@@ -315,7 +299,7 @@
"""
Save a code object
"""
- if PY3:
+ if PY3: # pragma: no branch
args = (
obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize,
obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames,
@@ -393,7 +377,7 @@
# So we pickle them here using save_reduce; have to do it differently
# for different python versions.
if not hasattr(obj, '__code__'):
- if PY3:
+ if PY3: # pragma: no branch
rv = obj.__reduce_ex__(self.proto)
else:
if hasattr(obj, '__self__'):
@@ -670,17 +654,26 @@
# save the dict
dct = func.__dict__
- base_globals = self.globals_ref.get(id(func.__globals__), None)
- if base_globals is None:
- # For functions defined in a well behaved module use
- # vars(func.__module__) for base_globals. This is necessary to
- # share the global variables across multiple pickled functions from
- # this module.
- if func.__module__ is not None:
- base_globals = func.__module__
- else:
- base_globals = {}
- self.globals_ref[id(func.__globals__)] = base_globals
+ # base_globals represents the future global namespace of func at
+ # unpickling time. Looking it up and storing it in globals_ref allow
+ # functions sharing the same globals at pickling time to also
+ # share them once unpickled, at one condition: since globals_ref is
+ # an attribute of a Cloudpickler instance, and that a new CloudPickler is
+ # created each time pickle.dump or pickle.dumps is called, functions
+ # also need to be saved within the same invokation of
+ # cloudpickle.dump/cloudpickle.dumps (for example: cloudpickle.dumps([f1, f2])). There
+ # is no such limitation when using Cloudpickler.dump, as long as the
+ # multiple invokations are bound to the same Cloudpickler.
+ base_globals = self.globals_ref.setdefault(id(func.__globals__), {})
+
+ if base_globals == {}:
+ # Add module attributes used to resolve relative imports
+ # instructions inside func.
+ for k in ["__package__", "__name__", "__path__", "__file__"]:
+ # Some built-in functions/methods such as object.__new__ have
+ # their __globals__ set to None in PyPy
+ if func.__globals__ is not None and k in func.__globals__:
+ base_globals[k] = func.__globals__[k]
return (code, f_globals, defaults, closure, dct, base_globals)
@@ -730,7 +723,7 @@
if obj.__self__ is None:
self.save_reduce(getattr, (obj.im_class, obj.__name__))
else:
- if PY3:
+ if PY3: # pragma: no branch
self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj)
else:
self.save_reduce(types.MethodType, (obj.__func__, obj.__self__, obj.__self__.__class__),
@@ -783,7 +776,7 @@
save(stuff)
write(pickle.BUILD)
- if not PY3:
+ if not PY3: # pragma: no branch
dispatch[types.InstanceType] = save_inst
def save_property(self, obj):
@@ -883,7 +876,7 @@
try: # Python 2
dispatch[file] = save_file
- except NameError: # Python 3
+ except NameError: # Python 3 # pragma: no branch
dispatch[io.TextIOWrapper] = save_file
dispatch[type(Ellipsis)] = save_ellipsis
@@ -904,6 +897,12 @@
dispatch[logging.RootLogger] = save_root_logger
+ if hasattr(types, "MappingProxyType"): # pragma: no branch
+ def save_mappingproxy(self, obj):
+ self.save_reduce(types.MappingProxyType, (dict(obj),), obj=obj)
+
+ dispatch[types.MappingProxyType] = save_mappingproxy
+
"""Special functions for Add-on libraries"""
def inject_addons(self):
"""Plug in system. Register additional pickling functions if modules already loaded"""
@@ -989,43 +988,6 @@
return obj
-def _get_module_builtins():
- return pickle.__builtins__
-
-
-def print_exec(stream):
- ei = sys.exc_info()
- traceback.print_exception(ei[0], ei[1], ei[2], None, stream)
-
-
-def _modules_to_main(modList):
- """Force every module in modList to be placed into main"""
- if not modList:
- return
-
- main = sys.modules['__main__']
- for modname in modList:
- if type(modname) is str:
- try:
- mod = __import__(modname)
- except Exception:
- sys.stderr.write('warning: could not import %s\n. '
- 'Your function may unexpectedly error due to this import failing;'
- 'A version mismatch is likely. Specific error was:\n' % modname)
- print_exec(sys.stderr)
- else:
- setattr(main, mod.__name__, mod)
-
-
-# object generators:
-def _genpartial(func, args, kwds):
- if not args:
- args = ()
- if not kwds:
- kwds = {}
- return partial(func, *args, **kwds)
-
-
def _gen_ellipsis():
return Ellipsis
@@ -1090,10 +1052,16 @@
else:
raise ValueError('Unexpected _fill_value arguments: %r' % (args,))
- # Only set global variables that do not exist.
- for k, v in state['globals'].items():
- if k not in func.__globals__:
- func.__globals__[k] = v
+ # - At pickling time, any dynamic global variable used by func is
+ # serialized by value (in state['globals']).
+ # - At unpickling time, func's __globals__ attribute is initialized by
+ # first retrieving an empty isolated namespace that will be shared
+ # with other functions pickled from the same original module
+ # by the same CloudPickler instance and then updated with the
+ # content of state['globals'] to populate the shared isolated
+ # namespace with all the global variables that are specifically
+ # referenced for this function.
+ func.__globals__.update(state['globals'])
func.__defaults__ = state['defaults']
func.__dict__ = state['dict']
@@ -1131,21 +1099,11 @@
code and the correct number of cells in func_closure. All other
func attributes (e.g. func_globals) are empty.
"""
- if base_globals is None:
+ # This is backward-compatibility code: for cloudpickle versions between
+ # 0.5.4 and 0.7, base_globals could be a string or None. base_globals
+ # should now always be a dictionary.
+ if base_globals is None or isinstance(base_globals, str):
base_globals = {}
- elif isinstance(base_globals, str):
- base_globals_name = base_globals
- try:
- # First try to reuse the globals from the module containing the
- # function. If it is not possible to retrieve it, fallback to an
- # empty dictionary.
- base_globals = vars(importlib.import_module(base_globals))
- except ImportError:
- base_globals = _dynamic_modules_globals.get(
- base_globals_name, None)
- if base_globals is None:
- base_globals = _DynamicModuleFuncGlobals()
- _dynamic_modules_globals[base_globals_name] = base_globals
base_globals['__builtins__'] = __builtins__
@@ -1202,18 +1160,9 @@
return False
-"""Constructors for 3rd party libraries
-Note: These can never be renamed due to client compatibility issues"""
-
-
-def _getobject(modname, attribute):
- mod = __import__(modname, fromlist=[attribute])
- return mod.__dict__[attribute]
-
-
""" Use copy_reg to extend global pickle definitions """
-if sys.version_info < (3, 4):
+if sys.version_info < (3, 4): # pragma: no branch
method_descriptor = type(str.upper)
def _reduce_method_descriptor(obj):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/cloudpickle.egg-info/PKG-INFO new/cloudpickle-0.8.1/cloudpickle.egg-info/PKG-INFO
--- old/cloudpickle-0.7.0/cloudpickle.egg-info/PKG-INFO 2019-01-23 17:36:05.000000000 +0100
+++ new/cloudpickle-0.8.1/cloudpickle.egg-info/PKG-INFO 2019-03-25 10:07:23.000000000 +0100
@@ -1,6 +1,6 @@
Metadata-Version: 1.1
Name: cloudpickle
-Version: 0.7.0
+Version: 0.8.1
Summary: Extended pickling support for Python objects
Home-page: https://github.com/cloudpipe/cloudpickle
Author: Cloudpipe
@@ -15,16 +15,20 @@
`cloudpickle` makes it possible to serialize Python constructs not supported
by the default `pickle` module from the Python standard library.
- `cloudpickle` is especially useful for cluster computing where Python
- expressions are shipped over the network to execute on remote hosts, possibly
- close to the data.
-
- Among other things, `cloudpickle` supports pickling for lambda expressions,
- functions and classes defined interactively in the `__main__` module.
-
- `cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default: it is meant to
- send objects between processes running the same version of Python. It is
- discouraged to use `cloudpickle` for long-term storage.
+ `cloudpickle` is especially useful for **cluster computing** where Python
+ code is shipped over the network to execute on remote hosts, possibly close
+ to the data.
+
+ Among other things, `cloudpickle` supports pickling for **lambda functions**
+ along with **functions and classes defined interactively** in the
+ `__main__` module (for instance in a script, a shell or a Jupyter notebook).
+
+ **`cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default**: it is meant to
+ send objects between processes running the **same version of Python**.
+
+ Using `cloudpickle` for **long-term object storage is not supported and
+ discouraged.**
+
Installation
------------
@@ -75,7 +79,7 @@
or alternatively for a specific environment:
- tox -e py27
+ tox -e py37
- With `py.test` to only run the tests for your current version of
@@ -108,9 +112,9 @@
Classifier: Operating System :: Microsoft :: Windows
Classifier: Operating System :: MacOS :: MacOS X
Classifier: Programming Language :: Python :: 2.7
-Classifier: Programming Language :: Python :: 3.4
Classifier: Programming Language :: Python :: 3.5
Classifier: Programming Language :: Python :: 3.6
+Classifier: Programming Language :: Python :: 3.7
Classifier: Programming Language :: Python :: Implementation :: CPython
Classifier: Programming Language :: Python :: Implementation :: PyPy
Classifier: Topic :: Software Development :: Libraries :: Python Modules
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/cloudpickle.egg-info/SOURCES.txt new/cloudpickle-0.8.1/cloudpickle.egg-info/SOURCES.txt
--- old/cloudpickle-0.7.0/cloudpickle.egg-info/SOURCES.txt 2019-01-23 17:36:05.000000000 +0100
+++ new/cloudpickle-0.8.1/cloudpickle.egg-info/SOURCES.txt 2019-03-25 10:07:23.000000000 +0100
@@ -12,4 +12,6 @@
tests/__init__.py
tests/cloudpickle_file_test.py
tests/cloudpickle_test.py
-tests/testutils.py
\ No newline at end of file
+tests/testutils.py
+tests/mypkg/__init__.py
+tests/mypkg/mod.py
\ No newline at end of file
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/setup.py new/cloudpickle-0.8.1/setup.py
--- old/cloudpickle-0.7.0/setup.py 2019-01-23 17:29:10.000000000 +0100
+++ new/cloudpickle-0.8.1/setup.py 2019-01-31 14:05:30.000000000 +0100
@@ -39,9 +39,9 @@
'Operating System :: Microsoft :: Windows',
'Operating System :: MacOS :: MacOS X',
'Programming Language :: Python :: 2.7',
- 'Programming Language :: Python :: 3.4',
'Programming Language :: Python :: 3.5',
'Programming Language :: Python :: 3.6',
+ 'Programming Language :: Python :: 3.7',
'Programming Language :: Python :: Implementation :: CPython',
'Programming Language :: Python :: Implementation :: PyPy',
'Topic :: Software Development :: Libraries :: Python Modules',
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/tests/cloudpickle_test.py new/cloudpickle-0.8.1/tests/cloudpickle_test.py
--- old/cloudpickle-0.7.0/tests/cloudpickle_test.py 2019-01-23 10:51:18.000000000 +0100
+++ new/cloudpickle-0.8.1/tests/cloudpickle_test.py 2019-03-25 09:45:31.000000000 +0100
@@ -451,57 +451,6 @@
mod1, mod2 = pickle_depickle([mod, mod])
self.assertEqual(id(mod1), id(mod2))
- def test_dynamic_modules_globals(self):
- # _dynamic_modules_globals is a WeakValueDictionary, so if a value
- # in this dict (containing a set of global variables from a dynamic
- # module created in the parent process) has no other reference than in
- # this dict in the child process, it will be garbage collected.
-
- # We first create a module
- mod = types.ModuleType('mod')
- code = '''
- x = 1
- def func():
- return
- '''
- exec(textwrap.dedent(code), mod.__dict__)
-
- pickled_module_path = os.path.join(self.tmpdir, 'mod_f.pkl')
- child_process_script = '''
- import pickle
- from cloudpickle.cloudpickle import _dynamic_modules_globals
- import gc
- with open("{pickled_module_path}", 'rb') as f:
- func = pickle.load(f)
-
- # A dictionnary storing the globals of the newly unpickled function
- # should have been created
- assert list(_dynamic_modules_globals.keys()) == ['mod']
-
- # func.__globals__ is the only non-weak reference to
- # _dynamic_modules_globals['mod']. By deleting func, we delete also
- # _dynamic_modules_globals['mod']
- del func
- gc.collect()
-
- # There is no reference to the globals of func since func has been
- # deleted and _dynamic_modules_globals is a WeakValueDictionary,
- # so _dynamic_modules_globals should now be empty
- assert list(_dynamic_modules_globals.keys()) == []
- '''
-
- child_process_script = child_process_script.format(
- pickled_module_path=_escape(pickled_module_path))
-
- try:
- with open(pickled_module_path, 'wb') as f:
- cloudpickle.dump(mod.func, f, protocol=self.protocol)
-
- assert_run_python_script(textwrap.dedent(child_process_script))
-
- finally:
- os.unlink(pickled_module_path)
-
def test_module_locals_behavior(self):
# Makes sure that a local function defined in another module is
# correctly serialized. This notably checks that the globals are
@@ -1029,6 +978,12 @@
def f4(x):
return foo.method(x)
+ def f5(x):
+ # Recursive call to a dynamically defined function.
+ if x <= 0:
+ return f4(x)
+ return f5(x - 1) + 1
+
cloned = subprocess_pickle_echo(lambda x: x**2, protocol={protocol})
assert cloned(3) == 9
@@ -1052,6 +1007,9 @@
cloned = subprocess_pickle_echo(f4, protocol={protocol})
assert cloned(2) == f4(2)
+
+ cloned = subprocess_pickle_echo(f5, protocol={protocol})
+ assert cloned(7) == f5(7) == 7
""".format(protocol=self.protocol)
assert_run_python_script(textwrap.dedent(code))
@@ -1074,20 +1032,42 @@
def f1():
return VARIABLE
- cloned_f0 = {clone_func}(f0, protocol={protocol})
- cloned_f1 = {clone_func}(f1, protocol={protocol})
+ assert f0.__globals__ is f1.__globals__
+
+ # pickle f0 and f1 inside the same pickle_string
+ cloned_f0, cloned_f1 = {clone_func}([f0, f1], protocol={protocol})
+
+ # cloned_f0 and cloned_f1 now share a global namespace that is isolated
+ # from any previously existing namespace
+ assert cloned_f0.__globals__ is cloned_f1.__globals__
+ assert cloned_f0.__globals__ is not f0.__globals__
+
+ # pickle f1 another time, but in a new pickle string
pickled_f1 = dumps(f1, protocol={protocol})
- # Change the value of the global variable
+ # Change the value of the global variable in f0's new global namespace
cloned_f0()
- # Ensure that the global variable is the same for another function
- result_f1 = cloned_f1()
- assert result_f1 == "changed_by_f0", result_f1
-
- # Ensure that unpickling the global variable does not change its value
- result_pickled_f1 = loads(pickled_f1)()
- assert result_pickled_f1 == "changed_by_f0", result_pickled_f1
+ # thanks to cloudpickle isolation, depickling and calling f0 and f1
+ # should not affect the globals of already existing modules
+ assert VARIABLE == "default_value", VARIABLE
+
+ # Ensure that cloned_f1 and cloned_f0 share the same globals, as f1 and
+ # f0 shared the same globals at pickling time, and cloned_f1 was
+ # depickled from the same pickle string as cloned_f0
+ shared_global_var = cloned_f1()
+ assert shared_global_var == "changed_by_f0", shared_global_var
+
+ # f1 is unpickled another time, but because it comes from another
+ # pickle string than pickled_f1 and pickled_f0, it will not share the
+ # same globals as the latter two.
+ new_cloned_f1 = loads(pickled_f1)
+ assert new_cloned_f1.__globals__ is not cloned_f1.__globals__
+ assert new_cloned_f1.__globals__ is not f1.__globals__
+
+ # get the value of new_cloned_f1's VARIABLE
+ new_global_var = new_cloned_f1()
+ assert new_global_var == "default_value", new_global_var
"""
for clone_func in ['local_clone', 'subprocess_pickle_echo']:
code = code_template.format(protocol=self.protocol,
@@ -1106,116 +1086,154 @@
def f1():
return _TEST_GLOBAL_VARIABLE
- cloned_f0 = cloudpickle.loads(cloudpickle.dumps(
- f0, protocol=self.protocol))
- cloned_f1 = cloudpickle.loads(cloudpickle.dumps(
- f1, protocol=self.protocol))
+ # pickle f0 and f1 inside the same pickle_string
+ cloned_f0, cloned_f1 = pickle_depickle([f0, f1],
+ protocol=self.protocol)
+
+ # cloned_f0 and cloned_f1 now share a global namespace that is
+ # isolated from any previously existing namespace
+ assert cloned_f0.__globals__ is cloned_f1.__globals__
+ assert cloned_f0.__globals__ is not f0.__globals__
+
+ # pickle f1 another time, but in a new pickle string
pickled_f1 = cloudpickle.dumps(f1, protocol=self.protocol)
- # Change the value of the global variable
+ # Change the global variable's value in f0's new global namespace
cloned_f0()
- assert _TEST_GLOBAL_VARIABLE == "changed_by_f0"
- # Ensure that the global variable is the same for another function
- result_cloned_f1 = cloned_f1()
- assert result_cloned_f1 == "changed_by_f0", result_cloned_f1
- assert f1() == result_cloned_f1
-
- # Ensure that unpickling the global variable does not change its
- # value
- result_pickled_f1 = cloudpickle.loads(pickled_f1)()
- assert result_pickled_f1 == "changed_by_f0", result_pickled_f1
+ # depickling f0 and f1 should not affect the globals of already
+ # existing modules
+ assert _TEST_GLOBAL_VARIABLE == "default_value"
+
+ # Ensure that cloned_f1 and cloned_f0 share the same globals, as f1
+ # and f0 shared the same globals at pickling time, and cloned_f1
+ # was depickled from the same pickle string as cloned_f0
+ shared_global_var = cloned_f1()
+ assert shared_global_var == "changed_by_f0", shared_global_var
+
+ # f1 is unpickled another time, but because it comes from another
+ # pickle string than pickled_f1 and pickled_f0, it will not share
+ # the same globals as the latter two.
+ new_cloned_f1 = pickle.loads(pickled_f1)
+ assert new_cloned_f1.__globals__ is not cloned_f1.__globals__
+ assert new_cloned_f1.__globals__ is not f1.__globals__
+
+ # get the value of new_cloned_f1's VARIABLE
+ new_global_var = new_cloned_f1()
+ assert new_global_var == "default_value", new_global_var
finally:
_TEST_GLOBAL_VARIABLE = orig_value
- def test_function_from_dynamic_module_with_globals_modifications(self):
- # This test verifies that the global variable state of a function
- # defined in a dynamic module in a child process are not reset by
- # subsequent uplickling.
+ def test_interactive_remote_function_calls(self):
+ code = """if __name__ == "__main__":
+ from testutils import subprocess_worker
- # first, we create a dynamic module in the parent process
- mod = types.ModuleType('mod')
- code = '''
- GLOBAL_STATE = "initial value"
+ def interactive_function(x):
+ return x + 1
- def func_defined_in_dynamic_module(v=None):
- global GLOBAL_STATE
- if v is not None:
- GLOBAL_STATE = v
- return GLOBAL_STATE
- '''
- exec(textwrap.dedent(code), mod.__dict__)
+ with subprocess_worker(protocol={protocol}) as w:
- with_initial_globals_file = os.path.join(
- self.tmpdir, 'function_with_initial_globals.pkl')
- with_modified_globals_file = os.path.join(
- self.tmpdir, 'function_with_modified_globals.pkl')
+ assert w.run(interactive_function, 41) == 42
- try:
- # Simple sanity check on the function's output
- assert mod.func_defined_in_dynamic_module() == "initial value"
+ # Define a new function that will call an updated version of
+ # the previously called function:
- # The function of mod is pickled two times, with two different
- # values for the global variable GLOBAL_STATE.
- # Then we launch a child process that sequentially unpickles the
- # two functions. Those unpickle functions should share the same
- # global variables in the child process:
- # Once the first function gets unpickled, mod is created and
- # tracked in the child environment. This is state is preserved
- # when unpickling the second function whatever the global variable
- # GLOBAL_STATE's value at the time of pickling.
-
- with open(with_initial_globals_file, 'wb') as f:
- cloudpickle.dump(mod.func_defined_in_dynamic_module, f)
-
- # Change the mod's global variable
- mod.GLOBAL_STATE = 'changed value'
-
- # At this point, mod.func_defined_in_dynamic_module()
- # returns the updated value. Let's pickle it again.
- assert mod.func_defined_in_dynamic_module() == 'changed value'
- with open(with_modified_globals_file, 'wb') as f:
- cloudpickle.dump(mod.func_defined_in_dynamic_module, f,
- protocol=self.protocol)
+ def wrapper_func(x):
+ return interactive_function(x)
- child_process_code = """
- import pickle
+ def interactive_function(x):
+ return x - 1
+
+ # The change in the definition of interactive_function in the main
+ # module of the main process should be reflected transparently
+ # in the worker process: the worker process does not recall the
+ # previous definition of `interactive_function`:
- with open({with_initial_globals_file!r},'rb') as f:
- func_with_initial_globals = pickle.load(f)
+ assert w.run(wrapper_func, 41) == 40
+ """.format(protocol=self.protocol)
+ assert_run_python_script(code)
- # At this point, a module called 'mod' should exist in
- # _dynamic_modules_globals. Further function loading
- # will use the globals living in mod.
-
- assert func_with_initial_globals() == 'initial value'
-
- # Load a function with initial global variable that was
- # pickled after a change in the global variable
- with open({with_initial_globals_file!r},'rb') as f:
- func_with_modified_globals = pickle.load(f)
-
- # assert the this unpickling did not modify the value of
- # the local
- assert func_with_modified_globals() == 'initial value'
-
- # Update the value from the child process and check that
- # unpickling again does not reset our change.
- assert func_with_initial_globals('new value') == 'new value'
- assert func_with_modified_globals() == 'new value'
-
- with open({with_initial_globals_file!r},'rb') as f:
- func_with_initial_globals = pickle.load(f)
- assert func_with_initial_globals() == 'new value'
- assert func_with_modified_globals() == 'new value'
- """.format(
- with_initial_globals_file=_escape(with_initial_globals_file),
- with_modified_globals_file=_escape(with_modified_globals_file))
- assert_run_python_script(textwrap.dedent(child_process_code))
+ def test_interactive_remote_function_calls_no_side_effect(self):
+ code = """if __name__ == "__main__":
+ from testutils import subprocess_worker
+ import sys
- finally:
- os.unlink(with_initial_globals_file)
- os.unlink(with_modified_globals_file)
+ with subprocess_worker(protocol={protocol}) as w:
+
+ GLOBAL_VARIABLE = 0
+
+ class CustomClass(object):
+
+ def mutate_globals(self):
+ global GLOBAL_VARIABLE
+ GLOBAL_VARIABLE += 1
+ return GLOBAL_VARIABLE
+
+ custom_object = CustomClass()
+ assert w.run(custom_object.mutate_globals) == 1
+
+ # The caller global variable is unchanged in the main process.
+
+ assert GLOBAL_VARIABLE == 0
+
+ # Calling the same function again starts again from zero. The
+ # worker process is stateless: it has no memory of the past call:
+
+ assert w.run(custom_object.mutate_globals) == 1
+
+ # The symbols defined in the main process __main__ module are
+ # not set in the worker process main module to leave the worker
+ # as stateless as possible:
+
+ def is_in_main(name):
+ return hasattr(sys.modules["__main__"], name)
+
+ assert is_in_main("CustomClass")
+ assert not w.run(is_in_main, "CustomClass")
+
+ assert is_in_main("GLOBAL_VARIABLE")
+ assert not w.run(is_in_main, "GLOBAL_VARIABLE")
+
+ """.format(protocol=self.protocol)
+ assert_run_python_script(code)
+
+ @pytest.mark.skipif(platform.python_implementation() == 'PyPy',
+ reason="Skip PyPy because memory grows too much")
+ def test_interactive_remote_function_calls_no_memory_leak(self):
+ code = """if __name__ == "__main__":
+ from testutils import subprocess_worker
+ import struct
+
+ with subprocess_worker(protocol={protocol}) as w:
+
+ reference_size = w.memsize()
+ assert reference_size > 0
+
+
+ def make_big_closure(i):
+ # Generate a byte string of size 1MB
+ itemsize = len(struct.pack("l", 1))
+ data = struct.pack("l", i) * (int(1e6) // itemsize)
+ def process_data():
+ return len(data)
+ return process_data
+
+ for i in range(100):
+ func = make_big_closure(i)
+ result = w.run(func)
+ assert result == int(1e6), result
+
+ import gc
+ w.run(gc.collect)
+
+ # By this time the worker process has processed worth of 100MB of
+ # data passed in the closures its memory size should now have
+ # grown by more than a few MB.
+ growth = w.memsize() - reference_size
+ assert growth < 1e7, growth
+
+ """.format(protocol=self.protocol)
+ assert_run_python_script(code)
@pytest.mark.skipif(sys.version_info >= (3, 0),
reason="hardcoded pickle bytes for 2.7")
@@ -1335,6 +1353,45 @@
with pytest.raises(AttributeError):
obj.non_registered_attribute = 1
+ @unittest.skipIf(not hasattr(types, "MappingProxyType"),
+ "Old versions of Python do not have this type.")
+ def test_mappingproxy(self):
+ mp = types.MappingProxyType({"some_key": "some value"})
+ assert mp == pickle_depickle(mp, protocol=self.protocol)
+
+ def test_dataclass(self):
+ dataclasses = pytest.importorskip("dataclasses")
+
+ DataClass = dataclasses.make_dataclass('DataClass', [('x', int)])
+ data = DataClass(x=42)
+
+ pickle_depickle(DataClass, protocol=self.protocol)
+ assert data.x == pickle_depickle(data, protocol=self.protocol).x == 42
+
+ def test_relative_import_inside_function(self):
+ # Make sure relative imports inside round-tripped functions is not
+ # broken.This was a bug in cloudpickle versions <= 0.5.3 and was
+ # re-introduced in 0.8.0.
+
+ # Both functions living inside modules and packages are tested.
+ def f():
+ # module_function belongs to mypkg.mod1, which is a module
+ from .mypkg import module_function
+ return module_function()
+
+ def g():
+ # package_function belongs to mypkg, which is a package
+ from .mypkg import package_function
+ return package_function()
+
+ for func, source in zip([f, g], ["module", "package"]):
+ # Make sure relative imports are initially working
+ assert func() == "hello from a {}!".format(source)
+
+ # Make sure relative imports still work after round-tripping
+ cloned_func = pickle_depickle(func, protocol=self.protocol)
+ assert cloned_func() == "hello from a {}!".format(source)
+
class Protocol2CloudPickleTest(CloudPickleTest):
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/tests/mypkg/__init__.py new/cloudpickle-0.8.1/tests/mypkg/__init__.py
--- old/cloudpickle-0.7.0/tests/mypkg/__init__.py 1970-01-01 01:00:00.000000000 +0100
+++ new/cloudpickle-0.8.1/tests/mypkg/__init__.py 2019-03-25 09:45:31.000000000 +0100
@@ -0,0 +1,6 @@
+from .mod import module_function
+
+
+def package_function():
+ """Function living inside a package, not a simple module"""
+ return "hello from a package!"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/tests/mypkg/mod.py new/cloudpickle-0.8.1/tests/mypkg/mod.py
--- old/cloudpickle-0.7.0/tests/mypkg/mod.py 1970-01-01 01:00:00.000000000 +0100
+++ new/cloudpickle-0.8.1/tests/mypkg/mod.py 2019-03-25 09:45:31.000000000 +0100
@@ -0,0 +1,2 @@
+def module_function():
+ return "hello from a module!"
diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/tests/testutils.py new/cloudpickle-0.8.1/tests/testutils.py
--- old/cloudpickle-0.7.0/tests/testutils.py 2019-01-23 16:35:51.000000000 +0100
+++ new/cloudpickle-0.8.1/tests/testutils.py 2019-02-19 14:29:43.000000000 +0100
@@ -4,8 +4,12 @@
import tempfile
import base64
from subprocess import Popen, check_output, PIPE, STDOUT, CalledProcessError
-from cloudpickle import dumps
from pickle import loads
+from contextlib import contextmanager
+from concurrent.futures import ProcessPoolExecutor
+
+import psutil
+from cloudpickle import dumps
TIMEOUT = 60
try:
@@ -42,6 +46,20 @@
return cloudpickle_repo_folder, env
+def _pack(input_data, protocol=None):
+ pickled_input_data = dumps(input_data, protocol=protocol)
+ # Under Windows + Python 2.7, subprocess / communicate truncate the data
+ # on some specific bytes. To avoid this issue, let's use the pure ASCII
+ # Base32 encoding to encapsulate the pickle message sent to the child
+ # process.
+ return base64.b32encode(pickled_input_data)
+
+
+def _unpack(packed_data):
+ decoded_data = base64.b32decode(packed_data)
+ return loads(decoded_data)
+
+
def subprocess_pickle_echo(input_data, protocol=None, timeout=TIMEOUT):
"""Echo function with a child Python process
@@ -53,18 +71,12 @@
[1, 'a', None]
"""
- pickled_input_data = dumps(input_data, protocol=protocol)
- # Under Windows + Python 2.7, subprocess / communicate truncate the data
- # on some specific bytes. To avoid this issue, let's use the pure ASCII
- # Base32 encoding to encapsulate the pickle message sent to the child
- # process.
- pickled_b32 = base64.b32encode(pickled_input_data)
-
# run then pickle_echo(protocol=protocol) in __main__:
cmd = [sys.executable, __file__, "--protocol", str(protocol)]
cwd, env = _make_cwd_env()
proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=cwd, env=env,
bufsize=4096)
+ pickled_b32 = _pack(input_data, protocol=protocol)
try:
comm_kwargs = {}
if timeout_supported:
@@ -74,7 +86,7 @@
message = "Subprocess returned %d: " % proc.returncode
message += err.decode('utf-8')
raise RuntimeError(message)
- return loads(base64.b32decode(out))
+ return _unpack(out)
except TimeoutExpired:
proc.kill()
out, err = proc.communicate()
@@ -113,6 +125,56 @@
stream_out.close()
+def call_func(payload, protocol):
+ """Remote function call that uses cloudpickle to transport everthing"""
+ func, args, kwargs = loads(payload)
+ try:
+ result = func(*args, **kwargs)
+ except BaseException as e:
+ result = e
+ return dumps(result, protocol=protocol)
+
+
+class _Worker(object):
+ def __init__(self, protocol=None):
+ self.protocol = protocol
+ self.pool = ProcessPoolExecutor(max_workers=1)
+ self.pool.submit(id, 42).result() # start the worker process
+
+ def run(self, func, *args, **kwargs):
+ """Synchronous remote function call"""
+
+ input_payload = dumps((func, args, kwargs), protocol=self.protocol)
+ result_payload = self.pool.submit(
+ call_func, input_payload, self.protocol).result()
+ result = loads(result_payload)
+
+ if isinstance(result, BaseException):
+ raise result
+ return result
+
+ def memsize(self):
+ workers_pids = [p.pid if hasattr(p, "pid") else p
+ for p in list(self.pool._processes)]
+ num_workers = len(workers_pids)
+ if num_workers == 0:
+ return 0
+ elif num_workers > 1:
+ raise RuntimeError("Unexpected number of workers: %d"
+ % num_workers)
+ return psutil.Process(workers_pids[0]).memory_info().rss
+
+ def close(self):
+ self.pool.shutdown(wait=True)
+
+
+@contextmanager
+def subprocess_worker(protocol=None):
+ worker = _Worker(protocol=protocol)
+ yield worker
+ worker.close()
+
+
def assert_run_python_script(source_code, timeout=TIMEOUT):
"""Utility to help check pickleability of objects defined in __main__