commit python-cloudpickle for openSUSE:Factory

Hello community, here is the log from the commit of package python-cloudpickle for openSUSE:Factory checked in at 2019-04-02 09:21:32 ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Comparing /work/SRC/openSUSE:Factory/python-cloudpickle (Old) and /work/SRC/openSUSE:Factory/.python-cloudpickle.new.25356 (New) ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Package is "python-cloudpickle" Tue Apr 2 09:21:32 2019 rev:5 rq:689381 version:0.8.1 Changes: -------- --- /work/SRC/openSUSE:Factory/python-cloudpickle/python-cloudpickle.changes 2019-02-06 15:48:19.751226078 +0100 +++ /work/SRC/openSUSE:Factory/.python-cloudpickle.new.25356/python-cloudpickle.changes 2019-04-02 09:21:46.028681327 +0200 @@ -1,0 +2,13 @@ +Thu Mar 28 14:20:54 UTC 2019 - Tomáš Chvátal + +- Update to 0.8.1: + * Fix a bug (already present before 0.5.3 and re-introduced in 0.8.0) affecting relative import instructions inside depickled functions (issue #254) + +------------------------------------------------------------------- +Thu Mar 7 13:00:07 UTC 2019 - Tomáš Chvátal + +- Update to 0.8.0: + * Add support for pickling interactively defined dataclasses. (issue #245) + * Global variables referenced by functions pickled by cloudpickle are now unpickled in a new and isolated namespace scoped by the CloudPickler instance. This restores the (previously untested) behavior of cloudpickle prior to changes done in 0.5.4 for functions defined in the __main__ module, and 0.6.0/1 for other dynamic functions. + +------------------------------------------------------------------- Old: ---- cloudpickle-0.7.0.tar.gz New: ---- cloudpickle-0.8.1.tar.gz ++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++++ Other differences: ------------------ ++++++ python-cloudpickle.spec ++++++ --- /var/tmp/diff_new_pack.87w4wu/_old 2019-04-02 09:21:47.204682428 +0200 +++ /var/tmp/diff_new_pack.87w4wu/_new 2019-04-02 09:21:47.208682432 +0200 @@ -18,7 +18,7 @@ %{?!python_module:%define python_module() python-%{**} python3-%{**}} Name: python-cloudpickle -Version: 0.7.0 +Version: 0.8.1 Release: 0 Summary: Extended pickling support for Python objects License: BSD-3-Clause @@ -27,11 +27,13 @@ Source: https://files.pythonhosted.org/packages/source/c/cloudpickle/cloudpickle-%{version}.tar.gz BuildRequires: %{python_module setuptools} BuildRequires: fdupes +BuildRequires: python-futures BuildRequires: python-rpm-macros BuildArch: noarch BuildRequires: %{python_module curses} BuildRequires: %{python_module mock} BuildRequires: %{python_module numpy >= 1.8.2} +BuildRequires: %{python_module psutil} BuildRequires: %{python_module pytest-cov} BuildRequires: %{python_module pytest} BuildRequires: %{python_module scipy} @@ -65,7 +67,6 @@ %python_expand %fdupes %{buildroot}%{$python_sitelib} %check -# Tests require very specific paths and py.test arguments export PYTHONPATH='.:tests' %python_expand py.test-%{$python_bin_suffix} -s ++++++ cloudpickle-0.7.0.tar.gz -> cloudpickle-0.8.1.tar.gz ++++++ diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/PKG-INFO new/cloudpickle-0.8.1/PKG-INFO --- old/cloudpickle-0.7.0/PKG-INFO 2019-01-23 17:36:06.000000000 +0100 +++ new/cloudpickle-0.8.1/PKG-INFO 2019-03-25 10:07:23.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cloudpickle -Version: 0.7.0 +Version: 0.8.1 Summary: Extended pickling support for Python objects Home-page: https://github.com/cloudpipe/cloudpickle Author: Cloudpipe @@ -15,16 +15,20 @@ `cloudpickle` makes it possible to serialize Python constructs not supported by the default `pickle` module from the Python standard library. - `cloudpickle` is especially useful for cluster computing where Python - expressions are shipped over the network to execute on remote hosts, possibly - close to the data. - - Among other things, `cloudpickle` supports pickling for lambda expressions, - functions and classes defined interactively in the `__main__` module. - - `cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default: it is meant to - send objects between processes running the same version of Python. It is - discouraged to use `cloudpickle` for long-term storage. + `cloudpickle` is especially useful for **cluster computing** where Python + code is shipped over the network to execute on remote hosts, possibly close + to the data. + + Among other things, `cloudpickle` supports pickling for **lambda functions** + along with **functions and classes defined interactively** in the + `__main__` module (for instance in a script, a shell or a Jupyter notebook). + + **`cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default**: it is meant to + send objects between processes running the **same version of Python**. + + Using `cloudpickle` for **long-term object storage is not supported and + discouraged.** + Installation ------------ @@ -75,7 +79,7 @@ or alternatively for a specific environment: - tox -e py27 + tox -e py37 - With `py.test` to only run the tests for your current version of @@ -108,9 +112,9 @@ Classifier: Operating System :: Microsoft :: Windows Classifier: Operating System :: MacOS :: MacOS X Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Software Development :: Libraries :: Python Modules diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/README.md new/cloudpickle-0.8.1/README.md --- old/cloudpickle-0.7.0/README.md 2017-11-15 17:06:41.000000000 +0100 +++ new/cloudpickle-0.8.1/README.md 2019-02-19 14:29:43.000000000 +0100 @@ -7,16 +7,20 @@ `cloudpickle` makes it possible to serialize Python constructs not supported by the default `pickle` module from the Python standard library. -`cloudpickle` is especially useful for cluster computing where Python -expressions are shipped over the network to execute on remote hosts, possibly -close to the data. - -Among other things, `cloudpickle` supports pickling for lambda expressions, -functions and classes defined interactively in the `__main__` module. - -`cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default: it is meant to -send objects between processes running the same version of Python. It is -discouraged to use `cloudpickle` for long-term storage. +`cloudpickle` is especially useful for **cluster computing** where Python +code is shipped over the network to execute on remote hosts, possibly close +to the data. + +Among other things, `cloudpickle` supports pickling for **lambda functions** +along with **functions and classes defined interactively** in the +`__main__` module (for instance in a script, a shell or a Jupyter notebook). + +**`cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default**: it is meant to +send objects between processes running the **same version of Python**. + +Using `cloudpickle` for **long-term object storage is not supported and +discouraged.** + Installation ------------ @@ -67,7 +71,7 @@ or alternatively for a specific environment: - tox -e py27 + tox -e py37 - With `py.test` to only run the tests for your current version of diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/cloudpickle/__init__.py new/cloudpickle-0.8.1/cloudpickle/__init__.py --- old/cloudpickle-0.7.0/cloudpickle/__init__.py 2019-01-23 17:34:22.000000000 +0100 +++ new/cloudpickle-0.8.1/cloudpickle/__init__.py 2019-03-25 10:07:01.000000000 +0100 @@ -2,4 +2,4 @@ from cloudpickle.cloudpickle import * -__version__ = '0.7.0' +__version__ = '0.8.1' diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/cloudpickle/cloudpickle.py new/cloudpickle-0.8.1/cloudpickle/cloudpickle.py --- old/cloudpickle-0.7.0/cloudpickle/cloudpickle.py 2019-01-23 17:32:14.000000000 +0100 +++ new/cloudpickle-0.8.1/cloudpickle/cloudpickle.py 2019-03-25 09:45:31.000000000 +0100 @@ -63,7 +63,7 @@ DEFAULT_PROTOCOL = pickle.HIGHEST_PROTOCOL -if sys.version < '3': +if sys.version_info[0] < 3: # pragma: no branch from pickle import Pickler try: from cStringIO import StringIO @@ -79,22 +79,6 @@ PY3 = True -# Container for the global namespace to ensure consistent unpickling of -# functions defined in dynamic modules (modules not registed in sys.modules). -_dynamic_modules_globals = weakref.WeakValueDictionary() - - -class _DynamicModuleFuncGlobals(dict): - """Global variables referenced by a function defined in a dynamic module - - To avoid leaking references we store such context in a WeakValueDictionary - instance. However instances of python builtin types such as dict cannot - be used directly as values in such a construct, hence the need for a - derived class. - """ - pass - - def _make_cell_set_template_code(): """Get the Python compiler to emit LOAD_FAST(arg); STORE_DEREF @@ -128,7 +112,7 @@ # NOTE: we are marking the cell variable as a free variable intentionally # so that we simulate an inner function instead of the outer function. This # is what gives us the ``nonlocal`` behavior in a Python 2 compatible way. - if not PY3: + if not PY3: # pragma: no branch return types.CodeType( co.co_argcount, co.co_nlocals, @@ -229,14 +213,14 @@ } -if sys.version_info < (3, 4): +if sys.version_info < (3, 4): # pragma: no branch def _walk_global_ops(code): """ Yield (opcode, argument number) tuples for all global-referencing instructions in *code*. """ code = getattr(code, 'co_code', b'') - if not PY3: + if not PY3: # pragma: no branch code = map(ord, code) n = len(code) @@ -293,7 +277,7 @@ dispatch[memoryview] = save_memoryview - if not PY3: + if not PY3: # pragma: no branch def save_buffer(self, obj): self.save(str(obj)) @@ -315,7 +299,7 @@ """ Save a code object """ - if PY3: + if PY3: # pragma: no branch args = ( obj.co_argcount, obj.co_kwonlyargcount, obj.co_nlocals, obj.co_stacksize, obj.co_flags, obj.co_code, obj.co_consts, obj.co_names, obj.co_varnames, @@ -393,7 +377,7 @@ # So we pickle them here using save_reduce; have to do it differently # for different python versions. if not hasattr(obj, '__code__'): - if PY3: + if PY3: # pragma: no branch rv = obj.__reduce_ex__(self.proto) else: if hasattr(obj, '__self__'): @@ -670,17 +654,26 @@ # save the dict dct = func.__dict__ - base_globals = self.globals_ref.get(id(func.__globals__), None) - if base_globals is None: - # For functions defined in a well behaved module use - # vars(func.__module__) for base_globals. This is necessary to - # share the global variables across multiple pickled functions from - # this module. - if func.__module__ is not None: - base_globals = func.__module__ - else: - base_globals = {} - self.globals_ref[id(func.__globals__)] = base_globals + # base_globals represents the future global namespace of func at + # unpickling time. Looking it up and storing it in globals_ref allow + # functions sharing the same globals at pickling time to also + # share them once unpickled, at one condition: since globals_ref is + # an attribute of a Cloudpickler instance, and that a new CloudPickler is + # created each time pickle.dump or pickle.dumps is called, functions + # also need to be saved within the same invokation of + # cloudpickle.dump/cloudpickle.dumps (for example: cloudpickle.dumps([f1, f2])). There + # is no such limitation when using Cloudpickler.dump, as long as the + # multiple invokations are bound to the same Cloudpickler. + base_globals = self.globals_ref.setdefault(id(func.__globals__), {}) + + if base_globals == {}: + # Add module attributes used to resolve relative imports + # instructions inside func. + for k in ["__package__", "__name__", "__path__", "__file__"]: + # Some built-in functions/methods such as object.__new__ have + # their __globals__ set to None in PyPy + if func.__globals__ is not None and k in func.__globals__: + base_globals[k] = func.__globals__[k] return (code, f_globals, defaults, closure, dct, base_globals) @@ -730,7 +723,7 @@ if obj.__self__ is None: self.save_reduce(getattr, (obj.im_class, obj.__name__)) else: - if PY3: + if PY3: # pragma: no branch self.save_reduce(types.MethodType, (obj.__func__, obj.__self__), obj=obj) else: self.save_reduce(types.MethodType, (obj.__func__, obj.__self__, obj.__self__.__class__), @@ -783,7 +776,7 @@ save(stuff) write(pickle.BUILD) - if not PY3: + if not PY3: # pragma: no branch dispatch[types.InstanceType] = save_inst def save_property(self, obj): @@ -883,7 +876,7 @@ try: # Python 2 dispatch[file] = save_file - except NameError: # Python 3 + except NameError: # Python 3 # pragma: no branch dispatch[io.TextIOWrapper] = save_file dispatch[type(Ellipsis)] = save_ellipsis @@ -904,6 +897,12 @@ dispatch[logging.RootLogger] = save_root_logger + if hasattr(types, "MappingProxyType"): # pragma: no branch + def save_mappingproxy(self, obj): + self.save_reduce(types.MappingProxyType, (dict(obj),), obj=obj) + + dispatch[types.MappingProxyType] = save_mappingproxy + """Special functions for Add-on libraries""" def inject_addons(self): """Plug in system. Register additional pickling functions if modules already loaded""" @@ -989,43 +988,6 @@ return obj -def _get_module_builtins(): - return pickle.__builtins__ - - -def print_exec(stream): - ei = sys.exc_info() - traceback.print_exception(ei[0], ei[1], ei[2], None, stream) - - -def _modules_to_main(modList): - """Force every module in modList to be placed into main""" - if not modList: - return - - main = sys.modules['__main__'] - for modname in modList: - if type(modname) is str: - try: - mod = __import__(modname) - except Exception: - sys.stderr.write('warning: could not import %s\n. ' - 'Your function may unexpectedly error due to this import failing;' - 'A version mismatch is likely. Specific error was:\n' % modname) - print_exec(sys.stderr) - else: - setattr(main, mod.__name__, mod) - - -# object generators: -def _genpartial(func, args, kwds): - if not args: - args = () - if not kwds: - kwds = {} - return partial(func, *args, **kwds) - - def _gen_ellipsis(): return Ellipsis @@ -1090,10 +1052,16 @@ else: raise ValueError('Unexpected _fill_value arguments: %r' % (args,)) - # Only set global variables that do not exist. - for k, v in state['globals'].items(): - if k not in func.__globals__: - func.__globals__[k] = v + # - At pickling time, any dynamic global variable used by func is + # serialized by value (in state['globals']). + # - At unpickling time, func's __globals__ attribute is initialized by + # first retrieving an empty isolated namespace that will be shared + # with other functions pickled from the same original module + # by the same CloudPickler instance and then updated with the + # content of state['globals'] to populate the shared isolated + # namespace with all the global variables that are specifically + # referenced for this function. + func.__globals__.update(state['globals']) func.__defaults__ = state['defaults'] func.__dict__ = state['dict'] @@ -1131,21 +1099,11 @@ code and the correct number of cells in func_closure. All other func attributes (e.g. func_globals) are empty. """ - if base_globals is None: + # This is backward-compatibility code: for cloudpickle versions between + # 0.5.4 and 0.7, base_globals could be a string or None. base_globals + # should now always be a dictionary. + if base_globals is None or isinstance(base_globals, str): base_globals = {} - elif isinstance(base_globals, str): - base_globals_name = base_globals - try: - # First try to reuse the globals from the module containing the - # function. If it is not possible to retrieve it, fallback to an - # empty dictionary. - base_globals = vars(importlib.import_module(base_globals)) - except ImportError: - base_globals = _dynamic_modules_globals.get( - base_globals_name, None) - if base_globals is None: - base_globals = _DynamicModuleFuncGlobals() - _dynamic_modules_globals[base_globals_name] = base_globals base_globals['__builtins__'] = __builtins__ @@ -1202,18 +1160,9 @@ return False -"""Constructors for 3rd party libraries -Note: These can never be renamed due to client compatibility issues""" - - -def _getobject(modname, attribute): - mod = __import__(modname, fromlist=[attribute]) - return mod.__dict__[attribute] - - """ Use copy_reg to extend global pickle definitions """ -if sys.version_info < (3, 4): +if sys.version_info < (3, 4): # pragma: no branch method_descriptor = type(str.upper) def _reduce_method_descriptor(obj): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/cloudpickle.egg-info/PKG-INFO new/cloudpickle-0.8.1/cloudpickle.egg-info/PKG-INFO --- old/cloudpickle-0.7.0/cloudpickle.egg-info/PKG-INFO 2019-01-23 17:36:05.000000000 +0100 +++ new/cloudpickle-0.8.1/cloudpickle.egg-info/PKG-INFO 2019-03-25 10:07:23.000000000 +0100 @@ -1,6 +1,6 @@ Metadata-Version: 1.1 Name: cloudpickle -Version: 0.7.0 +Version: 0.8.1 Summary: Extended pickling support for Python objects Home-page: https://github.com/cloudpipe/cloudpickle Author: Cloudpipe @@ -15,16 +15,20 @@ `cloudpickle` makes it possible to serialize Python constructs not supported by the default `pickle` module from the Python standard library. - `cloudpickle` is especially useful for cluster computing where Python - expressions are shipped over the network to execute on remote hosts, possibly - close to the data. - - Among other things, `cloudpickle` supports pickling for lambda expressions, - functions and classes defined interactively in the `__main__` module. - - `cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default: it is meant to - send objects between processes running the same version of Python. It is - discouraged to use `cloudpickle` for long-term storage. + `cloudpickle` is especially useful for **cluster computing** where Python + code is shipped over the network to execute on remote hosts, possibly close + to the data. + + Among other things, `cloudpickle` supports pickling for **lambda functions** + along with **functions and classes defined interactively** in the + `__main__` module (for instance in a script, a shell or a Jupyter notebook). + + **`cloudpickle` uses `pickle.HIGHEST_PROTOCOL` by default**: it is meant to + send objects between processes running the **same version of Python**. + + Using `cloudpickle` for **long-term object storage is not supported and + discouraged.** + Installation ------------ @@ -75,7 +79,7 @@ or alternatively for a specific environment: - tox -e py27 + tox -e py37 - With `py.test` to only run the tests for your current version of @@ -108,9 +112,9 @@ Classifier: Operating System :: Microsoft :: Windows Classifier: Operating System :: MacOS :: MacOS X Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3.4 Classifier: Programming Language :: Python :: 3.5 Classifier: Programming Language :: Python :: 3.6 +Classifier: Programming Language :: Python :: 3.7 Classifier: Programming Language :: Python :: Implementation :: CPython Classifier: Programming Language :: Python :: Implementation :: PyPy Classifier: Topic :: Software Development :: Libraries :: Python Modules diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/cloudpickle.egg-info/SOURCES.txt new/cloudpickle-0.8.1/cloudpickle.egg-info/SOURCES.txt --- old/cloudpickle-0.7.0/cloudpickle.egg-info/SOURCES.txt 2019-01-23 17:36:05.000000000 +0100 +++ new/cloudpickle-0.8.1/cloudpickle.egg-info/SOURCES.txt 2019-03-25 10:07:23.000000000 +0100 @@ -12,4 +12,6 @@ tests/__init__.py tests/cloudpickle_file_test.py tests/cloudpickle_test.py -tests/testutils.py \ No newline at end of file +tests/testutils.py +tests/mypkg/__init__.py +tests/mypkg/mod.py \ No newline at end of file diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/setup.py new/cloudpickle-0.8.1/setup.py --- old/cloudpickle-0.7.0/setup.py 2019-01-23 17:29:10.000000000 +0100 +++ new/cloudpickle-0.8.1/setup.py 2019-01-31 14:05:30.000000000 +0100 @@ -39,9 +39,9 @@ 'Operating System :: Microsoft :: Windows', 'Operating System :: MacOS :: MacOS X', 'Programming Language :: Python :: 2.7', - 'Programming Language :: Python :: 3.4', 'Programming Language :: Python :: 3.5', 'Programming Language :: Python :: 3.6', + 'Programming Language :: Python :: 3.7', 'Programming Language :: Python :: Implementation :: CPython', 'Programming Language :: Python :: Implementation :: PyPy', 'Topic :: Software Development :: Libraries :: Python Modules', diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/tests/cloudpickle_test.py new/cloudpickle-0.8.1/tests/cloudpickle_test.py --- old/cloudpickle-0.7.0/tests/cloudpickle_test.py 2019-01-23 10:51:18.000000000 +0100 +++ new/cloudpickle-0.8.1/tests/cloudpickle_test.py 2019-03-25 09:45:31.000000000 +0100 @@ -451,57 +451,6 @@ mod1, mod2 = pickle_depickle([mod, mod]) self.assertEqual(id(mod1), id(mod2)) - def test_dynamic_modules_globals(self): - # _dynamic_modules_globals is a WeakValueDictionary, so if a value - # in this dict (containing a set of global variables from a dynamic - # module created in the parent process) has no other reference than in - # this dict in the child process, it will be garbage collected. - - # We first create a module - mod = types.ModuleType('mod') - code = ''' - x = 1 - def func(): - return - ''' - exec(textwrap.dedent(code), mod.__dict__) - - pickled_module_path = os.path.join(self.tmpdir, 'mod_f.pkl') - child_process_script = ''' - import pickle - from cloudpickle.cloudpickle import _dynamic_modules_globals - import gc - with open("{pickled_module_path}", 'rb') as f: - func = pickle.load(f) - - # A dictionnary storing the globals of the newly unpickled function - # should have been created - assert list(_dynamic_modules_globals.keys()) == ['mod'] - - # func.__globals__ is the only non-weak reference to - # _dynamic_modules_globals['mod']. By deleting func, we delete also - # _dynamic_modules_globals['mod'] - del func - gc.collect() - - # There is no reference to the globals of func since func has been - # deleted and _dynamic_modules_globals is a WeakValueDictionary, - # so _dynamic_modules_globals should now be empty - assert list(_dynamic_modules_globals.keys()) == [] - ''' - - child_process_script = child_process_script.format( - pickled_module_path=_escape(pickled_module_path)) - - try: - with open(pickled_module_path, 'wb') as f: - cloudpickle.dump(mod.func, f, protocol=self.protocol) - - assert_run_python_script(textwrap.dedent(child_process_script)) - - finally: - os.unlink(pickled_module_path) - def test_module_locals_behavior(self): # Makes sure that a local function defined in another module is # correctly serialized. This notably checks that the globals are @@ -1029,6 +978,12 @@ def f4(x): return foo.method(x) + def f5(x): + # Recursive call to a dynamically defined function. + if x <= 0: + return f4(x) + return f5(x - 1) + 1 + cloned = subprocess_pickle_echo(lambda x: x**2, protocol={protocol}) assert cloned(3) == 9 @@ -1052,6 +1007,9 @@ cloned = subprocess_pickle_echo(f4, protocol={protocol}) assert cloned(2) == f4(2) + + cloned = subprocess_pickle_echo(f5, protocol={protocol}) + assert cloned(7) == f5(7) == 7 """.format(protocol=self.protocol) assert_run_python_script(textwrap.dedent(code)) @@ -1074,20 +1032,42 @@ def f1(): return VARIABLE - cloned_f0 = {clone_func}(f0, protocol={protocol}) - cloned_f1 = {clone_func}(f1, protocol={protocol}) + assert f0.__globals__ is f1.__globals__ + + # pickle f0 and f1 inside the same pickle_string + cloned_f0, cloned_f1 = {clone_func}([f0, f1], protocol={protocol}) + + # cloned_f0 and cloned_f1 now share a global namespace that is isolated + # from any previously existing namespace + assert cloned_f0.__globals__ is cloned_f1.__globals__ + assert cloned_f0.__globals__ is not f0.__globals__ + + # pickle f1 another time, but in a new pickle string pickled_f1 = dumps(f1, protocol={protocol}) - # Change the value of the global variable + # Change the value of the global variable in f0's new global namespace cloned_f0() - # Ensure that the global variable is the same for another function - result_f1 = cloned_f1() - assert result_f1 == "changed_by_f0", result_f1 - - # Ensure that unpickling the global variable does not change its value - result_pickled_f1 = loads(pickled_f1)() - assert result_pickled_f1 == "changed_by_f0", result_pickled_f1 + # thanks to cloudpickle isolation, depickling and calling f0 and f1 + # should not affect the globals of already existing modules + assert VARIABLE == "default_value", VARIABLE + + # Ensure that cloned_f1 and cloned_f0 share the same globals, as f1 and + # f0 shared the same globals at pickling time, and cloned_f1 was + # depickled from the same pickle string as cloned_f0 + shared_global_var = cloned_f1() + assert shared_global_var == "changed_by_f0", shared_global_var + + # f1 is unpickled another time, but because it comes from another + # pickle string than pickled_f1 and pickled_f0, it will not share the + # same globals as the latter two. + new_cloned_f1 = loads(pickled_f1) + assert new_cloned_f1.__globals__ is not cloned_f1.__globals__ + assert new_cloned_f1.__globals__ is not f1.__globals__ + + # get the value of new_cloned_f1's VARIABLE + new_global_var = new_cloned_f1() + assert new_global_var == "default_value", new_global_var """ for clone_func in ['local_clone', 'subprocess_pickle_echo']: code = code_template.format(protocol=self.protocol, @@ -1106,116 +1086,154 @@ def f1(): return _TEST_GLOBAL_VARIABLE - cloned_f0 = cloudpickle.loads(cloudpickle.dumps( - f0, protocol=self.protocol)) - cloned_f1 = cloudpickle.loads(cloudpickle.dumps( - f1, protocol=self.protocol)) + # pickle f0 and f1 inside the same pickle_string + cloned_f0, cloned_f1 = pickle_depickle([f0, f1], + protocol=self.protocol) + + # cloned_f0 and cloned_f1 now share a global namespace that is + # isolated from any previously existing namespace + assert cloned_f0.__globals__ is cloned_f1.__globals__ + assert cloned_f0.__globals__ is not f0.__globals__ + + # pickle f1 another time, but in a new pickle string pickled_f1 = cloudpickle.dumps(f1, protocol=self.protocol) - # Change the value of the global variable + # Change the global variable's value in f0's new global namespace cloned_f0() - assert _TEST_GLOBAL_VARIABLE == "changed_by_f0" - # Ensure that the global variable is the same for another function - result_cloned_f1 = cloned_f1() - assert result_cloned_f1 == "changed_by_f0", result_cloned_f1 - assert f1() == result_cloned_f1 - - # Ensure that unpickling the global variable does not change its - # value - result_pickled_f1 = cloudpickle.loads(pickled_f1)() - assert result_pickled_f1 == "changed_by_f0", result_pickled_f1 + # depickling f0 and f1 should not affect the globals of already + # existing modules + assert _TEST_GLOBAL_VARIABLE == "default_value" + + # Ensure that cloned_f1 and cloned_f0 share the same globals, as f1 + # and f0 shared the same globals at pickling time, and cloned_f1 + # was depickled from the same pickle string as cloned_f0 + shared_global_var = cloned_f1() + assert shared_global_var == "changed_by_f0", shared_global_var + + # f1 is unpickled another time, but because it comes from another + # pickle string than pickled_f1 and pickled_f0, it will not share + # the same globals as the latter two. + new_cloned_f1 = pickle.loads(pickled_f1) + assert new_cloned_f1.__globals__ is not cloned_f1.__globals__ + assert new_cloned_f1.__globals__ is not f1.__globals__ + + # get the value of new_cloned_f1's VARIABLE + new_global_var = new_cloned_f1() + assert new_global_var == "default_value", new_global_var finally: _TEST_GLOBAL_VARIABLE = orig_value - def test_function_from_dynamic_module_with_globals_modifications(self): - # This test verifies that the global variable state of a function - # defined in a dynamic module in a child process are not reset by - # subsequent uplickling. + def test_interactive_remote_function_calls(self): + code = """if __name__ == "__main__": + from testutils import subprocess_worker - # first, we create a dynamic module in the parent process - mod = types.ModuleType('mod') - code = ''' - GLOBAL_STATE = "initial value" + def interactive_function(x): + return x + 1 - def func_defined_in_dynamic_module(v=None): - global GLOBAL_STATE - if v is not None: - GLOBAL_STATE = v - return GLOBAL_STATE - ''' - exec(textwrap.dedent(code), mod.__dict__) + with subprocess_worker(protocol={protocol}) as w: - with_initial_globals_file = os.path.join( - self.tmpdir, 'function_with_initial_globals.pkl') - with_modified_globals_file = os.path.join( - self.tmpdir, 'function_with_modified_globals.pkl') + assert w.run(interactive_function, 41) == 42 - try: - # Simple sanity check on the function's output - assert mod.func_defined_in_dynamic_module() == "initial value" + # Define a new function that will call an updated version of + # the previously called function: - # The function of mod is pickled two times, with two different - # values for the global variable GLOBAL_STATE. - # Then we launch a child process that sequentially unpickles the - # two functions. Those unpickle functions should share the same - # global variables in the child process: - # Once the first function gets unpickled, mod is created and - # tracked in the child environment. This is state is preserved - # when unpickling the second function whatever the global variable - # GLOBAL_STATE's value at the time of pickling. - - with open(with_initial_globals_file, 'wb') as f: - cloudpickle.dump(mod.func_defined_in_dynamic_module, f) - - # Change the mod's global variable - mod.GLOBAL_STATE = 'changed value' - - # At this point, mod.func_defined_in_dynamic_module() - # returns the updated value. Let's pickle it again. - assert mod.func_defined_in_dynamic_module() == 'changed value' - with open(with_modified_globals_file, 'wb') as f: - cloudpickle.dump(mod.func_defined_in_dynamic_module, f, - protocol=self.protocol) + def wrapper_func(x): + return interactive_function(x) - child_process_code = """ - import pickle + def interactive_function(x): + return x - 1 + + # The change in the definition of interactive_function in the main + # module of the main process should be reflected transparently + # in the worker process: the worker process does not recall the + # previous definition of `interactive_function`: - with open({with_initial_globals_file!r},'rb') as f: - func_with_initial_globals = pickle.load(f) + assert w.run(wrapper_func, 41) == 40 + """.format(protocol=self.protocol) + assert_run_python_script(code) - # At this point, a module called 'mod' should exist in - # _dynamic_modules_globals. Further function loading - # will use the globals living in mod. - - assert func_with_initial_globals() == 'initial value' - - # Load a function with initial global variable that was - # pickled after a change in the global variable - with open({with_initial_globals_file!r},'rb') as f: - func_with_modified_globals = pickle.load(f) - - # assert the this unpickling did not modify the value of - # the local - assert func_with_modified_globals() == 'initial value' - - # Update the value from the child process and check that - # unpickling again does not reset our change. - assert func_with_initial_globals('new value') == 'new value' - assert func_with_modified_globals() == 'new value' - - with open({with_initial_globals_file!r},'rb') as f: - func_with_initial_globals = pickle.load(f) - assert func_with_initial_globals() == 'new value' - assert func_with_modified_globals() == 'new value' - """.format( - with_initial_globals_file=_escape(with_initial_globals_file), - with_modified_globals_file=_escape(with_modified_globals_file)) - assert_run_python_script(textwrap.dedent(child_process_code)) + def test_interactive_remote_function_calls_no_side_effect(self): + code = """if __name__ == "__main__": + from testutils import subprocess_worker + import sys - finally: - os.unlink(with_initial_globals_file) - os.unlink(with_modified_globals_file) + with subprocess_worker(protocol={protocol}) as w: + + GLOBAL_VARIABLE = 0 + + class CustomClass(object): + + def mutate_globals(self): + global GLOBAL_VARIABLE + GLOBAL_VARIABLE += 1 + return GLOBAL_VARIABLE + + custom_object = CustomClass() + assert w.run(custom_object.mutate_globals) == 1 + + # The caller global variable is unchanged in the main process. + + assert GLOBAL_VARIABLE == 0 + + # Calling the same function again starts again from zero. The + # worker process is stateless: it has no memory of the past call: + + assert w.run(custom_object.mutate_globals) == 1 + + # The symbols defined in the main process __main__ module are + # not set in the worker process main module to leave the worker + # as stateless as possible: + + def is_in_main(name): + return hasattr(sys.modules["__main__"], name) + + assert is_in_main("CustomClass") + assert not w.run(is_in_main, "CustomClass") + + assert is_in_main("GLOBAL_VARIABLE") + assert not w.run(is_in_main, "GLOBAL_VARIABLE") + + """.format(protocol=self.protocol) + assert_run_python_script(code) + + @pytest.mark.skipif(platform.python_implementation() == 'PyPy', + reason="Skip PyPy because memory grows too much") + def test_interactive_remote_function_calls_no_memory_leak(self): + code = """if __name__ == "__main__": + from testutils import subprocess_worker + import struct + + with subprocess_worker(protocol={protocol}) as w: + + reference_size = w.memsize() + assert reference_size > 0 + + + def make_big_closure(i): + # Generate a byte string of size 1MB + itemsize = len(struct.pack("l", 1)) + data = struct.pack("l", i) * (int(1e6) // itemsize) + def process_data(): + return len(data) + return process_data + + for i in range(100): + func = make_big_closure(i) + result = w.run(func) + assert result == int(1e6), result + + import gc + w.run(gc.collect) + + # By this time the worker process has processed worth of 100MB of + # data passed in the closures its memory size should now have + # grown by more than a few MB. + growth = w.memsize() - reference_size + assert growth < 1e7, growth + + """.format(protocol=self.protocol) + assert_run_python_script(code) @pytest.mark.skipif(sys.version_info >= (3, 0), reason="hardcoded pickle bytes for 2.7") @@ -1335,6 +1353,45 @@ with pytest.raises(AttributeError): obj.non_registered_attribute = 1 + @unittest.skipIf(not hasattr(types, "MappingProxyType"), + "Old versions of Python do not have this type.") + def test_mappingproxy(self): + mp = types.MappingProxyType({"some_key": "some value"}) + assert mp == pickle_depickle(mp, protocol=self.protocol) + + def test_dataclass(self): + dataclasses = pytest.importorskip("dataclasses") + + DataClass = dataclasses.make_dataclass('DataClass', [('x', int)]) + data = DataClass(x=42) + + pickle_depickle(DataClass, protocol=self.protocol) + assert data.x == pickle_depickle(data, protocol=self.protocol).x == 42 + + def test_relative_import_inside_function(self): + # Make sure relative imports inside round-tripped functions is not + # broken.This was a bug in cloudpickle versions <= 0.5.3 and was + # re-introduced in 0.8.0. + + # Both functions living inside modules and packages are tested. + def f(): + # module_function belongs to mypkg.mod1, which is a module + from .mypkg import module_function + return module_function() + + def g(): + # package_function belongs to mypkg, which is a package + from .mypkg import package_function + return package_function() + + for func, source in zip([f, g], ["module", "package"]): + # Make sure relative imports are initially working + assert func() == "hello from a {}!".format(source) + + # Make sure relative imports still work after round-tripping + cloned_func = pickle_depickle(func, protocol=self.protocol) + assert cloned_func() == "hello from a {}!".format(source) + class Protocol2CloudPickleTest(CloudPickleTest): diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/tests/mypkg/__init__.py new/cloudpickle-0.8.1/tests/mypkg/__init__.py --- old/cloudpickle-0.7.0/tests/mypkg/__init__.py 1970-01-01 01:00:00.000000000 +0100 +++ new/cloudpickle-0.8.1/tests/mypkg/__init__.py 2019-03-25 09:45:31.000000000 +0100 @@ -0,0 +1,6 @@ +from .mod import module_function + + +def package_function(): + """Function living inside a package, not a simple module""" + return "hello from a package!" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/tests/mypkg/mod.py new/cloudpickle-0.8.1/tests/mypkg/mod.py --- old/cloudpickle-0.7.0/tests/mypkg/mod.py 1970-01-01 01:00:00.000000000 +0100 +++ new/cloudpickle-0.8.1/tests/mypkg/mod.py 2019-03-25 09:45:31.000000000 +0100 @@ -0,0 +1,2 @@ +def module_function(): + return "hello from a module!" diff -urN '--exclude=CVS' '--exclude=.cvsignore' '--exclude=.svn' '--exclude=.svnignore' old/cloudpickle-0.7.0/tests/testutils.py new/cloudpickle-0.8.1/tests/testutils.py --- old/cloudpickle-0.7.0/tests/testutils.py 2019-01-23 16:35:51.000000000 +0100 +++ new/cloudpickle-0.8.1/tests/testutils.py 2019-02-19 14:29:43.000000000 +0100 @@ -4,8 +4,12 @@ import tempfile import base64 from subprocess import Popen, check_output, PIPE, STDOUT, CalledProcessError -from cloudpickle import dumps from pickle import loads +from contextlib import contextmanager +from concurrent.futures import ProcessPoolExecutor + +import psutil +from cloudpickle import dumps TIMEOUT = 60 try: @@ -42,6 +46,20 @@ return cloudpickle_repo_folder, env +def _pack(input_data, protocol=None): + pickled_input_data = dumps(input_data, protocol=protocol) + # Under Windows + Python 2.7, subprocess / communicate truncate the data + # on some specific bytes. To avoid this issue, let's use the pure ASCII + # Base32 encoding to encapsulate the pickle message sent to the child + # process. + return base64.b32encode(pickled_input_data) + + +def _unpack(packed_data): + decoded_data = base64.b32decode(packed_data) + return loads(decoded_data) + + def subprocess_pickle_echo(input_data, protocol=None, timeout=TIMEOUT): """Echo function with a child Python process @@ -53,18 +71,12 @@ [1, 'a', None] """ - pickled_input_data = dumps(input_data, protocol=protocol) - # Under Windows + Python 2.7, subprocess / communicate truncate the data - # on some specific bytes. To avoid this issue, let's use the pure ASCII - # Base32 encoding to encapsulate the pickle message sent to the child - # process. - pickled_b32 = base64.b32encode(pickled_input_data) - # run then pickle_echo(protocol=protocol) in __main__: cmd = [sys.executable, __file__, "--protocol", str(protocol)] cwd, env = _make_cwd_env() proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=cwd, env=env, bufsize=4096) + pickled_b32 = _pack(input_data, protocol=protocol) try: comm_kwargs = {} if timeout_supported: @@ -74,7 +86,7 @@ message = "Subprocess returned %d: " % proc.returncode message += err.decode('utf-8') raise RuntimeError(message) - return loads(base64.b32decode(out)) + return _unpack(out) except TimeoutExpired: proc.kill() out, err = proc.communicate() @@ -113,6 +125,56 @@ stream_out.close() +def call_func(payload, protocol): + """Remote function call that uses cloudpickle to transport everthing""" + func, args, kwargs = loads(payload) + try: + result = func(*args, **kwargs) + except BaseException as e: + result = e + return dumps(result, protocol=protocol) + + +class _Worker(object): + def __init__(self, protocol=None): + self.protocol = protocol + self.pool = ProcessPoolExecutor(max_workers=1) + self.pool.submit(id, 42).result() # start the worker process + + def run(self, func, *args, **kwargs): + """Synchronous remote function call""" + + input_payload = dumps((func, args, kwargs), protocol=self.protocol) + result_payload = self.pool.submit( + call_func, input_payload, self.protocol).result() + result = loads(result_payload) + + if isinstance(result, BaseException): + raise result + return result + + def memsize(self): + workers_pids = [p.pid if hasattr(p, "pid") else p + for p in list(self.pool._processes)] + num_workers = len(workers_pids) + if num_workers == 0: + return 0 + elif num_workers > 1: + raise RuntimeError("Unexpected number of workers: %d" + % num_workers) + return psutil.Process(workers_pids[0]).memory_info().rss + + def close(self): + self.pool.shutdown(wait=True) + + +@contextmanager +def subprocess_worker(protocol=None): + worker = _Worker(protocol=protocol) + yield worker + worker.close() + + def assert_run_python_script(source_code, timeout=TIMEOUT): """Utility to help check pickleability of objects defined in __main__