diff --git a/MANIFEST.in b/MANIFEST.in index 33355d0..815c602 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -1,3 +1,4 @@ recursive-include srsly *.h *.pyx *.pxd *.cc *.c *.cpp *.json include LICENSE include README.md +recursive-include srsly/tests/cloudpickle/mock_local_folder *.py diff --git a/srsly/tests/cloudpickle/__init__.py b/srsly/tests/cloudpickle/__init__.py index e69de29..10cd20e 100644 --- a/srsly/tests/cloudpickle/__init__.py +++ b/srsly/tests/cloudpickle/__init__.py @@ -0,0 +1,3 @@ +import pytest + +pytest.register_assert_rewrite("tests.testutils") diff --git a/srsly/tests/cloudpickle/cloudpickle_file_test.py b/srsly/tests/cloudpickle/cloudpickle_file_test.py index 218566f..ab9156e 100644 --- a/srsly/tests/cloudpickle/cloudpickle_file_test.py +++ b/srsly/tests/cloudpickle/cloudpickle_file_test.py @@ -3,11 +3,11 @@ import sys import tempfile import unittest +import pickle import pytest import srsly.cloudpickle as cloudpickle -from srsly.cloudpickle.compat import pickle class CloudPickleFileTests(unittest.TestCase): @@ -16,22 +16,22 @@ class CloudPickleFileTests(unittest.TestCase): def setUp(self): self.tmpdir = tempfile.mkdtemp() - self.tmpfilepath = os.path.join(self.tmpdir, 'testfile') - self.teststring = 'Hello world!' + self.tmpfilepath = os.path.join(self.tmpdir, "testfile") + self.teststring = "Hello world!" def tearDown(self): shutil.rmtree(self.tmpdir) def test_empty_file(self): # Empty file - open(self.tmpfilepath, 'w').close() - with open(self.tmpfilepath, 'r') as f: - self.assertEqual('', pickle.loads(cloudpickle.dumps(f)).read()) + open(self.tmpfilepath, "w").close() + with open(self.tmpfilepath) as f: + self.assertEqual("", pickle.loads(cloudpickle.dumps(f)).read()) os.remove(self.tmpfilepath) def test_closed_file(self): # Write & close - with open(self.tmpfilepath, 'w') as f: + with open(self.tmpfilepath, "w") as f: f.write(self.teststring) with pytest.raises(pickle.PicklingError) as excinfo: cloudpickle.dumps(f) @@ -40,25 +40,24 @@ def test_closed_file(self): def test_r_mode(self): # Write & close - with open(self.tmpfilepath, 'w') as f: + with open(self.tmpfilepath, "w") as f: f.write(self.teststring) # Open for reading - with open(self.tmpfilepath, 'r') as f: + with open(self.tmpfilepath) as f: new_f = pickle.loads(cloudpickle.dumps(f)) self.assertEqual(self.teststring, new_f.read()) os.remove(self.tmpfilepath) def test_w_mode(self): - with open(self.tmpfilepath, 'w') as f: + with open(self.tmpfilepath, "w") as f: f.write(self.teststring) f.seek(0) - self.assertRaises(pickle.PicklingError, - lambda: cloudpickle.dumps(f)) + self.assertRaises(pickle.PicklingError, lambda: cloudpickle.dumps(f)) os.remove(self.tmpfilepath) def test_plus_mode(self): # Write, then seek to 0 - with open(self.tmpfilepath, 'w+') as f: + with open(self.tmpfilepath, "w+") as f: f.write(self.teststring) f.seek(0) new_f = pickle.loads(cloudpickle.dumps(f)) @@ -67,7 +66,7 @@ def test_plus_mode(self): def test_seek(self): # Write, then seek to arbitrary position - with open(self.tmpfilepath, 'w+') as f: + with open(self.tmpfilepath, "w+") as f: f.write(self.teststring) f.seek(4) unpickled = pickle.loads(cloudpickle.dumps(f)) @@ -84,9 +83,8 @@ def test_pickling_special_file_handles(self): # Warning: if you want to run your tests with nose, add -s option for out in sys.stdout, sys.stderr: # Regression test for SPARK-3415 self.assertEqual(out, pickle.loads(cloudpickle.dumps(out))) - self.assertRaises(pickle.PicklingError, - lambda: cloudpickle.dumps(sys.stdin)) + self.assertRaises(pickle.PicklingError, lambda: cloudpickle.dumps(sys.stdin)) -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/srsly/tests/cloudpickle/cloudpickle_test.py b/srsly/tests/cloudpickle/cloudpickle_test.py index e4dba00..c69e47b 100644 --- a/srsly/tests/cloudpickle/cloudpickle_test.py +++ b/srsly/tests/cloudpickle/cloudpickle_test.py @@ -2,6 +2,7 @@ import abc import collections import base64 +import dataclasses import functools import io import itertools @@ -25,8 +26,10 @@ import enum import typing from functools import wraps +import pickle import pytest +from pathlib import Path try: # try importing numpy and scipy. These are not hard dependencies and @@ -37,26 +40,20 @@ np = None spp = None -try: - # Ditto for Tornado - import tornado -except ImportError: - tornado = None - import srsly.cloudpickle as cloudpickle -from srsly.cloudpickle.compat import pickle from srsly.cloudpickle import register_pickle_by_value from srsly.cloudpickle import unregister_pickle_by_value from srsly.cloudpickle import list_registry_pickle_by_value from srsly.cloudpickle.cloudpickle import _should_pickle_by_reference -from srsly.cloudpickle.cloudpickle import _make_empty_cell, cell_set +from srsly.cloudpickle.cloudpickle import _make_empty_cell from srsly.cloudpickle.cloudpickle import _extract_class_dict, _whichmodule from srsly.cloudpickle.cloudpickle import _lookup_module_and_qualname -from .testutils import subprocess_pickle_echo -from .testutils import subprocess_pickle_string -from .testutils import assert_run_python_script -from .testutils import subprocess_worker +from srsly.tests.cloudpickle.testutils import subprocess_worker +from srsly.tests.cloudpickle.testutils import subprocess_pickle_echo +from srsly.tests.cloudpickle.testutils import subprocess_pickle_string +from srsly.tests.cloudpickle.testutils import assert_run_python_script +from srsly.tests.cloudpickle.testutils import check_deterministic_pickle _TEST_GLOBAL_VARIABLE = "default_value" @@ -64,7 +61,6 @@ class RaiserOnPickle: - def __init__(self, exc): self.exc = exc @@ -95,11 +91,13 @@ def _maybe_remove(list_, item): def test_extract_class_dict(): class A(int): """A docstring""" + def method(self): return "a" class B: """B docstring""" + B_CONSTANT = 42 def method(self): @@ -112,14 +110,18 @@ def method_c(self): return "c" clsdict = _extract_class_dict(C) - assert sorted(clsdict.keys()) == ["C_CONSTANT", "__doc__", "method_c"] + expected_keys = ["C_CONSTANT", "__doc__", "method_c"] + # New attribute in Python 3.13 beta 1 + # https://github.com/python/cpython/pull/118475 + if sys.version_info >= (3, 13): + expected_keys.insert(2, "__firstlineno__") + assert list(clsdict.keys()) == expected_keys assert clsdict["C_CONSTANT"] == 43 assert clsdict["__doc__"] is None assert clsdict["method_c"](C()) == C().method_c() class CloudPickleTest(unittest.TestCase): - protocol = cloudpickle.DEFAULT_PROTOCOL def setUp(self): @@ -129,9 +131,9 @@ def tearDown(self): shutil.rmtree(self.tmpdir) @pytest.mark.skipif( - platform.python_implementation() != "CPython" or - (sys.version_info >= (3, 8, 0) and sys.version_info < (3, 8, 2)), - reason="Underlying bug fixed upstream starting Python 3.8.2") + platform.python_implementation() != "CPython" or sys.version_info < (3, 8, 2), + reason="Underlying bug fixed upstream starting Python 3.8.2", + ) def test_reducer_override_reference_cycle(self): # Early versions of Python 3.8 introduced a reference cycle between a # Pickler and it's reducer_override method. Because a Pickler @@ -165,6 +167,7 @@ def test_attrgetter(self): class C: def __getattr__(self, item): return item + d = C() getter = attrgetter("a") getter2 = pickle_depickle(getter, protocol=self.protocol) @@ -196,8 +199,9 @@ def __reduce__(self): global exit exit = Unpicklable() - self.assertRaises(Exception, lambda: cloudpickle.dumps( - exit, protocol=self.protocol)) + self.assertRaises( + Exception, lambda: cloudpickle.dumps(exit, protocol=self.protocol) + ) def foo(): sys.exit(0) @@ -205,21 +209,11 @@ def foo(): self.assertTrue("exit" in foo.__code__.co_names) cloudpickle.dumps(foo) - def test_buffer(self): - try: - buffer_obj = buffer("Hello") - buffer_clone = pickle_depickle(buffer_obj, protocol=self.protocol) - self.assertEqual(buffer_clone, str(buffer_obj)) - buffer_obj = buffer("Hello", 2, 3) - buffer_clone = pickle_depickle(buffer_obj, protocol=self.protocol) - self.assertEqual(buffer_clone, str(buffer_obj)) - except NameError: # Python 3 does no longer support buffers - pass - def test_memoryview(self): buffer_obj = memoryview(b"Hello") - self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), - buffer_obj.tobytes()) + self.assertEqual( + pickle_depickle(buffer_obj, protocol=self.protocol), buffer_obj.tobytes() + ) def test_dict_keys(self): keys = {"a": 1, "b": 2}.keys() @@ -243,49 +237,52 @@ def test_odict_keys(self): keys = collections.OrderedDict([("a", 1), ("b", 2)]).keys() results = pickle_depickle(keys) self.assertEqual(results, keys) - assert type(keys) == type(results) + assert type(keys) is type(results) def test_odict_values(self): values = collections.OrderedDict([("a", 1), ("b", 2)]).values() results = pickle_depickle(values) self.assertEqual(list(results), list(values)) - assert type(values) == type(results) + assert type(values) is type(results) def test_odict_items(self): items = collections.OrderedDict([("a", 1), ("b", 2)]).items() results = pickle_depickle(items) self.assertEqual(results, items) - assert type(items) == type(results) + assert type(items) is type(results) def test_sliced_and_non_contiguous_memoryview(self): buffer_obj = memoryview(b"Hello!" * 3)[2:15:2] - self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), - buffer_obj.tobytes()) + self.assertEqual( + pickle_depickle(buffer_obj, protocol=self.protocol), buffer_obj.tobytes() + ) def test_large_memoryview(self): buffer_obj = memoryview(b"Hello!" * int(1e7)) - self.assertEqual(pickle_depickle(buffer_obj, protocol=self.protocol), - buffer_obj.tobytes()) + self.assertEqual( + pickle_depickle(buffer_obj, protocol=self.protocol), buffer_obj.tobytes() + ) def test_lambda(self): - self.assertEqual( - pickle_depickle(lambda: 1, protocol=self.protocol)(), 1) + self.assertEqual(pickle_depickle(lambda: 1, protocol=self.protocol)(), 1) def test_nested_lambdas(self): a, b = 1, 2 - f1 = lambda x: x + a - f2 = lambda x: f1(x) // b + f1 = lambda x: x + a # noqa: E731 + f2 = lambda x: f1(x) // b # noqa: E731 self.assertEqual(pickle_depickle(f2, protocol=self.protocol)(1), 1) def test_recursive_closure(self): def f1(): def g(): return g + return g def f2(base): def g(n): return base if n <= 1 else n * g(n - 1) + return g g1 = pickle_depickle(f1(), protocol=self.protocol) @@ -296,19 +293,18 @@ def g(n): def test_closure_none_is_preserved(self): def f(): - """a function with no closure cells - """ + """A function with no closure cells""" self.assertTrue( f.__closure__ is None, - msg='f actually has closure cells!', + msg="f actually has closure cells!", ) g = pickle_depickle(f, protocol=self.protocol) self.assertTrue( g.__closure__ is None, - msg='g now has closure cells even though f does not', + msg="g now has closure cells even though f does not", ) def test_empty_cell_preserved(self): @@ -341,14 +337,33 @@ def g(): g = pickle_depickle(f(), protocol=self.protocol) self.assertEqual(g(), 2) - def test_dynamically_generated_class_that_uses_super(self): + def test_class_no_firstlineno_deletion_(self): + # `__firstlineno__` is a new attribute of classes introduced in Python 3.13. + # This attribute used to be automatically deleted when unpickling a class as a + # consequence of cloudpickle setting a class's `__module__` attribute at + # unpickling time (see https://github.com/python/cpython/blob/73c152b346a18ed8308e469bdd232698e6cd3a63/Objects/typeobject.c#L1353-L1356). + # This deletion would cause tests like + # `test_deterministic_dynamic_class_attr_ordering_for_chained_pickling` to fail. + # This test makes sure that the attribute `__firstlineno__` is preserved + # across a cloudpickle roundtrip. + + class A: + pass + + if hasattr(A, "__firstlineno__"): + A_roundtrip = pickle_depickle(A, protocol=self.protocol) + assert hasattr(A_roundtrip, "__firstlineno__") + assert A_roundtrip.__firstlineno__ == A.__firstlineno__ + + def test_dynamically_generated_class_that_uses_super(self): class Base: def method(self): return 1 class Derived(Base): "Derived Docstring" + def method(self): return super().method() + 1 @@ -368,9 +383,7 @@ def method(self): self.assertEqual(d.method(), 2) def test_cycle_in_classdict_globals(self): - class C: - def it_works(self): return "woohoo!" @@ -395,13 +408,16 @@ def some_function(x, y): return (x + y) / LOCAL_CONSTANT # pickle the function definition - self.assertEqual(pickle_depickle(some_function, protocol=self.protocol)(41, 1), 1) - self.assertEqual(pickle_depickle(some_function, protocol=self.protocol)(81, 3), 2) + result = pickle_depickle(some_function, protocol=self.protocol)(41, 1) + assert result == 1 + result = pickle_depickle(some_function, protocol=self.protocol)(81, 3) + assert result == 2 - hidden_constant = lambda: LOCAL_CONSTANT + hidden_constant = lambda: LOCAL_CONSTANT # noqa: E731 class SomeClass: """Overly complicated class with nested references to symbols""" + def __init__(self, value): self.value = value @@ -421,15 +437,15 @@ def some_method(self, x): # pickle the class instances self.assertEqual(pickle_depickle(SomeClass(1)).one(), 1) self.assertEqual(pickle_depickle(SomeClass(5)).some_method(41), 7) - new_instance = subprocess_pickle_echo(SomeClass(5), - protocol=self.protocol) + new_instance = subprocess_pickle_echo(SomeClass(5), protocol=self.protocol) self.assertEqual(new_instance.some_method(41), 7) # pickle the method instances self.assertEqual(pickle_depickle(SomeClass(1).one)(), 1) self.assertEqual(pickle_depickle(SomeClass(5).some_method)(41), 7) - new_method = subprocess_pickle_echo(SomeClass(5).some_method, - protocol=self.protocol) + new_method = subprocess_pickle_echo( + SomeClass(5).some_method, protocol=self.protocol + ) self.assertEqual(new_method(41), 7) def test_partial(self): @@ -437,30 +453,29 @@ def test_partial(self): partial_clone = pickle_depickle(partial_obj, protocol=self.protocol) self.assertEqual(partial_clone(4), 1) - @pytest.mark.skipif(platform.python_implementation() == 'PyPy', - reason="Skip numpy and scipy tests on PyPy") + @pytest.mark.skipif( + platform.python_implementation() == "PyPy", + reason="Skip numpy and scipy tests on PyPy", + ) def test_ufunc(self): # test a numpy ufunc (universal function), which is a C-based function # that is applied on a numpy array if np: # simple ufunc: np.add - self.assertEqual(pickle_depickle(np.add, protocol=self.protocol), - np.add) + self.assertEqual(pickle_depickle(np.add, protocol=self.protocol), np.add) else: # skip if numpy is not available pass if spp: # custom ufunc: scipy.special.iv - self.assertEqual(pickle_depickle(spp.iv, protocol=self.protocol), - spp.iv) + self.assertEqual(pickle_depickle(spp.iv, protocol=self.protocol), spp.iv) else: # skip if scipy is not available pass def test_loads_namespace(self): obj = 1, 2, 3, 4 - returned_obj = cloudpickle.loads(cloudpickle.dumps( - obj, protocol=self.protocol)) + returned_obj = cloudpickle.loads(cloudpickle.dumps(obj, protocol=self.protocol)) self.assertEqual(obj, returned_obj) def test_load_namespace(self): @@ -472,14 +487,12 @@ def test_load_namespace(self): self.assertEqual(obj, returned_obj) def test_generator(self): - def some_generator(cnt): - for i in range(cnt): - yield i + yield from range(cnt) gen2 = pickle_depickle(some_generator, protocol=self.protocol) - assert type(gen2(3)) == type(some_generator(3)) + assert isinstance(gen2(3), type(some_generator(3))) assert list(gen2(3)) == list(range(3)) def test_classmethod(self): @@ -487,6 +500,7 @@ class A: @staticmethod def test_sm(): return "sm" + @classmethod def test_cm(cls): return "cm" @@ -511,7 +525,7 @@ def test_cm(cls): def test_method_descriptors(self): f = pickle_depickle(str.upper) - self.assertEqual(f('abc'), 'ABC') + self.assertEqual(f("abc"), "ABC") def test_instancemethods_without_self(self): class F: @@ -526,9 +540,9 @@ def test_module(self): pickle_clone = pickle_depickle(pickle, protocol=self.protocol) self.assertEqual(pickle, pickle_clone) - def test_dynamic_module(self): - mod = types.ModuleType('mod') - code = ''' + def _check_dynamic_module(self, mod): + mod = types.ModuleType("mod") + code = """ x = 1 def f(y): return x + y @@ -536,14 +550,14 @@ def f(y): class Foo: def method(self, x): return f(x) - ''' + """ exec(textwrap.dedent(code), mod.__dict__) mod2 = pickle_depickle(mod, protocol=self.protocol) self.assertEqual(mod.x, mod2.x) self.assertEqual(mod.f(5), mod2.f(5)) self.assertEqual(mod.Foo().method(5), mod2.Foo().method(5)) - if platform.python_implementation() != 'PyPy': + if platform.python_implementation() != "PyPy": # XXX: this fails with excessive recursion on PyPy. mod3 = subprocess_pickle_echo(mod, protocol=self.protocol) self.assertEqual(mod.x, mod3.x) @@ -557,37 +571,49 @@ def method(self, x): # Ensure proper pickling of mod's functions when module "looks" like a # file-backed module even though it is not: try: - sys.modules['mod'] = mod + sys.modules["mod"] = mod depickled_f = pickle_depickle(mod.f, protocol=self.protocol) self.assertEqual(mod.f(5), depickled_f(5)) finally: - sys.modules.pop('mod', None) + sys.modules.pop("mod", None) + + def test_dynamic_module(self): + mod = types.ModuleType("mod") + assert mod.__package__ is None + self._check_dynamic_module(mod) + + def test_dynamic_module_no_package(self): + # non-regression test for #116 + mod = types.ModuleType("mod") + del mod.__package__ + assert not hasattr(mod, "__package__") + self._check_dynamic_module(mod) def test_module_locals_behavior(self): # Makes sure that a local function defined in another module is # correctly serialized. This notably checks that the globals are # accessible and that there is no issue with the builtins (see #211) - pickled_func_path = os.path.join(self.tmpdir, 'local_func_g.pkl') + pickled_func_path = os.path.join(self.tmpdir, "local_func_g.pkl") - child_process_script = ''' - from srsly.cloudpickle.compat import pickle + child_process_script = """ + import pickle import gc with open("{pickled_func_path}", 'rb') as f: func = pickle.load(f) assert func(range(10)) == 45 - ''' + """ child_process_script = child_process_script.format( - pickled_func_path=_escape(pickled_func_path)) + pickled_func_path=_escape(pickled_func_path) + ) try: - from srsly.tests.cloudpickle.testutils import make_local_function g = make_local_function() - with open(pickled_func_path, 'wb') as f: + with open(pickled_func_path, "wb") as f: cloudpickle.dump(g, f, protocol=self.protocol) assert_run_python_script(textwrap.dedent(child_process_script)) @@ -603,13 +629,13 @@ def test_dynamic_module_with_unpicklable_builtin(self): # when pickling dynamic modules. class UnpickleableObject: def __reduce__(self): - raise ValueError('Unpicklable object') + raise ValueError("Unpicklable object") mod = types.ModuleType("mod") - exec('f = lambda x: abs(x)', mod.__dict__) + exec("f = lambda x: abs(x)", mod.__dict__) assert mod.f(-1) == 1 - assert '__builtins__' in mod.__dict__ + assert "__builtins__" in mod.__dict__ unpicklable_obj = UnpickleableObject() with pytest.raises(ValueError): @@ -620,18 +646,17 @@ def __reduce__(self): # The __builtins__ entry of mod's __dict__ can either be the # __builtins__ module, or the __builtins__ module's __dict__. #316 # happens only in the latter case. - if isinstance(mod.__dict__['__builtins__'], dict): - mod.__dict__['__builtins__']['unpickleable_obj'] = unpicklable_obj - elif isinstance(mod.__dict__['__builtins__'], types.ModuleType): - mod.__dict__['__builtins__'].unpickleable_obj = unpicklable_obj + if isinstance(mod.__dict__["__builtins__"], dict): + mod.__dict__["__builtins__"]["unpickleable_obj"] = unpicklable_obj + elif isinstance(mod.__dict__["__builtins__"], types.ModuleType): + mod.__dict__["__builtins__"].unpickleable_obj = unpicklable_obj depickled_mod = pickle_depickle(mod, protocol=self.protocol) - assert '__builtins__' in depickled_mod.__dict__ + assert "__builtins__" in depickled_mod.__dict__ - if isinstance(depickled_mod.__dict__['__builtins__'], dict): + if isinstance(depickled_mod.__dict__["__builtins__"], dict): assert "abs" in depickled_mod.__builtins__ - elif isinstance( - depickled_mod.__dict__['__builtins__'], types.ModuleType): + elif isinstance(depickled_mod.__dict__["__builtins__"], types.ModuleType): assert hasattr(depickled_mod.__builtins__, "abs") assert depickled_mod.f(-1) == 1 @@ -647,10 +672,10 @@ def test_load_dynamic_module_in_grandchild_process(self): # the child process and reloaded in another one. # We create a new dynamic module - mod = types.ModuleType('mod') - code = ''' + mod = types.ModuleType("mod") + code = """ x = 1 - ''' + """ exec(textwrap.dedent(code), mod.__dict__) # This script will be ran in a separate child process. It will import @@ -658,11 +683,13 @@ def test_load_dynamic_module_in_grandchild_process(self): # Finally, it will create a child process that will load the re-pickled # dynamic module. parent_process_module_file = os.path.join( - self.tmpdir, 'dynamic_module_from_parent_process.pkl') + self.tmpdir, "dynamic_module_from_parent_process.pkl" + ) child_process_module_file = os.path.join( - self.tmpdir, 'dynamic_module_from_child_process.pkl') - child_process_script = ''' - from srsly.cloudpickle.compat import pickle + self.tmpdir, "dynamic_module_from_child_process.pkl" + ) + child_process_script = """ + import pickle import textwrap import srsly.cloudpickle as cloudpickle @@ -678,11 +705,11 @@ def test_load_dynamic_module_in_grandchild_process(self): cloudpickle.dump(mod, f, protocol={protocol}) assert_run_python_script(textwrap.dedent(child_of_child_process_script)) - ''' + """ # The script ran by the process created by the child process child_of_child_process_script = """ ''' - from srsly.cloudpickle.compat import pickle + import pickle with open('{child_process_module_file}','rb') as fid: mod = pickle.load(fid) ''' """ @@ -691,16 +718,18 @@ def test_load_dynamic_module_in_grandchild_process(self): # for the first child process, the script to be executed by its # own child process. child_of_child_process_script = child_of_child_process_script.format( - child_process_module_file=child_process_module_file) + child_process_module_file=child_process_module_file + ) child_process_script = child_process_script.format( parent_process_module_file=_escape(parent_process_module_file), child_process_module_file=_escape(child_process_module_file), child_of_child_process_script=_escape(child_of_child_process_script), - protocol=self.protocol) + protocol=self.protocol, + ) try: - with open(parent_process_module_file, 'wb') as fid: + with open(parent_process_module_file, "wb") as fid: cloudpickle.dump(mod, fid, protocol=self.protocol) assert_run_python_script(textwrap.dedent(child_process_script)) @@ -730,15 +759,14 @@ def my_small_function(x, y): # module are not included so as to keep the pickle payload as # lightweight as possible. - assert b'my_small_function' in b - assert b'nested_function' in b + assert b"my_small_function" in b + assert b"nested_function" in b - assert b'unwanted_function' not in b - assert b'math' not in b + assert b"unwanted_function" not in b + assert b"math" not in b def test_module_importability(self): - pytest.importorskip("_cloudpickle_testpkg") - from srsly.cloudpickle.compat import pickle + import pickle import os.path import collections import collections.abc @@ -748,26 +776,28 @@ def test_module_importability(self): assert _should_pickle_by_reference(collections) # package assert _should_pickle_by_reference(collections.abc) # module in package - dynamic_module = types.ModuleType('dynamic_module') + dynamic_module = types.ModuleType("dynamic_module") assert not _should_pickle_by_reference(dynamic_module) - if platform.python_implementation() == 'PyPy': + if platform.python_implementation() == "PyPy": import _codecs + assert _should_pickle_by_reference(_codecs) # #354: Check that modules created dynamically during the import of # their parent modules are considered importable by cloudpickle. # See the mod_with_dynamic_submodule documentation for more # details of this use case. - import _cloudpickle_testpkg.mod.dynamic_submodule as m + m = pytest.importorskip( + "_cloudpickle_testpkg.mod.dynamic_submodule" + ) # noqa F841 assert _should_pickle_by_reference(m) assert pickle_depickle(m, protocol=self.protocol) is m # Check for similar behavior for a module that cannot be imported by # attribute lookup. from _cloudpickle_testpkg.mod import dynamic_submodule_two as m2 - # Note: import _cloudpickle_testpkg.mod.dynamic_submodule_two as m2 - # works only for Python 3.7+ + assert _should_pickle_by_reference(m2) assert pickle_depickle(m2, protocol=self.protocol) is m2 @@ -775,11 +805,12 @@ def test_module_importability(self): with pytest.raises(ImportError): import _cloudpickle_testpkg.mod.submodule_three # noqa from _cloudpickle_testpkg.mod import submodule_three as m3 + assert not _should_pickle_by_reference(m3) # This module cannot be pickled using attribute lookup (as it does not # have a `__module__` attribute like classes and functions. - assert not hasattr(m3, '__module__') + assert not hasattr(m3, "__module__") depickled_m3 = pickle_depickle(m3, protocol=self.protocol) assert depickled_m3 is not m3 assert m3.f(1) == depickled_m3.f(1) @@ -787,6 +818,7 @@ def test_module_importability(self): # Do the same for an importable dynamic submodule inside a dynamic # module inside a file-backed module. import _cloudpickle_testpkg.mod.dynamic_submodule.dynamic_subsubmodule as sm # noqa + assert _should_pickle_by_reference(sm) assert pickle_depickle(sm, protocol=self.protocol) is sm @@ -795,8 +827,7 @@ def test_module_importability(self): _should_pickle_by_reference(object()) def test_Ellipsis(self): - self.assertEqual(Ellipsis, - pickle_depickle(Ellipsis, protocol=self.protocol)) + self.assertEqual(Ellipsis, pickle_depickle(Ellipsis, protocol=self.protocol)) def test_NotImplemented(self): ExcClone = pickle_depickle(NotImplemented, protocol=self.protocol) @@ -822,6 +853,7 @@ def test_builtin_function(self): assert pickle_depickle(zip, protocol=self.protocol) is zip from os import mkdir + # builtin function from a "regular" module assert pickle_depickle(mkdir, protocol=self.protocol) is mkdir @@ -851,16 +883,19 @@ def test_builtin_classicmethod(self): bound_classicmethod = obj.hex # builtin_function_or_method unbound_classicmethod = type(obj).hex # method_descriptor - clsdict_classicmethod = type(obj).__dict__['hex'] # method_descriptor + clsdict_classicmethod = type(obj).__dict__["hex"] # method_descriptor assert unbound_classicmethod is clsdict_classicmethod depickled_bound_meth = pickle_depickle( - bound_classicmethod, protocol=self.protocol) + bound_classicmethod, protocol=self.protocol + ) depickled_unbound_meth = pickle_depickle( - unbound_classicmethod, protocol=self.protocol) + unbound_classicmethod, protocol=self.protocol + ) depickled_clsdict_meth = pickle_depickle( - clsdict_classicmethod, protocol=self.protocol) + clsdict_classicmethod, protocol=self.protocol + ) # No identity on the bound methods they are bound to different float # instances @@ -868,28 +903,16 @@ def test_builtin_classicmethod(self): assert depickled_unbound_meth is unbound_classicmethod assert depickled_clsdict_meth is clsdict_classicmethod - - @pytest.mark.skipif( - (platform.machine() == "aarch64" and sys.version_info[:2] >= (3, 10)) - or platform.python_implementation() == "PyPy" - or (sys.version_info[:2] == (3, 10) and sys.version_info >= (3, 10, 8)) - # Skipping tests on 3.11 due to https://github.com/cloudpipe/cloudpickle/pull/486. - or sys.version_info[:2] >= (3, 11), - reason="Fails on aarch64 + python 3.10+ in cibuildwheel, currently unable to replicate failure elsewhere; fails sometimes for pypy on conda-forge; fails for python 3.10.8+ and 3.11+") def test_builtin_classmethod(self): obj = 1.5 # float object bound_clsmethod = obj.fromhex # builtin_function_or_method unbound_clsmethod = type(obj).fromhex # builtin_function_or_method - clsdict_clsmethod = type( - obj).__dict__['fromhex'] # classmethod_descriptor - depickled_bound_meth = pickle_depickle( - bound_clsmethod, protocol=self.protocol) + depickled_bound_meth = pickle_depickle(bound_clsmethod, protocol=self.protocol) depickled_unbound_meth = pickle_depickle( - unbound_clsmethod, protocol=self.protocol) - depickled_clsdict_meth = pickle_depickle( - clsdict_clsmethod, protocol=self.protocol) + unbound_clsmethod, protocol=self.protocol + ) # float.fromhex takes a string as input. arg = "0x1" @@ -900,33 +923,67 @@ def test_builtin_classmethod(self): assert depickled_bound_meth(arg) == bound_clsmethod(arg) assert depickled_unbound_meth(arg) == unbound_clsmethod(arg) - if platform.python_implementation() == 'CPython': + @pytest.mark.skipif( + ( + sys.version_info >= (3, 10, 8) + and platform.python_implementation() == "CPython" + ), + reason=( + "CPython dropped support for pickling classmethod_descriptor," + "https://github.com/python/cpython/issues/95196" + ), + ) + def test_builtin_classmethod_descriptor(self): + # `classmethod_descriptor` is the analogue `classmethod` (used for + # pure Python classes) for builtin types. Until CPython 3.10.8, + # `classmethod_descriptor` implemented an (incorrect) reducer. After + # https://github.com/python/cpython/issues/95196 revealed its + # incorrectness, this reducer was dropped (and not fixed), on the + # ground that pickling its Pythonic equivalent, `classmethod`, + # was never supported in the first place. + # Note that cloudpickle supports pickling `classmethod` objects, + # but never patched pickle's incorrect `classmethod_descriptor` + # reducer: pickling `classmethod_descriptor` objects using cloudpickle + # has always been broken. + obj = 1.5 # float object + + clsdict_clsmethod = type(obj).__dict__["fromhex"] # classmethod_descriptor + + depickled_clsdict_meth = pickle_depickle( + clsdict_clsmethod, protocol=self.protocol + ) + + # float.fromhex takes a string as input. + arg = "0x1" + if platform.python_implementation() == "CPython": # Roundtripping a classmethod_descriptor results in a # builtin_function_or_method (CPython upstream issue). assert depickled_clsdict_meth(arg) == clsdict_clsmethod(float, arg) - if platform.python_implementation() == 'PyPy': + if platform.python_implementation() == "PyPy": # builtin-classmethods are simple classmethod in PyPy (not # callable). We test equality of types and the functionality of the # __func__ attribute instead. We do not test the the identity of # the functions as __func__ attributes of classmethods are not # pickleable and must be reconstructed at depickling time. - assert type(depickled_clsdict_meth) == type(clsdict_clsmethod) + assert type(depickled_clsdict_meth) is type(clsdict_clsmethod) assert depickled_clsdict_meth.__func__( - float, arg) == clsdict_clsmethod.__func__(float, arg) + float, arg + ) == clsdict_clsmethod.__func__(float, arg) def test_builtin_slotmethod(self): obj = 1.5 # float object bound_slotmethod = obj.__repr__ # method-wrapper unbound_slotmethod = type(obj).__repr__ # wrapper_descriptor - clsdict_slotmethod = type(obj).__dict__['__repr__'] # ditto + clsdict_slotmethod = type(obj).__dict__["__repr__"] # ditto - depickled_bound_meth = pickle_depickle( - bound_slotmethod, protocol=self.protocol) + depickled_bound_meth = pickle_depickle(bound_slotmethod, protocol=self.protocol) depickled_unbound_meth = pickle_depickle( - unbound_slotmethod, protocol=self.protocol) + unbound_slotmethod, protocol=self.protocol + ) depickled_clsdict_meth = pickle_depickle( - clsdict_slotmethod, protocol=self.protocol) + clsdict_slotmethod, protocol=self.protocol + ) # No identity tests on the bound slotmethod are they are bound to # different float instances @@ -936,22 +993,26 @@ def test_builtin_slotmethod(self): @pytest.mark.skipif( platform.python_implementation() == "PyPy", - reason="No known staticmethod example in the pypy stdlib") + reason="No known staticmethod example in the pypy stdlib", + ) def test_builtin_staticmethod(self): obj = "foo" # str object bound_staticmethod = obj.maketrans # builtin_function_or_method unbound_staticmethod = type(obj).maketrans # ditto - clsdict_staticmethod = type(obj).__dict__['maketrans'] # staticmethod + clsdict_staticmethod = type(obj).__dict__["maketrans"] # staticmethod assert bound_staticmethod is unbound_staticmethod depickled_bound_meth = pickle_depickle( - bound_staticmethod, protocol=self.protocol) + bound_staticmethod, protocol=self.protocol + ) depickled_unbound_meth = pickle_depickle( - unbound_staticmethod, protocol=self.protocol) + unbound_staticmethod, protocol=self.protocol + ) depickled_clsdict_meth = pickle_depickle( - clsdict_staticmethod, protocol=self.protocol) + clsdict_staticmethod, protocol=self.protocol + ) assert depickled_bound_meth is bound_staticmethod assert depickled_unbound_meth is unbound_staticmethod @@ -961,11 +1022,10 @@ def test_builtin_staticmethod(self): assert depickled_clsdict_meth.__func__ is clsdict_staticmethod.__func__ type(depickled_clsdict_meth) is type(clsdict_staticmethod) - @pytest.mark.skipif(tornado is None, - reason="test needs Tornado installed") def test_tornado_coroutine(self): # Pickling a locally defined coroutine function - from tornado import gen, ioloop + gen = pytest.importorskip("tornado.gen") + ioloop = pytest.importorskip("tornado.ioloop") @gen.coroutine def f(x, y): @@ -974,26 +1034,29 @@ def f(x, y): @gen.coroutine def g(y): - res = yield f(0.01, y) + res = yield f(0.01, y) # noqa: F821 raise gen.Return(res + 1) + with pytest.warns(DeprecationWarning): + assert cloudpickle.is_tornado_coroutine(g) + data = cloudpickle.dumps([g, g], protocol=self.protocol) - f = g = None + del f, g g2, g3 = pickle.loads(data) - self.assertTrue(g2 is g3) - loop = ioloop.IOLoop.current() + assert g2 is g3 + loop = ioloop.IOLoop(make_current=False) res = loop.run_sync(functools.partial(g2, 5)) - self.assertEqual(res, 7) + assert res == 7 @pytest.mark.skipif( - (3, 11, 0, 'beta') <= sys.version_info < (3, 11, 0, 'beta', 4), - reason="https://github.com/python/cpython/issues/92932" + (3, 11, 0, "beta") <= sys.version_info < (3, 11, 0, "beta", 4), + reason="https://github.com/python/cpython/issues/92932", ) def test_extended_arg(self): # Functions with more than 65535 global vars prefix some global # variable references with the EXTENDED_ARG opcode. nvars = 65537 + 258 - names = ['g%d' % i for i in range(1, nvars)] + names = ["g%d" % i for i in range(1, nvars)] r = random.Random(42) d = {name: r.randrange(100) for name in names} # def f(x): @@ -1005,9 +1068,9 @@ def test_extended_arg(self): def f(): x = {tup} return zlib.crc32(bytes(bytearray(x))) - """.format(tup=', '.join(names)) + """.format(tup=", ".join(names)) exec(textwrap.dedent(code), d, d) - f = d['f'] + f = d["f"] res = f() data = cloudpickle.dumps([f, f], protocol=self.protocol) d = f = None @@ -1020,90 +1083,105 @@ def test_submodule(self): # Choose any module NOT imported by __init__ of its parent package # examples in standard library include: - # - http.cookies, unittest.mock, curses.textpad, xml.etree.ElementTree - - global xml # imitate performing this import at top of file + # http.cookies, unittest.mock, curses.textpad, xml.etree.ElementTree + import xml import xml.etree.ElementTree + def example(): - x = xml.etree.ElementTree.Comment # potential AttributeError + _ = xml.etree.ElementTree.Comment # noqa: F821 + + example() # smoke test s = cloudpickle.dumps(example, protocol=self.protocol) # refresh the environment, i.e., unimport the dependency del xml for item in list(sys.modules): - if item.split('.')[0] == 'xml': + if item.split(".")[0] == "xml": del sys.modules[item] # deserialise f = pickle.loads(s) - f() # perform test for error + f() # smoke test def test_submodule_closure(self): - # Same as test_submodule except the package is not a global + # Same as test_submodule except the xml package has not been imported def scope(): import xml.etree.ElementTree + def example(): - x = xml.etree.ElementTree.Comment # potential AttributeError + _ = xml.etree.ElementTree.Comment # potential AttributeError + return example + example = scope() + example() # smoke test s = cloudpickle.dumps(example, protocol=self.protocol) # refresh the environment (unimport dependency) for item in list(sys.modules): - if item.split('.')[0] == 'xml': + if item.split(".")[0] == "xml": del sys.modules[item] f = cloudpickle.loads(s) - f() # test + f() # smoke test def test_multiprocess(self): # running a function pickled by another process (a la dask.distributed) def scope(): def example(): - x = xml.etree.ElementTree.Comment + _ = xml.etree.ElementTree.Comment + return example + global xml import xml.etree.ElementTree + example = scope() s = cloudpickle.dumps(example, protocol=self.protocol) # choose "subprocess" rather than "multiprocessing" because the latter # library uses fork to preserve the parent environment. - command = ("import base64; " - "from srsly.cloudpickle.compat import pickle; " - "pickle.loads(base64.b32decode('" + - base64.b32encode(s).decode('ascii') + - "'))()") - assert not subprocess.call([sys.executable, '-c', command]) + command = ( + "import base64; import pickle; pickle.loads(base64.b32decode('" + + base64.b32encode(s).decode("ascii") + + "'))()" + ) + assert not subprocess.call([sys.executable, "-c", command]) def test_import(self): # like test_multiprocess except subpackage modules referenced directly # (unlike test_submodule) global etree + def scope(): import xml.etree as foobar + def example(): - x = etree.Comment - x = foobar.ElementTree + _ = etree.Comment + _ = foobar.ElementTree + return example + example = scope() import xml.etree.ElementTree as etree s = cloudpickle.dumps(example, protocol=self.protocol) - command = ("import base64; " - "from srsly.cloudpickle.compat import pickle; " - "pickle.loads(base64.b32decode('" + - base64.b32encode(s).decode('ascii') + - "'))()") - assert not subprocess.call([sys.executable, '-c', command]) + command = ( + "import base64; from pickle import loads; loads(base64.b32decode('" + + base64.b32encode(s).decode("ascii") + + "'))()" + ) + assert not subprocess.call([sys.executable, "-c", command]) def test_multiprocessing_lock_raises(self): lock = multiprocessing.Lock() - with pytest.raises(RuntimeError, match="only be shared between processes through inheritance"): + with pytest.raises( + RuntimeError, match="only be shared between processes through inheritance" + ): cloudpickle.dumps(lock) def test_cell_manipulation(self): @@ -1113,11 +1191,8 @@ def test_cell_manipulation(self): cell.cell_contents ob = object() - cell_set(cell, ob) - self.assertTrue( - cell.cell_contents is ob, - msg='cell contents not set correctly', - ) + cell.cell_contents = ob + assert cell.cell_contents is ob def check_logger(self, name): logger = logging.getLogger(name) @@ -1132,20 +1207,21 @@ def check_logger(self, name): logging.basicConfig(level=logging.INFO) logger = cloudpickle.loads(base64.b32decode(b'{}')) logger.info('hello') - """.format(base64.b32encode(dumped).decode('ascii')) - proc = subprocess.Popen([sys.executable, "-W ignore", "-c", code], - stdout=subprocess.PIPE, - stderr=subprocess.STDOUT) + """.format(base64.b32encode(dumped).decode("ascii")) + proc = subprocess.Popen( + [sys.executable, "-W ignore", "-c", code], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) out, _ = proc.communicate() self.assertEqual(proc.wait(), 0) - self.assertEqual(out.strip().decode(), - f'INFO:{logger.name}:hello') + self.assertEqual(out.strip().decode(), f"INFO:{logger.name}:hello") def test_logger(self): # logging.RootLogger object self.check_logger(None) # logging.Logger object - self.check_logger('cloudpickle.dummy_test_logger') + self.check_logger("cloudpickle.dummy_test_logger") def test_getset_descriptor(self): assert isinstance(float.real, types.GetSetDescriptorType) @@ -1156,7 +1232,7 @@ def test_abc_cache_not_pickled(self): # cloudpickle issue #302: make sure that cloudpickle does not pickle # the caches populated during instance/subclass checks of abc.ABCMeta # instances. - MyClass = abc.ABCMeta('MyClass', (), {}) + MyClass = abc.ABCMeta("MyClass", (), {}) class MyUnrelatedClass: pass @@ -1179,7 +1255,6 @@ class MyRelatedClass: assert issubclass(MyRelatedClass, depickled_class) def test_abc(self): - class AbstractClass(abc.ABC): @abc.abstractmethod def some_method(self): @@ -1202,20 +1277,20 @@ def some_property(): class ConcreteClass(AbstractClass): def some_method(self): - return 'it works!' + return "it works!" @classmethod def some_classmethod(cls): assert cls == ConcreteClass - return 'it works!' + return "it works!" @staticmethod def some_staticmethod(): - return 'it works!' + return "it works!" @property def some_property(self): - return 'it works!' + return "it works!" # This abstract class is locally defined so we can safely register # tuple in it to verify the unpickled class also register tuple. @@ -1223,49 +1298,47 @@ def some_property(self): concrete_instance = ConcreteClass() depickled_base = pickle_depickle(AbstractClass, protocol=self.protocol) - depickled_class = pickle_depickle(ConcreteClass, - protocol=self.protocol) + depickled_class = pickle_depickle(ConcreteClass, protocol=self.protocol) depickled_instance = pickle_depickle(concrete_instance) assert issubclass(tuple, AbstractClass) assert issubclass(tuple, depickled_base) - self.assertEqual(depickled_class().some_method(), 'it works!') - self.assertEqual(depickled_instance.some_method(), 'it works!') + self.assertEqual(depickled_class().some_method(), "it works!") + self.assertEqual(depickled_instance.some_method(), "it works!") - self.assertEqual(depickled_class.some_classmethod(), 'it works!') - self.assertEqual(depickled_instance.some_classmethod(), 'it works!') + self.assertEqual(depickled_class.some_classmethod(), "it works!") + self.assertEqual(depickled_instance.some_classmethod(), "it works!") - self.assertEqual(depickled_class().some_staticmethod(), 'it works!') - self.assertEqual(depickled_instance.some_staticmethod(), 'it works!') + self.assertEqual(depickled_class().some_staticmethod(), "it works!") + self.assertEqual(depickled_instance.some_staticmethod(), "it works!") - self.assertEqual(depickled_class().some_property, 'it works!') - self.assertEqual(depickled_instance.some_property, 'it works!') + self.assertEqual(depickled_class().some_property, "it works!") + self.assertEqual(depickled_instance.some_property, "it works!") self.assertRaises(TypeError, depickled_base) class DepickledBaseSubclass(depickled_base): def some_method(self): - return 'it works for realz!' + return "it works for realz!" @classmethod def some_classmethod(cls): assert cls == DepickledBaseSubclass - return 'it works for realz!' + return "it works for realz!" @staticmethod def some_staticmethod(): - return 'it works for realz!' + return "it works for realz!" @property def some_property(): - return 'it works for realz!' + return "it works for realz!" - self.assertEqual(DepickledBaseSubclass().some_method(), - 'it works for realz!') + self.assertEqual(DepickledBaseSubclass().some_method(), "it works for realz!") class IncompleteBaseSubclass(depickled_base): def some_method(self): - return 'this class lacks some concrete methods' + return "this class lacks some concrete methods" self.assertRaises(TypeError, IncompleteBaseSubclass) @@ -1292,20 +1365,20 @@ def some_property(self): class ConcreteClass(AbstractClass): def some_method(self): - return 'it works!' + return "it works!" @classmethod def some_classmethod(cls): assert cls == ConcreteClass - return 'it works!' + return "it works!" @staticmethod def some_staticmethod(): - return 'it works!' + return "it works!" @property def some_property(self): - return 'it works!' + return "it works!" # This abstract class is locally defined so we can safely register # tuple in it to verify the unpickled class also register tuple. @@ -1313,49 +1386,47 @@ def some_property(self): concrete_instance = ConcreteClass() depickled_base = pickle_depickle(AbstractClass, protocol=self.protocol) - depickled_class = pickle_depickle(ConcreteClass, - protocol=self.protocol) + depickled_class = pickle_depickle(ConcreteClass, protocol=self.protocol) depickled_instance = pickle_depickle(concrete_instance) assert issubclass(tuple, AbstractClass) assert issubclass(tuple, depickled_base) - self.assertEqual(depickled_class().some_method(), 'it works!') - self.assertEqual(depickled_instance.some_method(), 'it works!') + self.assertEqual(depickled_class().some_method(), "it works!") + self.assertEqual(depickled_instance.some_method(), "it works!") - self.assertEqual(depickled_class.some_classmethod(), 'it works!') - self.assertEqual(depickled_instance.some_classmethod(), 'it works!') + self.assertEqual(depickled_class.some_classmethod(), "it works!") + self.assertEqual(depickled_instance.some_classmethod(), "it works!") - self.assertEqual(depickled_class().some_staticmethod(), 'it works!') - self.assertEqual(depickled_instance.some_staticmethod(), 'it works!') + self.assertEqual(depickled_class().some_staticmethod(), "it works!") + self.assertEqual(depickled_instance.some_staticmethod(), "it works!") - self.assertEqual(depickled_class().some_property, 'it works!') - self.assertEqual(depickled_instance.some_property, 'it works!') + self.assertEqual(depickled_class().some_property, "it works!") + self.assertEqual(depickled_instance.some_property, "it works!") self.assertRaises(TypeError, depickled_base) class DepickledBaseSubclass(depickled_base): def some_method(self): - return 'it works for realz!' + return "it works for realz!" @classmethod def some_classmethod(cls): assert cls == DepickledBaseSubclass - return 'it works for realz!' + return "it works for realz!" @staticmethod def some_staticmethod(): - return 'it works for realz!' + return "it works for realz!" @property def some_property(self): - return 'it works for realz!' + return "it works for realz!" - self.assertEqual(DepickledBaseSubclass().some_method(), - 'it works for realz!') + self.assertEqual(DepickledBaseSubclass().some_method(), "it works for realz!") class IncompleteBaseSubclass(depickled_base): def some_method(self): - return 'this class lacks some concrete methods' + return "this class lacks some concrete methods" self.assertRaises(TypeError, IncompleteBaseSubclass) @@ -1390,7 +1461,7 @@ def test_non_module_object_passing_whichmodule_test(self): # carried out on the entries of sys.modules, causing cloupdickle to # then error in unexpected ways def func(x): - return x ** 2 + return x**2 # Trigger a loop during the execution of whichmodule(func) by # explicitly setting the function's module to None @@ -1406,7 +1477,7 @@ def __getattr__(self, name): # entries of sys.modules is not carried out, but manipulating # this instance thinking it really is a module later on in the # pickling process of func errors out - if name == 'func': + if name == "func": return func else: raise AttributeError @@ -1422,17 +1493,44 @@ def __getattr__(self, name): _ = non_module_object.some_attr try: - sys.modules['NonModuleObject'] = non_module_object + sys.modules["NonModuleObject"] = non_module_object - func_module_name = _whichmodule(func, None) - assert func_module_name != 'NonModuleObject' + func_module_name = _whichmodule(func, "func") + assert func_module_name != "NonModuleObject" assert func_module_name is None depickled_func = pickle_depickle(func, protocol=self.protocol) assert depickled_func(2) == 4 finally: - sys.modules.pop('NonModuleObject') + sys.modules.pop("NonModuleObject") + + def test_importing_multiprocessing_does_not_impact_whichmodule(self): + # non-regression test for #528 + script = textwrap.dedent(""" + import multiprocessing + import srsly.cloudpickle as cloudpickle + from srsly.cloudpickle.cloudpickle import dumps + + # Trigger a loop during the execution of whichmodule() by + # explicitly setting the function's module to None + dumps.__module__ = None + + print(cloudpickle.cloudpickle._whichmodule(dumps, dumps.__name__)) + """) + script_path = Path(self.tmpdir) / "whichmodule_and_multiprocessing.py" + with open(script_path, mode="w") as f: + f.write(script) + + proc = subprocess.Popen( + [sys.executable, str(script_path)], + stdout=subprocess.PIPE, + stderr=subprocess.STDOUT, + ) + out, _ = proc.communicate() + self.assertEqual(proc.wait(), 0, msg="Stdout: " + str(out)) + self.assertEqual(out.strip(), b"srsly.cloudpickle.cloudpickle") + def test_unrelated_faulty_module(self): # Check that pickling a dynamically defined function or class does not @@ -1440,7 +1538,8 @@ def test_unrelated_faulty_module(self): # as long as those faulty modules are unrelated to the class or # function we are currently pickling. for base_class in (object, types.ModuleType): - for module_name in ['_missing_module', None]: + for module_name in ["_missing_module", None]: + class FaultyModule(base_class): def __getattr__(self, name): # This throws an exception while looking up within @@ -1459,7 +1558,7 @@ def foo(): foo.__module__ = module_name if base_class is types.ModuleType: # noqa - faulty_module = FaultyModule('_faulty_module') + faulty_module = FaultyModule("_faulty_module") else: faulty_module = FaultyModule() sys.modules["_faulty_module"] = faulty_module @@ -1474,41 +1573,25 @@ def foo(): finally: sys.modules.pop("_faulty_module", None) - @pytest.mark.skip(reason="fails for pytest v7.2.0") - def test_dynamic_pytest_module(self): - # Test case for pull request https://github.com/cloudpipe/cloudpickle/pull/116 - import py - - def f(): - s = py.builtin.set([1]) - return s.pop() - - # some setup is required to allow pytest apimodules to be correctly - # serializable. - from srsly.cloudpickle import CloudPickler - from srsly.cloudpickle import cloudpickle_fast as cp_fast - CloudPickler.dispatch_table[type(py.builtin)] = cp_fast._module_reduce - - g = cloudpickle.loads(cloudpickle.dumps(f, protocol=self.protocol)) - - result = g() - self.assertEqual(1, result) - def test_function_module_name(self): - func = lambda x: x - cloned = pickle_depickle(func, protocol=self.protocol) - self.assertEqual(cloned.__module__, func.__module__) + def local_func(x): + return x + + for func in [local_func, lambda x: x]: + cloned = pickle_depickle(func, protocol=self.protocol) + self.assertEqual(cloned.__module__, func.__module__) def test_function_qualname(self): def func(x): return x + # Default __qualname__ attribute (Python 3 only) - if hasattr(func, '__qualname__'): + if hasattr(func, "__qualname__"): cloned = pickle_depickle(func, protocol=self.protocol) self.assertEqual(cloned.__qualname__, func.__qualname__) # Mutated __qualname__ attribute - func.__qualname__ = '' + func.__qualname__ = "" cloned = pickle_depickle(func, protocol=self.protocol) self.assertEqual(cloned.__qualname__, func.__qualname__) @@ -1532,8 +1615,6 @@ def read_write_value(self): def read_write_value(self, value): self._read_write_value = value - - my_object = MyObject() assert my_object.read_only_value == 1 @@ -1557,14 +1638,33 @@ def read_write_value(self, value): assert depickled_obj.read_write_value == 3 type(depickled_obj).read_only_value.__doc__ == "A read-only attribute" - def test_namedtuple(self): - MyTuple = collections.namedtuple('MyTuple', ['a', 'b', 'c']) + MyTuple = collections.namedtuple("MyTuple", ["a", "b", "c"]) + t1 = MyTuple(1, 2, 3) + t2 = MyTuple(3, 2, 1) + + depickled_t1, depickled_MyTuple, depickled_t2 = pickle_depickle( + [t1, MyTuple, t2], protocol=self.protocol + ) + + assert isinstance(depickled_t1, MyTuple) + assert depickled_t1 == t1 + assert depickled_MyTuple is MyTuple + assert isinstance(depickled_t2, MyTuple) + assert depickled_t2 == t2 + + def test_NamedTuple(self): + class MyTuple(typing.NamedTuple): + a: int + b: int + c: int + t1 = MyTuple(1, 2, 3) t2 = MyTuple(3, 2, 1) depickled_t1, depickled_MyTuple, depickled_t2 = pickle_depickle( - [t1, MyTuple, t2], protocol=self.protocol) + [t1, MyTuple, t2], protocol=self.protocol + ) assert isinstance(depickled_t1, MyTuple) assert depickled_t1 == t1 @@ -1572,8 +1672,6 @@ def test_namedtuple(self): assert isinstance(depickled_t2, MyTuple) assert depickled_t2 == t2 - @pytest.mark.skipif(platform.python_implementation() == "PyPy", - reason="fails sometimes for pypy on conda-forge") def test_interactively_defined_function(self): # Check that callables defined in the __main__ module of a Python # script (or jupyter kernel) can be pickled / unpickled / executed. @@ -1695,9 +1793,8 @@ def f1(): new_global_var = new_cloned_f1() assert new_global_var == "default_value", new_global_var """ - for clone_func in ['local_clone', 'subprocess_pickle_echo']: - code = code_template.format(protocol=self.protocol, - clone_func=clone_func) + for clone_func in ["local_clone", "subprocess_pickle_echo"]: + code = code_template.format(protocol=self.protocol, clone_func=clone_func) assert_run_python_script(textwrap.dedent(code)) def test_closure_interacting_with_a_global_variable(self): @@ -1705,6 +1802,7 @@ def test_closure_interacting_with_a_global_variable(self): assert _TEST_GLOBAL_VARIABLE == "default_value" orig_value = _TEST_GLOBAL_VARIABLE try: + def f0(): global _TEST_GLOBAL_VARIABLE _TEST_GLOBAL_VARIABLE = "changed_by_f0" @@ -1713,8 +1811,7 @@ def f1(): return _TEST_GLOBAL_VARIABLE # pickle f0 and f1 inside the same pickle_string - cloned_f0, cloned_f1 = pickle_depickle([f0, f1], - protocol=self.protocol) + cloned_f0, cloned_f1 = pickle_depickle([f0, f1], protocol=self.protocol) # cloned_f0 and cloned_f1 now share a global namespace that is # isolated from any previously existing namespace @@ -1908,7 +2005,6 @@ def lookup(obj_id): class A: '''Updated class definition''' - pass assert not w.run(lambda obj_id: isinstance(lookup(obj_id), A), id1) retrieved1 = w.run(lookup, id1) @@ -1940,7 +2036,152 @@ class A: """.format(protocol=self.protocol) assert_run_python_script(code) - @pytest.mark.skip(reason="Seems to have issues outside of linux and CPython") + def test_dynamic_func_deterministic_roundtrip(self): + # Check that the pickle serialization for a dynamic func is the same + # in two processes. + + def get_dynamic_func_pickle(): + def test_method(arg_1, arg_2): + pass + + return cloudpickle.dumps(test_method) + + with subprocess_worker(protocol=self.protocol) as w: + A_dump = w.run(get_dynamic_func_pickle) + check_deterministic_pickle(A_dump, get_dynamic_func_pickle()) + + def test_dynamic_class_deterministic_roundtrip(self): + # Check that the pickle serialization for a dynamic class is the same + # in two processes. + pytest.xfail("This test fails due to different tracker_id.") + + def get_dynamic_class_pickle(): + class A: + """Class with potential string interning issues.""" + + arg_1 = "class_value" + + def join(self): + pass + + def test_method(self, arg_1, join): + pass + + return cloudpickle.dumps(A) + + with subprocess_worker(protocol=self.protocol) as w: + A_dump = w.run(get_dynamic_class_pickle) + check_deterministic_pickle(A_dump, get_dynamic_class_pickle()) + + def test_deterministic_dynamic_class_attr_ordering_for_chained_pickling(self): + # Check that the pickle produced by pickling a reconstructed class definition + # in a remote process matches the pickle produced by pickling the original + # class definition. + # In particular, this test checks that the order of the class attributes is + # deterministic. + + with subprocess_worker(protocol=self.protocol) as w: + + class A: + """Simple class definition""" + + pass + + A_dump = w.run(cloudpickle.dumps, A) + check_deterministic_pickle(A_dump, cloudpickle.dumps(A)) + + # If the `__doc__` attribute is defined after some other class + # attribute, this can cause class attribute ordering changes due to + # the way we reconstruct the class definition in + # `_make_skeleton_class`, which creates the class and thus its + # `__doc__` attribute before populating the class attributes. + class A: + name = "A" + __doc__ = "Updated class definition" + + A_dump = w.run(cloudpickle.dumps, A) + check_deterministic_pickle(A_dump, cloudpickle.dumps(A)) + + # If a `__doc__` is defined on the `__init__` method, this can + # cause ordering changes due to the way we reconstruct the class + # with `_make_skeleton_class`. + class A: + def __init__(self): + """Class definition with explicit __init__""" + pass + + A_dump = w.run(cloudpickle.dumps, A) + check_deterministic_pickle(A_dump, cloudpickle.dumps(A)) + + def test_deterministic_str_interning_for_chained_dynamic_class_pickling(self): + # Check that the pickle produced by the unpickled instance is the same. + # This checks that there is no issue related to the string interning of + # the names of attributes of class definitions and names of attributes + # of the `__code__` objects of the methods. + + with subprocess_worker(protocol=self.protocol) as w: + # Due to interning of class attributes, check that this does not + # create issues with dynamic function definition. + class A: + """Class with potential string interning issues.""" + + arg_1 = "class_value" + + def join(self): + pass + + def test_method(self, arg_1, join): + pass + + A_dump = w.run(cloudpickle.dumps, A) + check_deterministic_pickle(A_dump, cloudpickle.dumps(A)) + + # Also check that memoization of string value inside the class does + # not cause non-deterministic pickle with interned method names. + class A: + """Class with potential string interning issues.""" + + arg_1 = "join" + + def join(self, arg_1): + pass + + # Set a custom method attribute that can potentially trigger + # undeterministic memoization depending on the interning state of + # the string used for the attribute name. + A.join.arg_1 = "join" + + A_dump = w.run(cloudpickle.dumps, A) + check_deterministic_pickle(A_dump, cloudpickle.dumps(A)) + + def test_dynamic_class_determinist_subworker_tuple_memoization(self): + # Check that the pickle produced by the unpickled instance is the same. + # This highlights some issues with tuple memoization. + + with subprocess_worker(protocol=self.protocol) as w: + # Arguments' tuple is memoized in the main process but not in the + # subprocess as the tuples do not share the same id in the loaded + # class. + class A: + """Class with potential tuple memoization issues.""" + + def func1(self): + pass + + def func2(self): + pass + + A_dump = w.run(cloudpickle.dumps, A) + check_deterministic_pickle(A_dump, cloudpickle.dumps(A)) + + @pytest.mark.skipif( + platform.python_implementation() == "PyPy", + reason="Skip PyPy because memory grows too much", + ) + @pytest.mark.skipif( + sys.platform == "darwin", + reason="Memory growth on macOS is close to the 50MB bound and it fails intermittently", + ) def test_interactive_remote_function_calls_no_memory_leak(self): code = """if __name__ == "__main__": from srsly.tests.cloudpickle.testutils import subprocess_worker @@ -1975,14 +2216,14 @@ def process_data(): growth = w.memsize() - reference_size # For some reason, the memory growth after processing 100MB of - # data is ~10MB on MacOS, and ~1MB on Linux, so the upper bound on + # data is ~50MB on MacOS, and ~1MB on Linux, so the upper bound on # memory growth we use is only tight for MacOS. However, - # - 10MB is still 10x lower than the expected memory growth in case + # - 50MB is still 2x lower than the expected memory growth in case # of a leak (which would be the total size of the processed data, # 100MB) # - the memory usage growth does not increase if using 10000 # iterations instead of 100 as used now (100x more data) - assert growth < 1.5e7, growth + assert growth < 5e7, growth """.format(protocol=self.protocol) assert_run_python_script(code) @@ -1994,12 +2235,18 @@ def test_pickle_reraise(self): cloudpickle.dumps(obj, protocol=self.protocol) def test_unhashable_function(self): - d = {'a': 1} + d = {"a": 1} depickled_method = pickle_depickle(d.get, protocol=self.protocol) - self.assertEqual(depickled_method('a'), 1) - self.assertEqual(depickled_method('b'), None) + self.assertEqual(depickled_method("a"), 1) + self.assertEqual(depickled_method("b"), None) - @pytest.mark.skipif(sys.version_info >= (3, 12), reason="Deprecation warning in python 3.12 about future deprecation in python 3.14") + @unittest.skipIf( + sys.version_info >= (3, 14), + "itertools.count() doesn't support pickle on Python 3.14+", + ) + @pytest.mark.filterwarnings( + "ignore:Pickle, copy, and deepcopy support will be removed from itertools:DeprecationWarning" + ) def test_itertools_count(self): counter = itertools.count(1, step=2) @@ -2044,11 +2291,9 @@ def g(): self.assertEqual(f2.__doc__, f.__doc__) def test_wraps_preserves_function_annotations(self): - def f(x): + def f(x: int) -> float: pass - f.__annotations__ = {'x': 1, 'return': float} - @wraps(f) def g(x): f(x) @@ -2063,6 +2308,7 @@ def test_type_hint(self): def test_instance_with_slots(self): for slots in [["registered_attribute"], "registered_attribute"]: + class ClassWithSlots: __slots__ = slots @@ -2070,8 +2316,9 @@ def __init__(self): self.registered_attribute = 42 initial_obj = ClassWithSlots() - depickled_obj = pickle_depickle( - initial_obj, protocol=self.protocol) + depickled_obj = pickle_depickle(initial_obj, protocol=self.protocol) + + assert depickled_obj.__class__.__slots__ == slots for obj in [initial_obj, depickled_obj]: self.assertEqual(obj.registered_attribute, 42) @@ -2088,9 +2335,10 @@ def __init__(self): depickled_obj = cloudpickle.loads(s) assert depickled_obj.unregistered_attribute == 1 - - @unittest.skipIf(not hasattr(types, "MappingProxyType"), - "Old versions of Python do not have this type.") + @unittest.skipIf( + not hasattr(types, "MappingProxyType"), + "Old versions of Python do not have this type.", + ) def test_mappingproxy(self): mp = types.MappingProxyType({"some_key": "some value"}) assert mp == pickle_depickle(mp, protocol=self.protocol) @@ -2098,7 +2346,7 @@ def test_mappingproxy(self): def test_dataclass(self): dataclasses = pytest.importorskip("dataclasses") - DataClass = dataclasses.make_dataclass('DataClass', [('x', int)]) + DataClass = dataclasses.make_dataclass("DataClass", [("x", int)]) data = DataClass(x=42) pickle_depickle(DataClass, protocol=self.protocol) @@ -2110,6 +2358,7 @@ class StringEnum(str, enum.Enum): class Color(StringEnum): """3-element color space""" + RED = "1" GREEN = "2" BLUE = "3" @@ -2118,7 +2367,8 @@ def is_green(self): return self is Color.GREEN green1, green2, ClonedColor = pickle_depickle( - [Color.GREEN, Color.GREEN, Color], protocol=self.protocol) + [Color.GREEN, Color.GREEN, Color], protocol=self.protocol + ) assert green1 is green2 assert green1 is ClonedColor.GREEN assert green1 is not ClonedColor.BLUE @@ -2139,7 +2389,8 @@ def test_locally_defined_intenum(self): green1, green2, ClonedDynamicColor = pickle_depickle( [DynamicColor.GREEN, DynamicColor.GREEN, DynamicColor], - protocol=self.protocol) + protocol=self.protocol, + ) assert green1 is green2 assert green1 is ClonedDynamicColor.GREEN @@ -2183,11 +2434,11 @@ def check_positive(x): assert_run_python_script(code) def test_relative_import_inside_function(self): - pytest.importorskip("_cloudpickle_testpkg") # Make sure relative imports inside round-tripped functions is not # broken. This was a bug in cloudpickle versions <= 0.5.3 and was # re-introduced in 0.8.0. - from _cloudpickle_testpkg import relative_imports_factory + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + relative_imports_factory = _cloudpickle_testpkg.relative_imports_factory f, g = relative_imports_factory() for func, source in zip([f, g], ["module", "package"]): # Make sure relative imports are initially working @@ -2206,10 +2457,12 @@ def f(a, *, b=1): for func in (f, depickled_f): assert func(2) == 3 - assert func.__kwdefaults__ == {'b': 1} + assert func.__kwdefaults__ == {"b": 1} - @pytest.mark.skipif(not hasattr(types.CodeType, "co_posonlyargcount"), - reason="Requires positional-only argument syntax") + @pytest.mark.skipif( + not hasattr(types.CodeType, "co_posonlyargcount"), + reason="Requires positional-only argument syntax", + ) def test_interactively_defined_func_with_positional_only_argument(self): # Fixes https://github.com/cloudpipe/cloudpickle/issues/266 # The source code of this test is bundled in a string and is ran from @@ -2237,11 +2490,10 @@ def f(a, /, b=1): def test___reduce___returns_string(self): # Non regression test for objects with a __reduce__ method returning a # string, meaning "save by attribute using save_global" - pytest.importorskip("_cloudpickle_testpkg") - from _cloudpickle_testpkg import some_singleton + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + some_singleton = _cloudpickle_testpkg.some_singleton assert some_singleton.__reduce__() == "some_singleton" - depickled_singleton = pickle_depickle( - some_singleton, protocol=self.protocol) + depickled_singleton = pickle_depickle(some_singleton, protocol=self.protocol) assert depickled_singleton is some_singleton def test_cloudpickle_extract_nested_globals(self): @@ -2249,24 +2501,33 @@ def function_factory(): def inner_function(): global _TEST_GLOBAL_VARIABLE return _TEST_GLOBAL_VARIABLE + return inner_function - globals_ = set(cloudpickle.cloudpickle._extract_code_globals( - function_factory.__code__).keys()) - assert globals_ == {'_TEST_GLOBAL_VARIABLE'} + globals_ = set( + cloudpickle.cloudpickle._extract_code_globals( + function_factory.__code__ + ).keys() + ) + assert globals_ == {"_TEST_GLOBAL_VARIABLE"} - depickled_factory = pickle_depickle(function_factory, - protocol=self.protocol) + depickled_factory = pickle_depickle(function_factory, protocol=self.protocol) inner_func = depickled_factory() assert inner_func() == _TEST_GLOBAL_VARIABLE + # TODO: remove this xfail when we drop support for Python 3.8. We don't + # plan to fix it because Python 3.8 is EOL. + @pytest.mark.skipif( + sys.version_info < (3, 9), + reason="Can cause CPython 3.8 to segfault", + ) def test_recursion_during_pickling(self): class A: def __getattribute__(self, name): return getattr(self, name) a = A() - with pytest.raises(pickle.PicklingError, match='recursion'): + with pytest.raises(pickle.PicklingError, match="deep recursion"): cloudpickle.dumps(a) def test_out_of_band_buffers(self): @@ -2279,18 +2540,22 @@ class LocallyDefinedClass: data_instance = LocallyDefinedClass() buffers = [] - pickle_bytes = cloudpickle.dumps(data_instance, protocol=self.protocol, - buffer_callback=buffers.append) + pickle_bytes = cloudpickle.dumps( + data_instance, protocol=self.protocol, buffer_callback=buffers.append + ) assert len(buffers) == 1 reconstructed = pickle.loads(pickle_bytes, buffers=buffers) np.testing.assert_allclose(reconstructed.data, data_instance.data) def test_pickle_dynamic_typevar(self): - T = typing.TypeVar('T') + T = typing.TypeVar("T") depickled_T = pickle_depickle(T, protocol=self.protocol) attr_list = [ - "__name__", "__bound__", "__constraints__", "__covariant__", - "__contravariant__" + "__name__", + "__bound__", + "__constraints__", + "__covariant__", + "__contravariant__", ] for attr in attr_list: assert getattr(T, attr) == getattr(depickled_T, attr) @@ -2301,23 +2566,22 @@ def test_pickle_dynamic_typevar_tracking(self): assert T is T2 def test_pickle_dynamic_typevar_memoization(self): - T = typing.TypeVar('T') - depickled_T1, depickled_T2 = pickle_depickle((T, T), - protocol=self.protocol) + T = typing.TypeVar("T") + depickled_T1, depickled_T2 = pickle_depickle((T, T), protocol=self.protocol) assert depickled_T1 is depickled_T2 def test_pickle_importable_typevar(self): - pytest.importorskip("_cloudpickle_testpkg") - from _cloudpickle_testpkg import T - T1 = pickle_depickle(T, protocol=self.protocol) - assert T1 is T + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + T1 = pickle_depickle(_cloudpickle_testpkg.T, protocol=self.protocol) + assert T1 is _cloudpickle_testpkg.T # Standard Library TypeVar from typing import AnyStr + assert AnyStr is pickle_depickle(AnyStr, protocol=self.protocol) def test_generic_type(self): - T = typing.TypeVar('T') + T = typing.TypeVar("T") class C(typing.Generic[T]): pass @@ -2330,31 +2594,24 @@ class C(typing.Generic[T]): with subprocess_worker(protocol=self.protocol) as worker: - def check_generic(generic, origin, type_value, use_args): + def check_generic(generic, origin, type_value): assert generic.__origin__ is origin assert len(origin.__orig_bases__) == 1 ob = origin.__orig_bases__[0] assert ob.__origin__ is typing.Generic - if use_args: - assert len(generic.__args__) == 1 - assert generic.__args__[0] is type_value - else: - assert len(generic.__parameters__) == 1 - assert generic.__parameters__[0] is type_value + assert len(generic.__args__) == 1 + assert generic.__args__[0] is type_value assert len(ob.__parameters__) == 1 return "ok" - # backward-compat for old Python 3.5 versions that sometimes relies - # on __parameters__ - use_args = getattr(C[int], '__args__', ()) != () - assert check_generic(C[int], C, int, use_args) == "ok" - assert worker.run(check_generic, C[int], C, int, use_args) == "ok" + assert check_generic(C[int], C, int) == "ok" + assert worker.run(check_generic, C[int], C, int) == "ok" def test_generic_subclass(self): - T = typing.TypeVar('T') + T = typing.TypeVar("T") class Base(typing.Generic[T]): pass @@ -2377,9 +2634,7 @@ class DerivedT(Base[T]): class LeafT(DerivedT[T]): pass - klasses = [ - Base, DerivedAny, LeafAny, DerivedInt, LeafInt, DerivedT, LeafT - ] + klasses = [Base, DerivedAny, LeafAny, DerivedInt, LeafInt, DerivedT, LeafT] for klass in klasses: assert pickle_depickle(klass, protocol=self.protocol) is klass @@ -2397,48 +2652,50 @@ def check_mro(klass, expected_mro): def test_locally_defined_class_with_type_hints(self): with subprocess_worker(protocol=self.protocol) as worker: for type_ in _all_types_to_test(): + class MyClass: def method(self, arg: type_) -> type_: return arg - MyClass.__annotations__ = {'attribute': type_} + + MyClass.__annotations__ = {"attribute": type_} def check_annotations(obj, expected_type, expected_type_str): - assert obj.__annotations__["attribute"] == expected_type - assert ( - obj.method.__annotations__["arg"] == expected_type - ) - assert ( - obj.method.__annotations__["return"] - == expected_type - ) + # On Python 3.14, it's no longer possible to access class + # annotations from an instance, so use type(). + assert type(obj).__annotations__["attribute"] == expected_type + assert obj.method.__annotations__["arg"] == expected_type + assert obj.method.__annotations__["return"] == expected_type return "ok" obj = MyClass() assert check_annotations(obj, type_, "type_") == "ok" - assert ( - worker.run(check_annotations, obj, type_, "type_") == "ok" - ) - - def test_generic_extensions_literal(self): - typing_extensions = pytest.importorskip('typing_extensions') - for obj in [typing_extensions.Literal, typing_extensions.Literal['a']]: - depickled_obj = pickle_depickle(obj, protocol=self.protocol) - assert depickled_obj == obj - - def test_generic_extensions_final(self): - typing_extensions = pytest.importorskip('typing_extensions') - for obj in [typing_extensions.Final, typing_extensions.Final[int]]: - depickled_obj = pickle_depickle(obj, protocol=self.protocol) - assert depickled_obj == obj + assert worker.run(check_annotations, obj, type_, "type_") == "ok" def test_class_annotations(self): class C: pass - C.__annotations__ = {'a': int} + + C.__annotations__ = {"a": int} C1 = pickle_depickle(C, protocol=self.protocol) assert C1.__annotations__ == C.__annotations__ + def test_class_annotations_abstractclass(self): + # see https://github.com/cloudpipe/cloudpickle/issues/572 + + class C(abc.ABC): + a: int + + C1 = pickle_depickle(C, protocol=self.protocol) + assert C1.__annotations__ == C.__annotations__ + C2 = pickle_depickle(C1, protocol=self.protocol) + if sys.version_info >= (3, 14): + # check that __annotate_func__ is created by Python + assert hasattr(C2, "__annotate_func__") + assert C2.__annotations__ == C1.__annotations__ + c2 = C2() + assert isinstance(c2, C2) + def test_function_annotations(self): def f(a: int) -> str: pass @@ -2450,17 +2707,19 @@ def test_always_use_up_to_date_copyreg(self): # test that updates of copyreg.dispatch_table are taken in account by # cloudpickle import copyreg + try: + class MyClass: pass def reduce_myclass(x): - return MyClass, (), {'custom_reduce': True} + return MyClass, (), {"custom_reduce": True} copyreg.dispatch_table[MyClass] = reduce_myclass my_obj = MyClass() depickled_myobj = pickle_depickle(my_obj, protocol=self.protocol) - assert hasattr(depickled_myobj, 'custom_reduce') + assert hasattr(depickled_myobj, "custom_reduce") finally: copyreg.dispatch_table.pop(MyClass) @@ -2484,8 +2743,9 @@ def __type__(self): o = MyClass() pickle_depickle(o, protocol=self.protocol) - @pytest.mark.skip(reason="Requires pytest -s to pass") - def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): # noqa + def test_pickle_constructs_from_module_registered_for_pickling_by_value( + self, + ): # noqa _prev_sys_path = sys.path.copy() try: # We simulate an interactive session that: @@ -2500,7 +2760,7 @@ def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): # First, remove sys.path entries that could point to # /path/to/cloudpickle/tests and be in inherited by the worker - _maybe_remove(sys.path, '') + _maybe_remove(sys.path, "") _maybe_remove(sys.path, _mock_interactive_session_cwd) # Add the desired session working directory @@ -2508,9 +2768,7 @@ def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): with subprocess_worker(protocol=self.protocol) as w: # Make the module unavailable in the remote worker - w.run( - lambda p: sys.path.remove(p), _mock_interactive_session_cwd - ) + w.run(lambda p: sys.path.remove(p), _mock_interactive_session_cwd) # Import the actual file after starting the module since the # worker is started using fork on Linux, which will inherits # the parent sys.modules. On Python>3.6, the worker can be @@ -2518,21 +2776,18 @@ def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): # TODO Once Python 3.6 reaches end of life, rely on mp_context # instead. import mock_local_folder.mod as mod + # The constructs whose pickling mechanism is changed using # register_pickle_by_value are functions, classes, TypeVar and # modules. - from mock_local_folder.mod import ( - local_function, LocalT, LocalClass - ) + from mock_local_folder.mod import local_function, LocalT, LocalClass # Make sure the module/constructs are unimportable in the # worker. with pytest.raises(ImportError): w.run(lambda: __import__("mock_local_folder.mod")) with pytest.raises(ImportError): - w.run( - lambda: __import__("mock_local_folder.subfolder.mod") - ) + w.run(lambda: __import__("mock_local_folder.subfolder.mod")) for o in [mod, local_function, LocalT, LocalClass]: with pytest.raises(ImportError): @@ -2544,14 +2799,9 @@ def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): # typevar assert w.run(lambda: LocalT.__name__) == LocalT.__name__ # classes - assert ( - w.run(lambda: LocalClass().method()) - == LocalClass().method() - ) + assert w.run(lambda: LocalClass().method()) == LocalClass().method() # modules - assert ( - w.run(lambda: mod.local_function()) == local_function() - ) + assert w.run(lambda: mod.local_function()) == local_function() # Constructs from modules inside subfolders should be pickled # by value if a namespace module pointing to some parent folder @@ -2559,16 +2809,19 @@ def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): # namespace module falls into that category, but a # "mock_local_folder.mod" one does not. from mock_local_folder.subfolder.submod import ( - LocalSubmodClass, LocalSubmodT, local_submod_function + LocalSubmodClass, + LocalSubmodT, + local_submod_function, ) + # Shorter aliases to comply with line-length limits _t, _func, _class = ( - LocalSubmodT, local_submod_function, LocalSubmodClass + LocalSubmodT, + local_submod_function, + LocalSubmodClass, ) with pytest.raises(ImportError): - w.run( - lambda: __import__("mock_local_folder.subfolder.mod") - ) + w.run(lambda: __import__("mock_local_folder.subfolder.mod")) with pytest.raises(ImportError): w.run(lambda: local_submod_function) @@ -2582,6 +2835,7 @@ def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): # Test the namespace folder case import mock_local_folder + register_pickle_by_value(mock_local_folder) assert w.run(lambda: local_function()) == local_function() assert w.run(lambda: _func()) == _func() @@ -2595,6 +2849,7 @@ def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): # Test the case of registering a single module inside a # subfolder. import mock_local_folder.subfolder.submod + register_pickle_by_value(mock_local_folder.subfolder.submod) assert w.run(lambda: _func()) == _func() assert w.run(lambda: _t.__name__) == _t.__name__ @@ -2608,14 +2863,13 @@ def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): with pytest.raises(ImportError): w.run(lambda: __import__("mock_local_folder.mod")) - unregister_pickle_by_value( - mock_local_folder.subfolder.submod - ) + unregister_pickle_by_value(mock_local_folder.subfolder.submod) with pytest.raises(ImportError): w.run(lambda: local_submod_function) # Test the subfolder namespace module case import mock_local_folder.subfolder + register_pickle_by_value(mock_local_folder.subfolder) assert w.run(lambda: _func()) == _func() assert w.run(lambda: _t.__name__) == _t.__name__ @@ -2625,23 +2879,26 @@ def test_pickle_constructs_from_module_registered_for_pickling_by_value(self): finally: _fname = "mock_local_folder" sys.path = _prev_sys_path - for m in [_fname, f"{_fname}.mod", f"{_fname}.subfolder", - f"{_fname}.subfolder.submod"]: + for m in [ + _fname, + f"{_fname}.mod", + f"{_fname}.subfolder", + f"{_fname}.subfolder.submod", + ]: mod = sys.modules.pop(m, None) if mod and mod.__name__ in list_registry_pickle_by_value(): unregister_pickle_by_value(mod) def test_pickle_constructs_from_installed_packages_registered_for_pickling_by_value( # noqa - self + self, ): - pytest.importorskip("_cloudpickle_testpkg") for package_or_module in ["package", "module"]: if package_or_module == "package": - import _cloudpickle_testpkg as m + m = pytest.importorskip("_cloudpickle_testpkg") f = m.package_function_with_global _original_global = m.global_variable elif package_or_module == "module": - import _cloudpickle_testpkg.mod as m + m = pytest.importorskip("_cloudpickle_testpkg.mod") f = m.module_function_with_global _original_global = m.global_variable try: @@ -2663,15 +2920,14 @@ def test_pickle_constructs_from_installed_packages_registered_for_pickling_by_va unregister_pickle_by_value(m) def test_pickle_various_versions_of_the_same_function_with_different_pickling_method( # noqa - self + self, ): - pytest.importorskip("_cloudpickle_testpkg") # Make sure that different versions of the same function (possibly # pickled in a different way - by value and/or by reference) can # peacefully co-exist (e.g. without globals interaction) in a remote # worker. - import _cloudpickle_testpkg - from _cloudpickle_testpkg import package_function_with_global as f + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") + f = _cloudpickle_testpkg.package_function_with_global _original_global = _cloudpickle_testpkg.global_variable def _create_registry(): @@ -2695,12 +2951,8 @@ def _call_from_registry(k): register_pickle_by_value(_cloudpickle_testpkg) _cloudpickle_testpkg.global_variable = "modified global" w.run(_add_to_registry, f, "f_by_val") - assert ( - w.run(_call_from_registry, "f_by_ref") == _original_global - ) - assert ( - w.run(_call_from_registry, "f_by_val") == "modified global" - ) + assert w.run(_call_from_registry, "f_by_ref") == _original_global + assert w.run(_call_from_registry, "f_by_val") == "modified global" finally: _cloudpickle_testpkg.global_variable = _original_global @@ -2708,10 +2960,6 @@ def _call_from_registry(k): if "_cloudpickle_testpkg" in list_registry_pickle_by_value(): unregister_pickle_by_value(_cloudpickle_testpkg) - @pytest.mark.skipif( - sys.version_info < (3, 7), - reason="Determinism can only be guaranteed for Python 3.7+" - ) def test_deterministic_pickle_bytes_for_function(self): # Ensure that functions with references to several global names are # pickled to fixed bytes that do not depend on the PYTHONHASHSEED of @@ -2723,53 +2971,129 @@ def func_with_globals(): for i in range(5): vals.add( - subprocess_pickle_string(func_with_globals, - protocol=self.protocol, - add_env={"PYTHONHASHSEED": str(i)})) + subprocess_pickle_string( + func_with_globals, + protocol=self.protocol, + add_env={"PYTHONHASHSEED": str(i)}, + ) + ) if len(vals) > 1: # Print additional debug info on stdout with dis: for val in vals: pickletools.dis(val) - pytest.fail( - "Expected a single deterministic payload, got %d/5" % len(vals) - ) + pytest.fail("Expected a single deterministic payload, got %d/5" % len(vals)) + def test_dataclass_fields_are_preserved(self): + @dataclasses.dataclass + class SampleDataclass: + x: int + y: dataclasses.InitVar[int] + z: typing.ClassVar[int] -class Protocol2CloudPickleTest(CloudPickleTest): + PickledSampleDataclass = pickle_depickle( + SampleDataclass, protocol=self.protocol + ) + + found_fields = list(PickledSampleDataclass.__dataclass_fields__.values()) + assert set(f.name for f in found_fields) == {"x", "y", "z"} + + expected_ftypes = { + "x": dataclasses._FIELD, + "y": dataclasses._FIELD_INITVAR, + "z": dataclasses._FIELD_CLASSVAR, + } + + for f in found_fields: + assert f._field_type is expected_ftypes[f.name] + + def test_interactively_defined_dataclass_with_initvar_and_classvar(self): + code = """if __name__ == "__main__": + import dataclasses + from srsly.tests.cloudpickle.testutils import subprocess_worker + import typing + + with subprocess_worker(protocol={protocol}) as w: + + @dataclasses.dataclass + class SampleDataclass: + x: int + y: dataclasses.InitVar[int] = None + z: typing.ClassVar[int] = 42 + + def __post_init__(self, y=0): + self.x += y + def large_enough(self): + return self.x > self.z + + value = SampleDataclass(2, y=2) + + def check_dataclass_instance(value): + assert isinstance(value, SampleDataclass) + assert value.x == 4 + assert value.z == 42 + expected_dict = dict(x=4) + assert dataclasses.asdict(value) == expected_dict + assert not value.large_enough() + try: + SampleDataclass.z = 0 + assert value.z == 0 + assert value.large_enough() + finally: + SampleDataclass.z = 42 + return "ok" + + assert check_dataclass_instance(value) == "ok" + + # Check that this instance of an interactively defined dataclass + # behavesconsistently in a remote worker process: + assert w.run(check_dataclass_instance, value) == "ok" + + # Check class provenance tracking is not impacted by the + # @dataclass decorator: + def echo(*args): + return args + + cloned_value, cloned_type = w.run(echo, value, SampleDataclass) + assert cloned_type is SampleDataclass + assert isinstance(cloned_value, SampleDataclass) + """.format(protocol=self.protocol) + assert_run_python_script(code) + + +class Protocol2CloudPickleTest(CloudPickleTest): protocol = 2 def test_lookup_module_and_qualname_dynamic_typevar(): - T = typing.TypeVar('T') + T = typing.TypeVar("T") module_and_name = _lookup_module_and_qualname(T, name=T.__name__) assert module_and_name is None def test_lookup_module_and_qualname_importable_typevar(): - pytest.importorskip("_cloudpickle_testpkg") - import _cloudpickle_testpkg + _cloudpickle_testpkg = pytest.importorskip("_cloudpickle_testpkg") T = _cloudpickle_testpkg.T module_and_name = _lookup_module_and_qualname(T, name=T.__name__) assert module_and_name is not None module, name = module_and_name assert module is _cloudpickle_testpkg - assert name == 'T' + assert name == "T" def test_lookup_module_and_qualname_stdlib_typevar(): - module_and_name = _lookup_module_and_qualname(typing.AnyStr, - name=typing.AnyStr.__name__) + module_and_name = _lookup_module_and_qualname( + typing.AnyStr, name=typing.AnyStr.__name__ + ) assert module_and_name is not None module, name = module_and_name assert module is typing - assert name == 'AnyStr' + assert name == "AnyStr" def test_register_pickle_by_value(): - pytest.importorskip("_cloudpickle_testpkg") - import _cloudpickle_testpkg as pkg - import _cloudpickle_testpkg.mod as mod + pkg = pytest.importorskip("_cloudpickle_testpkg") + mod = pytest.importorskip("_cloudpickle_testpkg.mod") assert list_registry_pickle_by_value() == set() @@ -2797,25 +3121,29 @@ def test_register_pickle_by_value(): with pytest.raises(ValueError, match=msg): register_pickle_by_value(pkg.__name__) - dynamic_mod = types.ModuleType('dynamic_mod') + dynamic_mod = types.ModuleType("dynamic_mod") msg = ( f"{dynamic_mod} was not imported correctly, have you used an " - f"`import` statement to access it?" + "`import` statement to access it?" ) with pytest.raises(ValueError, match=re.escape(msg)): register_pickle_by_value(dynamic_mod) def _all_types_to_test(): - T = typing.TypeVar('T') + T = typing.TypeVar("T") class C(typing.Generic[T]): pass types_to_test = [ - C, C[int], - T, typing.Any, typing.Optional, - typing.Generic, typing.Union, + C, + C[int], + T, + typing.Any, + typing.Optional, + typing.Generic, + typing.Union, typing.Optional[int], typing.Generic[T], typing.Callable[[int], typing.Any], @@ -2839,5 +3167,5 @@ def test_module_level_pickler(): assert cloudpickle.Pickler is cloudpickle.CloudPickler -if __name__ == '__main__': +if __name__ == "__main__": unittest.main() diff --git a/srsly/tests/cloudpickle/mock_local_folder/mod.py b/srsly/tests/cloudpickle/mock_local_folder/mod.py index 1a1c1da..517d501 100644 --- a/srsly/tests/cloudpickle/mock_local_folder/mod.py +++ b/srsly/tests/cloudpickle/mock_local_folder/mod.py @@ -5,6 +5,7 @@ reference should instead flagged to cloudpickle for pickling by value: this is done using the register_pickle_by_value api exposed by cloudpickle. """ + import typing diff --git a/srsly/tests/cloudpickle/testutils.py b/srsly/tests/cloudpickle/testutils.py index e0890b4..f614924 100644 --- a/srsly/tests/cloudpickle/testutils.py +++ b/srsly/tests/cloudpickle/testutils.py @@ -1,9 +1,12 @@ import sys import os -import os.path as op +import io +import difflib import tempfile +import os.path as op from subprocess import Popen, check_output, PIPE, STDOUT, CalledProcessError -from srsly.cloudpickle.compat import pickle +import pickle +import pickletools from contextlib import contextmanager from concurrent.futures import ProcessPoolExecutor @@ -28,17 +31,16 @@ def g(x): def _make_cwd_env(): """Helper to prepare environment for the child processes""" - cloudpickle_repo_folder = op.normpath( - op.join(op.dirname(__file__), '..')) + cloudpickle_repo_folder = op.normpath(op.join(op.dirname(__file__), "..")) env = os.environ.copy() pythonpath = "{src}{sep}tests{pathsep}{src}".format( - src=cloudpickle_repo_folder, sep=os.sep, pathsep=os.pathsep) - env['PYTHONPATH'] = pythonpath + src=cloudpickle_repo_folder, sep=os.sep, pathsep=os.pathsep + ) + env["PYTHONPATH"] = pythonpath return cloudpickle_repo_folder, env -def subprocess_pickle_string(input_data, protocol=None, timeout=TIMEOUT, - add_env=None): +def subprocess_pickle_string(input_data, protocol=None, timeout=TIMEOUT, add_env=None): """Retrieve pickle string of an object generated by a child Python process Pickle the input data into a buffer, send it to a subprocess via @@ -54,31 +56,31 @@ def subprocess_pickle_string(input_data, protocol=None, timeout=TIMEOUT, # Protect stderr from any warning, as we will assume an error will happen # if it is not empty. A concrete example is pytest using the imp module, # which is deprecated in python 3.8 - cmd = [sys.executable, '-W ignore', __file__, "--protocol", str(protocol)] + cmd = [sys.executable, "-W ignore", __file__, "--protocol", str(protocol)] cwd, env = _make_cwd_env() if add_env: env.update(add_env) - proc = Popen(cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=cwd, env=env, - bufsize=4096) + proc = Popen( + cmd, stdin=PIPE, stdout=PIPE, stderr=PIPE, cwd=cwd, env=env, bufsize=4096 + ) pickle_string = dumps(input_data, protocol=protocol) try: comm_kwargs = {} - comm_kwargs['timeout'] = timeout + comm_kwargs["timeout"] = timeout out, err = proc.communicate(pickle_string, **comm_kwargs) if proc.returncode != 0 or len(err): message = "Subprocess returned %d: " % proc.returncode - message += err.decode('utf-8') + message += err.decode("utf-8") raise RuntimeError(message) return out except TimeoutExpired as e: proc.kill() out, err = proc.communicate() - message = "\n".join([out.decode('utf-8'), err.decode('utf-8')]) + message = "\n".join([out.decode("utf-8"), err.decode("utf-8")]) raise RuntimeError(message) from e -def subprocess_pickle_echo(input_data, protocol=None, timeout=TIMEOUT, - add_env=None): +def subprocess_pickle_echo(input_data, protocol=None, timeout=TIMEOUT, add_env=None): """Echo function with a child Python process Pickle the input data into a buffer, send it to a subprocess via stdin, expect the subprocess to unpickle, re-pickle that data back @@ -86,10 +88,9 @@ def subprocess_pickle_echo(input_data, protocol=None, timeout=TIMEOUT, >>> subprocess_pickle_echo([1, 'a', None]) [1, 'a', None] """ - out = subprocess_pickle_string(input_data, - protocol=protocol, - timeout=timeout, - add_env=add_env) + out = subprocess_pickle_string( + input_data, protocol=protocol, timeout=timeout, add_env=add_env + ) return loads(out) @@ -111,9 +112,9 @@ def pickle_echo(stream_in=None, stream_out=None, protocol=None): stream_out = sys.stdout # Force the use of bytes streams under Python 3 - if hasattr(stream_in, 'buffer'): + if hasattr(stream_in, "buffer"): stream_in = stream_in.buffer - if hasattr(stream_out, 'buffer'): + if hasattr(stream_out, "buffer"): stream_out = stream_out.buffer input_bytes = _read_all_bytes(stream_in) @@ -145,7 +146,8 @@ def run(self, func, *args, **kwargs): input_payload = dumps((func, args, kwargs), protocol=self.protocol) result_payload = self.pool.submit( - call_func, input_payload, self.protocol).result() + call_func, input_payload, self.protocol + ).result() result = loads(result_payload) if isinstance(result, BaseException): @@ -153,14 +155,14 @@ def run(self, func, *args, **kwargs): return result def memsize(self): - workers_pids = [p.pid if hasattr(p, "pid") else p - for p in list(self.pool._processes)] + workers_pids = [ + p.pid if hasattr(p, "pid") else p for p in list(self.pool._processes) + ] num_workers = len(workers_pids) if num_workers == 0: return 0 elif num_workers > 1: - raise RuntimeError("Unexpected number of workers: %d" - % num_workers) + raise RuntimeError("Unexpected number of workers: %d" % num_workers) return psutil.Process(workers_pids[0]).memory_info().rss def close(self): @@ -180,38 +182,73 @@ def assert_run_python_script(source_code, timeout=TIMEOUT): The script provided in the source code should return 0 and not print anything on stderr or stdout. """ - fd, source_file = tempfile.mkstemp(suffix='_src_test_cloudpickle.py') + fd, source_file = tempfile.mkstemp(suffix="_src_test_cloudpickle.py") os.close(fd) try: - with open(source_file, 'wb') as f: - f.write(source_code.encode('utf-8')) - cmd = [sys.executable, '-W ignore', source_file] + with open(source_file, "wb") as f: + f.write(source_code.encode("utf-8")) + cmd = [sys.executable, "-W ignore", source_file] cwd, env = _make_cwd_env() kwargs = { - 'cwd': cwd, - 'stderr': STDOUT, - 'env': env, + "cwd": cwd, + "stderr": STDOUT, + "env": env, } # If coverage is running, pass the config file to the subprocess coverage_rc = os.environ.get("COVERAGE_PROCESS_START") if coverage_rc: - kwargs['env']['COVERAGE_PROCESS_START'] = coverage_rc - kwargs['timeout'] = timeout + kwargs["env"]["COVERAGE_PROCESS_START"] = coverage_rc + kwargs["timeout"] = timeout try: try: out = check_output(cmd, **kwargs) except CalledProcessError as e: - raise RuntimeError("script errored with output:\n%s" - % e.output.decode('utf-8')) from e + raise RuntimeError( + "script errored with output:\n%s" % e.output.decode("utf-8") + ) from e if out != b"": - raise AssertionError(out.decode('utf-8')) + raise AssertionError(out.decode("utf-8")) except TimeoutExpired as e: - raise RuntimeError("script timeout, output so far:\n%s" - % e.output.decode('utf-8')) from e + raise RuntimeError( + "script timeout, output so far:\n%s" % e.output.decode("utf-8") + ) from e finally: os.unlink(source_file) -if __name__ == '__main__': - protocol = int(sys.argv[sys.argv.index('--protocol') + 1]) +def check_deterministic_pickle(a, b): + """Check that two pickle output are bitwise equal. + + If it is not the case, print the diff between the disassembled pickle + payloads. + + This helper is useful to investigate non-deterministic pickling. + """ + if a != b: + with io.StringIO() as out: + pickletools.dis(pickletools.optimize(a), out) + a_out = out.getvalue() + # Remove the 11 first characters of each line to remove the bytecode offset + # of each object, which is different on each line for very small differences, + # making the diff very hard to read. + a_out = "\n".join(ll[11:] for ll in a_out.splitlines()) + with io.StringIO() as out: + pickletools.dis(pickletools.optimize(b), out) + b_out = out.getvalue() + b_out = "\n".join(ll[11:] for ll in b_out.splitlines()) + assert a_out == b_out + full_diff = difflib.context_diff( + a_out.splitlines(keepends=True), b_out.splitlines(keepends=True) + ) + full_diff = "".join(full_diff) + if len(full_diff) > 1500: + full_diff = full_diff[:1494] + " [...]" + raise AssertionError( + "Pickle payloads are not bitwise equal:\n" + + full_diff + ) + + +if __name__ == "__main__": + protocol = int(sys.argv[sys.argv.index("--protocol") + 1]) pickle_echo(protocol=protocol)