BUG: update joblib to 0.7.0d

Fixes hashing bug that could lead to collisions in memory

Author: GaelVaroquaux

sklearn/externals/joblib/__init__.py

@@ -102,7 +102,7 @@
 
 """
 
-__version__ = '0.7.0b'
+__version__ = '0.7.0d'
 
 
 from .memory import Memory

sklearn/externals/joblib/hashing.py

@@ -7,6 +7,7 @@
 # Copyright (c) 2009 Gael Varoquaux
 # License: BSD Style, 3 clauses.
 
+import warnings
 import pickle
 import hashlib
 import sys
@@ -29,6 +30,13 @@ def __init__(self, set_sequence):
         self._sequence = sorted(set_sequence)
 
 
+class _MyHash(object):
+    """ Class used to hash objects that won't normaly pickle """
+
+    def __init__(self, *args):
+        self.args = args
+
+
 class Hasher(Pickler):
     """ A subclass of pickler, to do cryptographic hashing, rather than
         pickling.
@@ -43,9 +51,8 @@ def __init__(self, hash_name='md5'):
     def hash(self, obj, return_digest=True):
         try:
             self.dump(obj)
-        except pickle.PicklingError:
-            pass
-            #self.dump(
+        except pickle.PicklingError as e:
+            warnings.warn('PicklingError while hashing %r: %r' % (obj, e))
         dumps = self.stream.getvalue()
         self._hash.update(dumps)
         if return_digest:
@@ -60,8 +67,14 @@ def save(self, obj):
             else:
                 func_name = obj.__name__
             inst = obj.__self__
-            cls = obj.__self__.__class__
-            obj = (func_name, inst, cls)
+            if type(inst) == type(pickle):
+                obj = _MyHash(func_name, inst.__name__)
+            elif inst is None:
+                # type(None) or type(module) do not pickle
+                obj = _MyHash(func_name, inst)
+            else:
+                cls = obj.__self__.__class__
+                obj = _MyHash(func_name, inst, cls)
         Pickler.save(self, obj)
 
     # The dispatch table of the pickler is not accessible in Python
@@ -70,17 +83,20 @@ def save_global(self, obj, name=None, pack=struct.pack):
         # We have to override this method in order to deal with objects
         # defined interactively in IPython that are not injected in
         # __main__
-        module = getattr(obj, "__module__", None)
-        if module == '__main__':
-            my_name = name
-            if my_name is None:
-                my_name = obj.__name__
-            mod = sys.modules[module]
-            if not hasattr(mod, my_name):
-                # IPython doesn't inject the variables define
-                # interactively in __main__
-                setattr(mod, my_name, obj)
-        Pickler.save_global(self, obj, name=name, pack=struct.pack)
+        try:
+            Pickler.save_global(self, obj, name=name, pack=pack)
+        except pickle.PicklingError:
+            Pickler.save_global(self, obj, name=name, pack=pack)
+            module = getattr(obj, "__module__", None)
+            if module == '__main__':
+                my_name = name
+                if my_name is None:
+                    my_name = obj.__name__
+                mod = sys.modules[module]
+                if not hasattr(mod, my_name):
+                    # IPython doesn't inject the variables define
+                    # interactively in __main__
+                    setattr(mod, my_name, obj)
 
     dispatch = Pickler.dispatch.copy()
     # builtin

sklearn/externals/joblib/test/test_hashing.py

@@ -77,13 +77,18 @@ def f(self, x):
 def test_trival_hash():
     """ Smoke test hash on various types.
     """
-    obj_list = [1, 1., 1 + 1j,
-                'a',
-                (1, ), [1, ], {1:1},
+    obj_list = [1, 2, 1., 2., 1 + 1j, 2. + 1j,
+                'a', 'b',
+                (1, ), (1, 1, ), [1, ], [1, 1, ],
+                {1: 1}, {1: 2}, {2: 1},
                 None,
+                gc.collect,
+                [1, ].append,
                ]
     for obj1 in obj_list:
         for obj2 in obj_list:
+            # Check that 2 objects have the same hash only if they are
+            # the same.
             yield nose.tools.assert_equal, hash(obj1) == hash(obj2), \
                 obj1 is obj2
 
@@ -223,6 +228,15 @@ def test_hash_object_dtype():
                             hash(b))
 
 
+@with_numpy
+def test_numpy_scalar():
+    # Numpy scalars are built from compiled functions, and lead to
+    # strange pickling paths explored, that can give hash collisions
+    a = np.float64(2.0)
+    b = np.float64(3.0)
+    nose.tools.assert_not_equal(hash(a), hash(b))
+
+
 def test_dict_hash():
     # Check that dictionaries hash consistently, eventhough the ordering
     # of the keys is not garanteed