[MRG] Fix DeprecationWarning due to collections.abc in Python 3.7 (scikit-learn#11431)

Closes scikit-learn#11121

This PR removes the deprecation warning about ABC being moved from `collections` to `collections.abc` when importing scikit-learn in Python 3.7.

In the end, I put `collections.abc.{Sequence, Iterable, Mapping, Sized}` in the namespace of `sklearn.utils.fixes`. This was the simplest way I could find, and while it has the drawback of obfuscating the real module name, other approached appeared more problematic and a similar approach is currently used e.g. for `utils.fixes.signature` which is an alias for `inspect.signature`.

We can't just patch six with benjaminp/six#241, because sklearn uses six from 5 years ago, which would  need updating and I'm not sure if it could have side effects (e.g. for pickling backward compatibility etc).

**Edit**: This adds a test checking that generally no warnings are raised when importing scikit-learn top-level modules.

Author: rth

sklearn/covariance/graph_lasso_.py

@@ -19,11 +19,11 @@
 from ..exceptions import ConvergenceWarning
 from ..utils.validation import check_random_state, check_array
 from ..utils import deprecated
+from ..utils.fixes import _Sequence as Sequence
 from ..linear_model import lars_path
 from ..linear_model import cd_fast
 from ..model_selection import check_cv, cross_val_score
 from ..externals.joblib import Parallel, delayed
-import collections
 
 
 # Helper functions to compute the objective and dual objective functions
@@ -608,7 +608,7 @@ def fit(self, X, y=None):
         n_alphas = self.alphas
         inner_verbose = max(0, self.verbose - 1)
 
-        if isinstance(n_alphas, collections.Sequence):
+        if isinstance(n_alphas, Sequence):
             alphas = self.alphas
             n_refinements = 1
         else:
@@ -684,7 +684,7 @@ def fit(self, X, y=None):
                 alpha_1 = path[best_index - 1][0]
                 alpha_0 = path[best_index + 1][0]
 
-            if not isinstance(n_alphas, collections.Sequence):
+            if not isinstance(n_alphas, Sequence):
                 alphas = np.logspace(np.log10(alpha_1), np.log10(alpha_0),
                                      n_alphas + 2)
                 alphas = alphas[1:-1]

sklearn/datasets/samples_generator.py

@@ -11,11 +11,11 @@
 import numpy as np
 from scipy import linalg
 import scipy.sparse as sp
-from collections import Iterable
 
 from ..preprocessing import MultiLabelBinarizer
 from ..utils import check_array, check_random_state
 from ..utils import shuffle as util_shuffle
+from ..utils.fixes import _Iterable as Iterable
 from ..utils.random import sample_without_replacement
 from ..externals import six
 map = six.moves.map

sklearn/feature_extraction/dict_vectorizer.py

@@ -3,7 +3,6 @@
 # License: BSD 3 clause
 
 from array import array
-from collections import Mapping
 from operator import itemgetter
 
 import numpy as np
@@ -13,6 +12,7 @@
 from ..externals import six
 from ..externals.six.moves import xrange
 from ..utils import check_array, tosequence
+from ..utils.fixes import _Mapping as Mapping
 
 
 def _tosequence(X):

sklearn/feature_extraction/tests/test_text.py

@@ -33,8 +33,8 @@
                                    clean_warning_registry, ignore_warnings,
                                    SkipTest, assert_raises,
                                    assert_allclose_dense_sparse)
-
-from collections import defaultdict, Mapping
+from sklearn.utils.fixes import _Mapping as Mapping
+from collections import defaultdict
 from functools import partial
 import pickle
 from io import StringIO

sklearn/metrics/scorer.py

@@ -19,7 +19,6 @@
 # License: Simplified BSD
 
 from abc import ABCMeta
-from collections import Iterable
 
 import numpy as np
 
@@ -40,6 +39,7 @@
 from .cluster import fowlkes_mallows_score
 
 from ..utils.multiclass import type_of_target
+from ..utils.fixes import _Iterable as Iterable
 from ..externals import six
 from ..base import is_regressor
 

sklearn/model_selection/_search.py

@@ -13,7 +13,7 @@
 # License: BSD 3 clause
 
 from abc import ABCMeta, abstractmethod
-from collections import Mapping, namedtuple, defaultdict, Sequence, Iterable
+from collections import namedtuple, defaultdict
 from functools import partial, reduce
 from itertools import product
 import operator
@@ -34,6 +34,8 @@
 from ..utils import check_random_state
 from ..utils.fixes import sp_version
 from ..utils.fixes import MaskedArray
+from ..utils.fixes import _Mapping as Mapping, _Sequence as Sequence
+from ..utils.fixes import _Iterable as Iterable
 from ..utils.random import sample_without_replacement
 from ..utils.validation import indexable, check_is_fitted
 from ..utils.metaestimators import if_delegate_has_method

sklearn/model_selection/_split.py

@@ -15,7 +15,6 @@
 
 import warnings
 from itertools import chain, combinations
-from collections import Iterable
 from math import ceil, floor
 import numbers
 from abc import ABCMeta, abstractmethod
@@ -29,6 +28,7 @@
 from ..externals.six import with_metaclass
 from ..externals.six.moves import zip
 from ..utils.fixes import signature, comb
+from ..utils.fixes import _Iterable as Iterable
 from ..base import _pprint
 
 __all__ = ['BaseCrossValidator',

sklearn/model_selection/tests/test_search.py

@@ -1,6 +1,5 @@
 """Test the search module"""
 
-from collections import Iterable, Sized
 from sklearn.externals.six.moves import cStringIO as StringIO
 from sklearn.externals.six.moves import xrange
 from itertools import chain, product
@@ -15,6 +14,7 @@
 
 from sklearn.utils.fixes import sp_version
 from sklearn.utils.fixes import PY3_OR_LATER
+from sklearn.utils.fixes import _Iterable as Iterable, _Sized as Sized
 from sklearn.utils.testing import assert_equal
 from sklearn.utils.testing import assert_not_equal
 from sklearn.utils.testing import assert_raises

sklearn/tests/test_init.py

@@ -1,11 +1,18 @@
 # Basic unittests to test functioning of module's top-level
 
-__author__ = 'Yaroslav Halchenko'
-__license__ = 'BSD'
+import subprocess
+
+import pkgutil
 
+import pytest
 
+import sklearn
 from sklearn.utils.testing import assert_equal
 
+__author__ = 'Yaroslav Halchenko'
+__license__ = 'BSD'
+
+
 try:
     from sklearn import *  # noqa
     _top_import_error = None
@@ -18,3 +25,37 @@ def test_import_skl():
     # "import *" is discouraged outside of the module level, hence we
     # rely on setting up the variable above
     assert_equal(_top_import_error, None)
+
+
+def test_import_sklearn_no_warnings():
+    # Test that importing scikit-learn main modules doesn't raise any warnings.
+
+    try:
+        pkgs = pkgutil.iter_modules(path=sklearn.__path__, prefix='sklearn.')
+        import_modules = '; '.join(['import ' + modname
+                                    for _, modname, _ in pkgs
+                                    if (not modname.startswith('_') and
+                                        # add deprecated top level modules
+                                        # below to ignore them
+                                        modname not in [])])
+
+        message = subprocess.check_output(['python', '-Wdefault',
+                                           '-c', import_modules],
+                                          stderr=subprocess.STDOUT)
+        message = message.decode("utf-8")
+        message = '\n'.join([line for line in message.splitlines()
+                             if not (
+                                     # ignore ImportWarning due to Cython
+                                     "ImportWarning" in line or
+                                     # ignore DeprecationWarning due to pytest
+                                     "pytest" in line or
+                                     # ignore DeprecationWarnings due to
+                                     # numpy.oldnumeric
+                                     "oldnumeric" in line
+                                     )])
+        assert 'Warning' not in message
+        assert 'Error' not in message
+
+    except Exception as e:
+        pytest.skip('soft-failed test_import_sklearn_no_warnings.\n'
+                    ' %s, \n %s' % (e, message))

sklearn/utils/__init__.py

@@ -1,7 +1,7 @@
 """
 The :mod:`sklearn.utils` module includes various utilities.
 """
-from collections import Sequence
+
 import numbers
 
 import numpy as np
@@ -17,6 +17,7 @@
 from .class_weight import compute_class_weight, compute_sample_weight
 from ..externals.joblib import cpu_count
 from ..exceptions import DataConversionWarning
+from ..utils.fixes import _Sequence as Sequence
 from .deprecation import deprecated
 from .. import get_config
 

sklearn/utils/fixes.py

@@ -310,3 +310,16 @@ def _object_dtype_isnan(X):
 else:
     def _object_dtype_isnan(X):
         return np.frompyfunc(lambda x: x != x, 1, 1)(X).astype(bool)
+
+
+# To be removed once this fix is included in six
+try:
+    from collections.abc import Sequence as _Sequence  # noqa
+    from collections.abc import Iterable as _Iterable  # noqa
+    from collections.abc import Mapping as _Mapping  # noqa
+    from collections.abc import Sized as _Sized  # noqa
+except ImportError:  # python <3.3
+    from collections import Sequence as _Sequence  # noqa
+    from collections import Iterable as _Iterable  # noqa
+    from collections import Mapping as _Mapping  # noqa
+    from collections import Sized as _Sized  # noqa

sklearn/utils/multiclass.py

@@ -7,7 +7,6 @@
 
 """
 from __future__ import division
-from collections import Sequence
 from itertools import chain
 
 from scipy.sparse import issparse
@@ -18,10 +17,10 @@
 import numpy as np
 
 from ..externals.six import string_types
+from ..utils.fixes import _Sequence as Sequence
 from .validation import check_array
 
 
-
 def _unique_multiclass(y):
     if hasattr(y, '__array__'):
         return np.unique(np.asarray(y))