[MRG+1] FIX support memmap scalars as CV scores (scikit-learn#6789)

* FIX support memmap scalars as CV scores

* FIX test for Python 3.5 and NumPy 1.12

Author: jnothman

sklearn/cross_validation.py

@@ -1648,6 +1648,13 @@ def _score(estimator, X_test, y_test, scorer):
         score = scorer(estimator, X_test)
     else:
         score = scorer(estimator, X_test, y_test)
+    if hasattr(score, 'item'):
+        try:
+            # e.g. unwrap memmapped scalars
+            score = score.item()
+        except ValueError:
+            # non-scalar?
+            pass
     if not isinstance(score, numbers.Number):
         raise ValueError("scoring must return a number, got %s (%s) instead."
                          % (str(score), type(score)))

sklearn/model_selection/_validation.py

@@ -301,6 +301,13 @@ def _score(estimator, X_test, y_test, scorer):
         score = scorer(estimator, X_test)
     else:
         score = scorer(estimator, X_test, y_test)
+    if hasattr(score, 'item'):
+        try:
+            # e.g. unwrap memmapped scalars
+            score = score.item()
+        except ValueError:
+            # non-scalar?
+            pass
     if not isinstance(score, numbers.Number):
         raise ValueError("scoring must return a number, got %s (%s) instead."
                          % (str(score), type(score)))

sklearn/model_selection/tests/test_validation.py

@@ -3,6 +3,8 @@
 
 import sys
 import warnings
+import tempfile
+import os
 
 import numpy as np
 from scipy.sparse import coo_matrix, csr_matrix
@@ -769,3 +771,22 @@ def test_cross_val_predict_with_method():
         predictions = cross_val_predict(est, X, y, method=method,
                                         cv=kfold)
         assert_array_almost_equal(expected_predictions, predictions)
+
+
+def test_score_memmap():
+    # Ensure a scalar score of memmap type is accepted
+    iris = load_iris()
+    X, y = iris.data, iris.target
+    clf = MockClassifier()
+    tf = tempfile.NamedTemporaryFile(mode='wb', delete=False)
+    tf.write(b'Hello world!!!!!')
+    tf.close()
+    scores = np.memmap(tf.name, dtype=float)
+    score = np.memmap(tf.name, shape=(), mode='w+', dtype=float)
+    try:
+        cross_val_score(clf, X, y, scoring=lambda est, X, y: score)
+        # non-scalar should still fail
+        assert_raises(ValueError, cross_val_score, clf, X, y,
+                      scoring=lambda est, X, y: scores)
+    finally:
+        os.unlink(tf.name)