ENH move covtype loading to sklearn.datasets

Author: larsmans

benchmarks/bench_covertype.py

@@ -44,25 +44,29 @@
 
 print(__doc__)
 
-# Author: Peter Prettenhoer <[email protected]>
+# Author: Peter Prettenhofer <[email protected]>
 # License: BSD Style.
 
-# $Id$
-
-from time import time
+import logging
 import os
 import sys
-import numpy as np
+from time import time
 from optparse import OptionParser
 
+import numpy as np
+
+from sklearn.datasets import fetch_covtype
 from sklearn.svm import LinearSVC
 from sklearn.linear_model import SGDClassifier
 from sklearn.naive_bayes import GaussianNB
 from sklearn.tree import DecisionTreeClassifier
 from sklearn.ensemble import RandomForestClassifier, ExtraTreesClassifier
 from sklearn import metrics
 from sklearn.externals.joblib import Memory
-from sklearn.utils import check_random_state
+
+logging.basicConfig(level=logging.INFO,
+                    format='%(asctime)s %(levelname)s %(message)s')
+logger = logging.getLogger(__name__)
 
 op = OptionParser()
 op.add_option("--classifiers",
@@ -80,8 +84,7 @@
 # estimators.
 op.add_option("--random-seed",
               dest="random_seed", default=13, type=int,
-              help="Common seed used by random number generator."
-              )
+              help="Common seed used by random number generator.")
 
 op.print_help()
 
@@ -97,57 +100,31 @@
 joblib_cache_folder = os.path.join(bench_folder, 'bench_covertype_data')
 m = Memory(joblib_cache_folder, mmap_mode='r')
 
-# Set seed for rng
-rng = check_random_state(opts.random_seed)
-
 
 # Load the data, then cache and memmap the train/test split
 @m.cache
 def load_data(dtype=np.float32, order='F'):
-    ######################################################################
-    ## Download the data, if not already on disk
-    if not os.path.exists(original_archive):
-        # Download the data
-        import urllib
-        print("Downloading data, Please Wait (11MB)...")
-        opener = urllib.urlopen(
-            'http://archive.ics.uci.edu/ml/'
-            'machine-learning-databases/covtype/covtype.data.gz')
-        open(original_archive, 'wb').write(opener.read())
-
     ######################################################################
     ## Load dataset
     print("Loading dataset...")
-    import gzip
-    f = gzip.open(original_archive)
-    X = np.fromstring(f.read().replace(",", " "), dtype=dtype, sep=" ",
-                      count=-1)
-    X = X.reshape((581012, 55))
+    data = fetch_covtype(download_if_missing=True, shuffle=True,
+                         random_state=opts.random_seed)
+    X, y = data.data, data.target
     if order.lower() == 'f':
         X = np.asfortranarray(X)
-    f.close()
 
     # class 1 vs. all others.
-    y = np.ones(X.shape[0]) * -1
-    y[np.where(X[:, -1] == 1)] = 1
-    X = X[:, :-1]
+    y[np.where(y != 1)] = -1
 
     ######################################################################
     ## Create train-test split (as [Joachims, 2006])
-    print("Creating train-test split...")
-    idx = np.arange(X.shape[0])
-    rng.shuffle(idx)
-    train_idx = idx[:522911]
-    test_idx = idx[522911:]
+    logger.info("Creating train-test split...")
+    n_train = 522911
 
-    X_train = X[train_idx]
-    y_train = y[train_idx]
-    X_test = X[test_idx]
-    y_test = y[test_idx]
-
-    # free memory
-    del X
-    del y
+    X_train = X[:n_train]
+    y_train = y[:n_train]
+    X_test = X[n_train:]
+    y_test = y[n_train:]
 
     ######################################################################
     ## Standardize first 10 features (the numerical ones)
@@ -206,7 +183,7 @@ def benchmark(clf):
     'dual': False,
     'tol': 1e-3,
     "random_state": opts.random_seed,
-    }
+}
 classifiers['liblinear'] = LinearSVC(**liblinear_parameters)
 
 ######################################################################
@@ -220,7 +197,7 @@ def benchmark(clf):
     'n_iter': 2,
     'n_jobs': opts.n_jobs,
     "random_state": opts.random_seed,
-    }
+}
 classifiers['SGD'] = SGDClassifier(**sgd_parameters)
 
 ######################################################################

doc/datasets/covtype.rst

@@ -0,0 +1,19 @@
+
+.. _covtype:
+
+Forest covertypes
+=================
+
+The samples in this dataset correspond to 30×30m patches of forest in the US,
+collected for the task of predicting each patch's cover type,
+i.e. the dominant species of tree.
+There are seven covertypes, making this a multiclass classification problem.
+Each sample has 54 features, described on the
+`dataset's homepage <http://archive.ics.uci.edu/ml/datasets/Covertype>`_.
+Some of the features are boolean indicators,
+while others are discrete or continuous measurements.
+
+``sklearn.datasets.fetch_covtype`` will load the covertype dataset;
+it returns a ``Bunch`` object with the feature matrix in the ``data`` member
+and the target values in ``target``.
+The dataset will be downloaded from the web if necessary.

doc/datasets/index.rst

@@ -180,3 +180,4 @@ features::
 
 .. include:: labeled_faces.rst
 
+.. include:: covtype.rst

sklearn/datasets/__init__.py

@@ -14,6 +14,7 @@
 from .base import clear_data_home
 from .base import load_sample_images
 from .base import load_sample_image
+from .covtype import fetch_covtype
 from .mlcomp import load_mlcomp
 from .lfw import load_lfw_pairs
 from .lfw import load_lfw_people

sklearn/datasets/covtype.py

@@ -0,0 +1,94 @@
+"""Forest covertype dataset.
+
+A classic dataset for classification benchmarks, featuring categorical and
+real-valued features.
+"""
+
+# Author: Lars Buitinck <[email protected]>
+#         Peter Prettenhofer <[email protected]>
+# License: 3-clause BSD.
+
+import errno
+from gzip import GzipFile
+from io import BytesIO
+import logging
+import os
+from os.path import exists, join
+from urllib2 import urlopen
+
+import numpy as np
+
+from .base import get_data_home
+from .base import Bunch
+from ..externals import joblib
+from ..utils import check_random_state
+
+
+URL = ('http://archive.ics.uci.edu/ml/'
+       'machine-learning-databases/covtype/covtype.data.gz')
+
+
+logger = logging.getLogger()
+
+
+def fetch_covtype(data_home=None, download_if_missing=True,
+                  random_state=None, shuffle=False):
+    """Load the covertype dataset, downloading it if necessary.
+
+    Parameters
+    ----------
+    data_home : string, optional
+        Specify another download and cache folder for the datasets. By default
+        all scikit learn data is stored in '~/scikit_learn_data' subfolders.
+
+    download_if_missing: boolean, optional
+        If False, raise a IOError if the data is not locally available
+        instead of trying to download the data from the source site.
+
+    dtype: dtype specifier, optional
+        dtype of feature array.
+    """
+
+    data_home = get_data_home(data_home=data_home)
+    covtype_dir = join(data_home, "covertype")
+    samples_path = join(covtype_dir, "samples")
+    targets_path = join(covtype_dir, "targets")
+    available = exists(samples_path)
+
+    if download_if_missing and not available:
+        _mkdirp(covtype_dir)
+        logger.warn("Downloading %s" % URL)
+        f = BytesIO(urlopen(URL).read())
+        Xy = np.genfromtxt(GzipFile(fileobj=f), delimiter=',', dtype=dtype)
+
+        X = Xy[:, :-1]
+        y = Xy[:, -1].astype(np.int32)
+
+        joblib.dump(X, samples_path, compress=9)
+        joblib.dump(y, targets_path, compress=9)
+
+    try:
+       X, y
+    except NameError:
+        X = joblib.load(samples_path)
+        y = joblib.load(targets_path)
+
+    if shuffle:
+        ind = np.arange(X.shape[0])
+        rng = check_random_state(random_state)
+        rng.shuffle(ind)
+        X = X[ind]
+        y = y[ind]
+
+    return Bunch(data=X, target=y, DESCR=__doc__)
+
+
+def _mkdirp(d):
+    """Ensure directory d exists (like mkdir -p on Unix)
+    No guarantee that the directory is writable.
+    """
+    try:
+        os.makedirs(d)
+    except OSError as e:
+        if e.errno != errno.EEXIST:
+            raise