BUG: Build random forests the same way regardless of n_jobs and add a test for this. Don't predict in parallel since the cost of copying memory in joblib outweighs the speedups for random forests. Fixes scikit-learn#1685.

Author: erg

sklearn/ensemble/forest.py

@@ -65,12 +65,12 @@ class calls the ``fit`` method of each sub-estimator on random samples
 
 
 def _parallel_build_trees(n_trees, forest, X, y, sample_weight,
-                          sample_mask, X_argsorted, seed, verbose):
+                          sample_mask, X_argsorted, seeds, verbose):
     """Private function used to build a batch of trees within a job."""
-    random_state = check_random_state(seed)
     trees = []
 
     for i in range(n_trees):
+        random_state = check_random_state(seeds[i])
         if verbose > 1:
             print("building tree %d of %d" % (i + 1, n_trees))
         seed = random_state.randint(MAX_INT)
@@ -356,6 +356,9 @@ def fit(self, X, y, sample_weight=None):
         # Assign chunk of trees to jobs
         n_jobs, n_trees, _ = _partition_trees(self)
 
+        # Precalculate the random states
+        seeds = [random_state.randint(MAX_INT, size=n_trees[i]) for i in xrange(len(n_trees))]
+
         # Parallel loop
         all_trees = Parallel(n_jobs=n_jobs, verbose=self.verbose)(
             delayed(_parallel_build_trees)(
@@ -366,7 +369,7 @@ def fit(self, X, y, sample_weight=None):
                 sample_weight,
                 sample_mask,
                 X_argsorted,
-                random_state.randint(MAX_INT),
+                seeds[i],
                 verbose=self.verbose)
             for i in range(n_jobs))
 
@@ -563,32 +566,17 @@ def predict_proba(self, X):
         if getattr(X, "dtype", None) != DTYPE or X.ndim != 2:
             X = array2d(X, dtype=DTYPE)
 
-        # Assign chunk of trees to jobs
-        n_jobs, n_trees, starts = _partition_trees(self)
-
-        # Parallel loop
-        all_proba = Parallel(n_jobs=n_jobs, verbose=self.verbose)(
-            delayed(_parallel_predict_proba)(
-                self.estimators_[starts[i]:starts[i + 1]],
+        # Running with n_jobs > 1 is slower
+        proba = _parallel_predict_proba(
+                self.estimators_,
                 X,
                 self.n_classes_,
                 self.n_outputs_)
-            for i in range(n_jobs))
-
-        # Reduce
-        proba = all_proba[0]
 
         if self.n_outputs_ == 1:
-            for j in xrange(1, len(all_proba)):
-                proba += all_proba[j]
-
             proba /= self.n_estimators
 
         else:
-            for j in xrange(1, len(all_proba)):
-                for k in xrange(self.n_outputs_):
-                    proba[k] += all_proba[j][k]
-
             for k in xrange(self.n_outputs_):
                 proba[k] /= self.n_estimators
 
@@ -674,17 +662,8 @@ def predict(self, X):
         if getattr(X, "dtype", None) != DTYPE or X.ndim != 2:
             X = array2d(X, dtype=DTYPE)
 
-        # Assign chunk of trees to jobs
-        n_jobs, n_trees, starts = _partition_trees(self)
-
-        # Parallel loop
-        all_y_hat = Parallel(n_jobs=n_jobs, verbose=self.verbose)(
-            delayed(_parallel_predict_regression)(
-                self.estimators_[starts[i]:starts[i + 1]], X)
-            for i in range(n_jobs))
-
-        # Reduce
-        y_hat = sum(all_y_hat) / self.n_estimators
+        y_hat = _parallel_predict_regression(self.estimators_, X)
+        y_hat /= self.n_estimators
 
         return y_hat
 

sklearn/ensemble/tests/test_forest.py

@@ -396,12 +396,13 @@ def test_random_hasher():
     # test random forest hashing on circles dataset
     # make sure that it is linearly separable.
     # even after projected to two pca dimensions
-    hasher = RandomTreesEmbedding(n_estimators=30, random_state=0)
+    # Note: Not all random_states produce perfect results.
+    hasher = RandomTreesEmbedding(n_estimators=30, random_state=1)
     X, y = datasets.make_circles(factor=0.5)
     X_transformed = hasher.fit_transform(X)
 
     # test fit and transform:
-    hasher = RandomTreesEmbedding(n_estimators=30, random_state=0)
+    hasher = RandomTreesEmbedding(n_estimators=30, random_state=1)
     assert_array_equal(hasher.fit(X).transform(X).toarray(),
                        X_transformed.toarray())
 
@@ -415,6 +416,33 @@ def test_random_hasher():
     assert_equal(linear_clf.score(X_reduced, y), 1.)
 
 
+def test_parallel_train():
+    rng = np.random.RandomState(12321)
+    
+    X = rng.randn(100, 1000)
+    y = rng.randint(0, 2, 100)
+
+    clfs = [
+        RandomForestClassifier(n_estimators=20,
+                               n_jobs=n_jobs,
+                               random_state=12345)
+        for n_jobs in range(1, 9)
+    ]
+
+    for clf in clfs:
+        clf.fit(X, y)
+
+    X2 = rng.randn(100, 1000)
+
+    probas = []
+    for clf in clfs:
+        proba = clf.predict_proba(X2)
+        probas.append(proba)
+
+    for proba1, proba2 in zip(probas, probas[1:]):
+        assert np.allclose(proba1, proba2)
+
+
 if __name__ == "__main__":
     import nose
     nose.runmodule()