Default n_jobs to 1 for StackedEnsemblers until we can write our own Ensembler (#2295)

* Default n_jobs to None for StackedEnsemblers

* Release notes

* release notes

* test update

* test update

* test update

* test update

* don't mock fit and test n_jobs

* update test

* update release notes

* test updates

* add multiclass coverage to test

* cleaner logic

* improve test parametrization

* lint fixes

* change estimators for core dependencies true

Author: ParthivNaresh

docs/source/release_notes.rst

@@ -6,6 +6,7 @@ Release Notes
         * Added dictionary input functionality for ``Undersampler`` component :pr:`2271`
         * Changed the default parameter values for ``Elastic Net Classifier`` and ``Elastic Net Regressor`` :pr:`2269`
     * Fixes
+        * Set default `n_jobs` to 1 for `StackedEnsembleClassifier` and `StackedEnsembleRegressor` until fix for text-based parallelism in sklearn stacking can be found :pr:`2295`
     * Changes
         * Updated ``start_iteration_callback`` to accept a pipeline instance instead of a pipeline class and no longer accept pipeline parameters as a parameter :pr:`2290`
     * Documentation Changes

evalml/automl/automl_algorithm/iterative_algorithm.py

@@ -33,6 +33,7 @@ def __init__(self,
                  n_jobs=-1,  # TODO remove
                  number_features=None,  # TODO remove
                  ensembling=False,
+                 text_in_ensembling=False,
                  pipeline_params=None,
                  _frozen_pipeline_parameters=None,
                  _estimator_family_order=None):
@@ -47,6 +48,7 @@ def __init__(self,
             n_jobs (int or None): Non-negative integer describing level of parallelism used for pipelines.
             number_features (int): The number of columns in the input features.
             ensembling (boolean): If True, runs ensembling in a separate batch after every allowed pipeline class has been iterated over. Defaults to False.
+            text_in_ensembling (boolean): If True and ensembling is True, then n_jobs will be set to 1 to avoid downstream sklearn stacking issues related to nltk.
             pipeline_params (dict or None): Pipeline-level parameters that should be passed to the proposed pipelines.
             _frozen_pipeline_parameters (dict or None): Pipeline-level parameters are frozen and used in the proposed pipelines.
             _estimator_family_order (list(ModelFamily) or None): specify the sort order for the first batch. Defaults to _ESTIMATOR_FAMILY_ORDER.
@@ -75,6 +77,7 @@ def __init__(self,
         self._first_batch_results = []
         self._best_pipeline_info = {}
         self.ensembling = ensembling and len(self.allowed_pipelines) > 1
+        self.text_in_ensembling = text_in_ensembling
         self._pipeline_params = pipeline_params or {}
         self._frozen_pipeline_parameters = _frozen_pipeline_parameters or {}
 
@@ -105,9 +108,10 @@ def next_batch(self):
                 parameters = self._combine_parameters(pipeline, pipeline_params)
                 input_pipelines.append(pipeline.new(parameters=parameters,
                                                     random_seed=self.random_seed))
+            n_jobs_ensemble = 1 if self.text_in_ensembling else self.n_jobs
             ensemble = _make_stacked_ensemble_pipeline(input_pipelines, input_pipelines[0].problem_type,
                                                        random_seed=self.random_seed,
-                                                       n_jobs=self.n_jobs)
+                                                       n_jobs=n_jobs_ensemble)
 
             next_batch.append(ensemble)
         else:

evalml/automl/automl_search.py

@@ -385,6 +385,7 @@ def __init__(self,
         check_all_pipeline_names_unique(self.allowed_pipelines)
 
         run_ensembling = self.ensembling
+        text_in_ensembling = len(infer_feature_types(X_train).select('natural_language').columns) > 0
         if run_ensembling and len(self.allowed_pipelines) == 1:
             logger.warning("Ensembling is set to True, but the number of unique pipelines is one, so ensembling will not run.")
             run_ensembling = False
@@ -438,6 +439,7 @@ def __init__(self,
             number_features=self.X_train.shape[1],
             pipelines_per_batch=self._pipelines_per_batch,
             ensembling=run_ensembling,
+            text_in_ensembling=text_in_ensembling,
             pipeline_params=parameters,
             _frozen_pipeline_parameters=self._frozen_pipeline_parameters
         )

evalml/tests/automl_tests/test_automl.py

@@ -1899,6 +1899,49 @@ def test_search_with_text(mock_fit, mock_score):
     assert automl.rankings['pipeline_name'][1:].str.contains('Text').all()
 
 
+@pytest.mark.parametrize("problem_type,pipeline_name,ensemble_name",
+                         [('binary', 'Stacked Ensemble Classification Pipeline', 'Stacked Ensemble Classifier'),
+                          ('multiclass', 'Stacked Ensemble Classification Pipeline', 'Stacked Ensemble Classifier'),
+                          ('regression', 'Stacked Ensemble Regression Pipeline', 'Stacked Ensemble Regressor')])
+@pytest.mark.parametrize("df_text", [True, False])
+@patch('evalml.automl.automl_algorithm.IterativeAlgorithm.__init__')
+def test_search_with_text_and_ensembling(mock_iter, df_text, problem_type, pipeline_name, ensemble_name):
+    X_with_text = pd.DataFrame(
+        {'col_1': ['I\'m singing in the rain! Just singing in the rain, what a glorious feeling, I\'m happy again!',
+                   'In sleep he sang to me, in dreams he came... That voice which calls to me, and speaks my name.',
+                   'I\'m gonna be the main event, like no king was before! I\'m brushing up on looking down, I\'m working on my ROAR!',
+                   'In sleep he sang to me, in dreams he came... That voice which calls to me, and speaks my name.',
+                   'In sleep he sang to me, in dreams he came... That voice which calls to me, and speaks my name.',
+                   'I\'m singing in the rain! Just singing in the rain, what a glorious feeling, I\'m happy again!',
+                   'do you hear the people sing? Singing the songs of angry men\n\tIt is the music of a people who will NOT be slaves again!',
+                   'I dreamed a dream in days gone by, when hope was high and life worth living',
+                   'Red, the blood of angry men - black, the dark of ages past',
+                   'do you hear the people sing? Singing the songs of angry men\n\tIt is the music of a people who will NOT be slaves again!',
+                   'Red, the blood of angry men - black, the dark of ages past',
+                   'It was red and yellow and green and brown and scarlet and black and ochre and peach and ruby and olive and violet and fawn...']
+         })
+    X_no_text = pd.DataFrame({'col_1': [1, 1, 1, 1, 1, 2, 2, 2, 2, 2, 3, 3]})
+
+    if df_text:
+        X = X_with_text
+    else:
+        X = X_no_text
+    if problem_type == 'binary':
+        y = [0, 1, 1, 0, 1, 0, 0, 1, 1, 0, 1, 0]
+    elif problem_type == 'multiclass':
+        y = [0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2]
+    else:
+        y = [1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12]
+    mock_iter.return_value = None
+    _ = AutoMLSearch(X_train=X, y_train=y, problem_type=problem_type, allowed_model_families=["random_forest", "decision_tree"],
+                     max_batches=4, ensembling=True)
+    call_args = mock_iter.call_args_list[0][1]
+    if df_text:
+        assert call_args['text_in_ensembling']
+    else:
+        assert not call_args['text_in_ensembling']
+
+
 @patch('evalml.pipelines.BinaryClassificationPipeline.score', return_value={"Log Loss Binary": 0.8})
 @patch('evalml.pipelines.BinaryClassificationPipeline.fit')
 def test_pipelines_per_batch(mock_fit, mock_score, X_y_binary):

evalml/tests/automl_tests/test_iterative_algorithm.py

@@ -236,10 +236,11 @@ def test_iterative_algorithm_one_allowed_pipeline(ensembling_value, logistic_reg
         assert any([p != logistic_regression_binary_pipeline_class({}).default_parameters for p in all_parameters])
 
 
+@pytest.mark.parametrize("text_in_ensembling", [True, False])
 @pytest.mark.parametrize("n_jobs", [-1, 0, 1, 2, 3])
-def test_iterative_algorithm_stacked_ensemble_n_jobs_binary(n_jobs, dummy_binary_pipeline_classes):
+def test_iterative_algorithm_stacked_ensemble_n_jobs_binary(n_jobs, text_in_ensembling, dummy_binary_pipeline_classes):
     dummy_binary_pipeline_classes = dummy_binary_pipeline_classes()
-    algo = IterativeAlgorithm(allowed_pipelines=dummy_binary_pipeline_classes, ensembling=True, n_jobs=n_jobs)
+    algo = IterativeAlgorithm(allowed_pipelines=dummy_binary_pipeline_classes, ensembling=True, text_in_ensembling=text_in_ensembling, n_jobs=n_jobs)
     next_batch = algo.next_batch()
     seen_ensemble = False
     scores = range(0, len(next_batch))
@@ -251,13 +252,17 @@ def test_iterative_algorithm_stacked_ensemble_n_jobs_binary(n_jobs, dummy_binary
         for pipeline in next_batch:
             if isinstance(pipeline.estimator, StackedEnsembleClassifier):
                 seen_ensemble = True
-                assert pipeline.parameters['Stacked Ensemble Classifier']['n_jobs'] == n_jobs
+                if text_in_ensembling:
+                    assert pipeline.parameters['Stacked Ensemble Classifier']['n_jobs'] == 1
+                else:
+                    assert pipeline.parameters['Stacked Ensemble Classifier']['n_jobs'] == n_jobs
     assert seen_ensemble
 
 
+@pytest.mark.parametrize("text_in_ensembling", [True, False])
 @pytest.mark.parametrize("n_jobs", [-1, 0, 1, 2, 3])
-def test_iterative_algorithm_stacked_ensemble_n_jobs_regression(n_jobs, linear_regression_pipeline_class):
-    algo = IterativeAlgorithm(allowed_pipelines=[linear_regression_pipeline_class({}), linear_regression_pipeline_class({})], ensembling=True, n_jobs=n_jobs)
+def test_iterative_algorithm_stacked_ensemble_n_jobs_regression(n_jobs, text_in_ensembling, linear_regression_pipeline_class):
+    algo = IterativeAlgorithm(allowed_pipelines=[linear_regression_pipeline_class({}), linear_regression_pipeline_class({})], ensembling=True, text_in_ensembling=text_in_ensembling, n_jobs=n_jobs)
     next_batch = algo.next_batch()
     seen_ensemble = False
     scores = range(0, len(next_batch))
@@ -269,7 +274,10 @@ def test_iterative_algorithm_stacked_ensemble_n_jobs_regression(n_jobs, linear_r
         for pipeline in next_batch:
             if isinstance(pipeline.estimator, StackedEnsembleRegressor):
                 seen_ensemble = True
-                assert pipeline.parameters['Stacked Ensemble Regressor']['n_jobs'] == n_jobs
+                if text_in_ensembling:
+                    assert pipeline.parameters['Stacked Ensemble Regressor']['n_jobs'] == 1
+                else:
+                    assert pipeline.parameters['Stacked Ensemble Regressor']['n_jobs'] == n_jobs
     assert seen_ensemble