Moves permutation importance from evalml/pipeline/utils.py to graph_utils.py

docs/source/api_reference.rst

@@ -137,7 +137,6 @@ Pipeline Utils
     get_estimators
     make_pipeline
     list_model_families
-    calculate_permutation_importances
 
 
 Pipeline Graph Utils
@@ -153,6 +152,7 @@ Pipeline Graph Utils
     confusion_matrix
     normalize_confusion_matrix
     graph_confusion_matrix
+    calculate_permutation_importances
     graph_permutation_importances
 
 .. currentmodule:: evalml.pipelines.components

docs/source/changelog.rst

@@ -14,7 +14,7 @@ Changelog
         * Added preprocessing component to handle DateTime columns featurization :pr:`838`
         * Added ability to clone pipelines and components :pr:`842`
         * Define getter method for component `parameters` :pr:`847`
-        * Added utility methods to calculate and graph permutation importances :pr:`860`
+        * Added utility methods to calculate and graph permutation importances :pr:`860`, :pr:`880`
         * Added new utility functions necessary for generating dynamic preprocessing pipelines :pr:`852`
         * Added kwargs to all components :pr:`863`
         * Added SelectColumns transformer :pr:`873`

evalml/pipelines/__init__.py

@@ -65,9 +65,7 @@
     list_model_families,
     all_estimators,
     get_estimators,
-    make_pipeline,
-    calculate_permutation_importances,
-    graph_permutation_importances
+    make_pipeline
 )
 from .graph_utils import (
     precision_recall_curve,
@@ -77,4 +75,6 @@
     confusion_matrix,
     normalize_confusion_matrix,
     graph_confusion_matrix,
+    calculate_permutation_importances,
+    graph_permutation_importances
 )

evalml/pipelines/graph_utils.py

@@ -2,6 +2,8 @@
 
 import numpy as np
 import pandas as pd
+from sklearn.inspection import \
+    permutation_importance as sk_permutation_importance
 from sklearn.metrics import auc as sklearn_auc
 from sklearn.metrics import confusion_matrix as sklearn_confusion_matrix
 from sklearn.metrics import \
@@ -10,6 +12,7 @@
 from sklearn.preprocessing import LabelBinarizer
 from sklearn.utils.multiclass import unique_labels
 
+from evalml.objectives import get_objective
 from evalml.utils import import_or_raise
 
 
@@ -236,3 +239,82 @@ def graph_confusion_matrix(y_true, y_pred, normalize_method='true', title_additi
     # plotly Heatmap y axis defaults to the reverse of what we want: https://community.plotly.com/t/heatmap-y-axis-is-reversed-by-default-going-against-standard-convention-for-matrices/32180
     fig.update_yaxes(autorange="reversed")
     return fig
+
+
+def calculate_permutation_importances(pipeline, X, y, objective, n_repeats=5, n_jobs=None, random_state=0):
+    """Calculates permutation importance for features.
+
+    Arguments:
+        pipeline (PipelineBase or subclass): fitted pipeline
+        X (pd.DataFrame): the input data used to score and compute permutation importance
+        y (pd.Series): the target labels
+        objective (str, ObjectiveBase): objective to score on
+        n_repeats (int): Number of times to permute a feature. Defaults to 5.
+        n_jobs (int or None): Non-negative integer describing level of parallelism used for pipelines.
+            None and 1 are equivalent. If set to -1, all CPUs are used. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used.
+        random_state (int, np.random.RandomState): The random seed/state. Defaults to 0.
+
+    Returns:
+        Mean feature importance scores over 5 shuffles.
+    """
+    objective = get_objective(objective)
+    if objective.problem_type != pipeline.problem_type:
+        raise ValueError(f"Given objective '{objective.name}' cannot be used with '{pipeline.name}'")
+
+    def scorer(pipeline, X, y):
+        scores = pipeline.score(X, y, objectives=[objective])
+        return scores[objective.name] if objective.greater_is_better else -scores[objective.name]
+    perm_importance = sk_permutation_importance(pipeline, X, y, n_repeats=n_repeats, scoring=scorer, n_jobs=n_jobs, random_state=random_state)
+    mean_perm_importance = perm_importance["importances_mean"]
+    if not isinstance(X, pd.DataFrame):
+        X = pd.DataFrame(X)
+    feature_names = list(X.columns)
+    mean_perm_importance = list(zip(feature_names, mean_perm_importance))
+    mean_perm_importance.sort(key=lambda x: x[1], reverse=True)
+    return pd.DataFrame(mean_perm_importance, columns=["feature", "importance"])
+
+
+def graph_permutation_importances(pipeline, X, y, objective, show_all_features=False):
+    """Generate a bar graph of the pipeline's permutation importance.
+
+    Arguments:
+        pipeline (PipelineBase or subclass): Fitted pipeline
+        X (pd.DataFrame): The input data used to score and compute permutation importance
+        y (pd.Series): The target labels
+        objective (str, ObjectiveBase): Objective to score on
+        show_all_features (bool, optional) : If True, graph features with a permutation importance value of zero. Defaults to False.
+
+    Returns:
+        plotly.Figure, a bar graph showing features and their respective permutation importance.
+    """
+    go = import_or_raise("plotly.graph_objects", error_msg="Cannot find dependency plotly.graph_objects")
+    perm_importance = calculate_permutation_importances(pipeline, X, y, objective)
+    perm_importance['importance'] = perm_importance['importance']
+
+    if not show_all_features:
+        # Remove features with close to zero importance
+        perm_importance = perm_importance[abs(perm_importance['importance']) >= 1e-3]
+    # List is reversed to go from ascending order to descending order
+    perm_importance = perm_importance.iloc[::-1]
+
+    title = "Permutation Importance"
+    subtitle = "The relative importance of each input feature's "\
+               "overall influence on the pipelines' predictions, computed using "\
+               "the permutation importance algorithm."
+    data = [go.Bar(x=perm_importance['importance'],
+                   y=perm_importance['feature'],
+                   orientation='h'
+                   )]
+
+    layout = {
+        'title': '{0}<br><sub>{1}</sub>'.format(title, subtitle),
+        'height': 800,
+        'xaxis_title': 'Permutation Importance',
+        'yaxis_title': 'Feature',
+        'yaxis': {
+            'type': 'category'
+        }
+    }
+
+    fig = go.Figure(data=data, layout=layout)
+    return fig

evalml/pipelines/utils.py

@@ -1,7 +1,5 @@
 import numpy as np
 import pandas as pd
-from sklearn.inspection import \
-    permutation_importance as sk_permutation_importance
 
 from .binary_classification_pipeline import BinaryClassificationPipeline
 from .classification import (
@@ -27,7 +25,6 @@
 
 from evalml.exceptions import MissingComponentError
 from evalml.model_family import handle_model_family
-from evalml.objectives import get_objective
 from evalml.pipelines.components import (
     CatBoostClassifier,
     CatBoostRegressor,
@@ -44,7 +41,7 @@
     XGBoostRegressor
 )
 from evalml.problem_types import ProblemTypes, handle_problem_types
-from evalml.utils import get_logger, import_or_raise
+from evalml.utils import get_logger
 
 logger = get_logger(__file__)
 
@@ -271,82 +268,3 @@ def get_pipeline_base_class(problem_type):
     class GeneratedPipeline(base_class):
         component_graph = complete_component_graph
     return GeneratedPipeline
-
-
-def calculate_permutation_importances(pipeline, X, y, objective, n_repeats=5, n_jobs=None, random_state=0):
-    """Calculates permutation importance for features.
-
-    Arguments:
-        pipeline (PipelineBase or subclass): fitted pipeline
-        X (pd.DataFrame): the input data used to score and compute permutation importance
-        y (pd.Series): the target labels
-        objective (str, ObjectiveBase): objective to score on
-        n_repeats (int): Number of times to permute a feature. Defaults to 5.
-        n_jobs (int or None): Non-negative integer describing level of parallelism used for pipelines.
-            None and 1 are equivalent. If set to -1, all CPUs are used. For n_jobs below -1, (n_cpus + 1 + n_jobs) are used.
-        random_state (int, np.random.RandomState): The random seed/state. Defaults to 0.
-
-    Returns:
-        Mean feature importance scores over 5 shuffles.
-    """
-    objective = get_objective(objective)
-    if objective.problem_type != pipeline.problem_type:
-        raise ValueError(f"Given objective '{objective.name}' cannot be used with '{pipeline.name}'")
-
-    def scorer(pipeline, X, y):
-        scores = pipeline.score(X, y, objectives=[objective])
-        return scores[objective.name] if objective.greater_is_better else -scores[objective.name]
-    perm_importance = sk_permutation_importance(pipeline, X, y, n_repeats=n_repeats, scoring=scorer, n_jobs=n_jobs, random_state=random_state)
-    mean_perm_importance = perm_importance["importances_mean"]
-    if not isinstance(X, pd.DataFrame):
-        X = pd.DataFrame(X)
-    feature_names = list(X.columns)
-    mean_perm_importance = list(zip(feature_names, mean_perm_importance))
-    mean_perm_importance.sort(key=lambda x: x[1], reverse=True)
-    return pd.DataFrame(mean_perm_importance, columns=["feature", "importance"])
-
-
-def graph_permutation_importances(pipeline, X, y, objective, show_all_features=False):
-    """Generate a bar graph of the pipeline's permutation importance.
-
-    Arguments:
-        pipeline (PipelineBase or subclass): Fitted pipeline
-        X (pd.DataFrame): The input data used to score and compute permutation importance
-        y (pd.Series): The target labels
-        objective (str, ObjectiveBase): Objective to score on
-        show_all_features (bool, optional) : If True, graph features with a permutation importance value of zero. Defaults to False.
-
-    Returns:
-        plotly.Figure, a bar graph showing features and their respective permutation importance.
-    """
-    go = import_or_raise("plotly.graph_objects", error_msg="Cannot find dependency plotly.graph_objects")
-    perm_importance = calculate_permutation_importances(pipeline, X, y, objective)
-    perm_importance['importance'] = perm_importance['importance']
-
-    if not show_all_features:
-        # Remove features with close to zero importance
-        perm_importance = perm_importance[abs(perm_importance['importance']) >= 1e-3]
-    # List is reversed to go from ascending order to descending order
-    perm_importance = perm_importance.iloc[::-1]
-
-    title = "Permutation Importance"
-    subtitle = "The relative importance of each input feature's "\
-               "overall influence on the pipelines' predictions, computed using "\
-               "the permutation importance algorithm."
-    data = [go.Bar(x=perm_importance['importance'],
-                   y=perm_importance['feature'],
-                   orientation='h'
-                   )]
-
-    layout = {
-        'title': '{0}<br><sub>{1}</sub>'.format(title, subtitle),
-        'height': 800,
-        'xaxis_title': 'Permutation Importance',
-        'yaxis_title': 'Feature',
-        'yaxis': {
-            'type': 'category'
-        }
-    }
-
-    fig = go.Figure(data=data, layout=layout)
-    return fig

evalml/tests/pipeline_tests/test_graphs.py

@@ -7,11 +7,18 @@
 import pytest
 from skopt.space import Real
 
-from evalml.pipelines import BinaryClassificationPipeline
-from evalml.pipelines.utils import (
+from evalml.objectives import get_objectives
+from evalml.pipelines import (
+    BinaryClassificationPipeline,
+    LinearRegressionPipeline,
+    LogisticRegressionBinaryPipeline,
+    LogisticRegressionMulticlassPipeline
+)
+from evalml.pipelines.graph_utils import (
     calculate_permutation_importances,
     graph_permutation_importances
 )
+from evalml.problem_types import ProblemTypes
 
 
 @pytest.fixture
@@ -107,6 +114,63 @@ def test_graph_feature_importances_show_all_features(X_y, test_pipeline):
     assert (np.any(data['x'] == 0.0))
 
 
+def test_get_permutation_importance_invalid_objective(X_y_reg):
+    X, y = X_y_reg
+    pipeline = LinearRegressionPipeline(parameters={}, random_state=np.random.RandomState(42))
+    with pytest.raises(ValueError, match=f"Given objective 'MCC Multiclass' cannot be used with '{pipeline.name}'"):
+        calculate_permutation_importances(pipeline, X, y, "mcc_multi")
+
+
+@pytest.mark.parametrize("data_type", ['np', 'pd'])
+def test_get_permutation_importance_binary(X_y, data_type):
+    X, y = X_y
+    if data_type == 'pd':
+        X = pd.DataFrame(X)
+        y = pd.Series(y)
+    pipeline = LogisticRegressionBinaryPipeline(parameters={}, random_state=np.random.RandomState(42))
+    pipeline.fit(X, y)
+    for objective in get_objectives(ProblemTypes.BINARY):
+        permutation_importances = calculate_permutation_importances(pipeline, X, y, objective)
+        assert list(permutation_importances.columns) == ["feature", "importance"]
+        assert not permutation_importances.isnull().all().all()
+
+
+def test_get_permutation_importance_multiclass(X_y_multi):
+    X, y = X_y_multi
+    pipeline = LogisticRegressionMulticlassPipeline(parameters={}, random_state=np.random.RandomState(42))
+    pipeline.fit(X, y)
+    for objective in get_objectives(ProblemTypes.MULTICLASS):
+        permutation_importances = calculate_permutation_importances(pipeline, X, y, objective)
+        assert list(permutation_importances.columns) == ["feature", "importance"]
+        assert not permutation_importances.isnull().all().all()
+
+
+def test_get_permutation_importance_regression(X_y_reg):
+    X, y = X_y_reg
+    pipeline = LinearRegressionPipeline(parameters={}, random_state=np.random.RandomState(42))
+    pipeline.fit(X, y)
+    for objective in get_objectives(ProblemTypes.REGRESSION):
+        permutation_importances = calculate_permutation_importances(pipeline, X, y, objective)
+        assert list(permutation_importances.columns) == ["feature", "importance"]
+        assert not permutation_importances.isnull().all().all()
+
+
+def test_get_permutation_importance_correlated_features():
+    y = pd.Series([1, 0, 1, 1])
+    X = pd.DataFrame()
+    X["correlated"] = y * 2
+    X["not correlated"] = [-1, -1, -1, 0]
+    y = y.astype(bool)
+    pipeline = LogisticRegressionBinaryPipeline(parameters={}, random_state=np.random.RandomState(42))
+    pipeline.fit(X, y)
+    importances = calculate_permutation_importances(pipeline, X, y, objective="log_loss_binary", random_state=0)
+    assert list(importances.columns) == ["feature", "importance"]
+    assert not importances.isnull().all().all()
+    correlated_importance_val = importances["importance"][importances.index[importances["feature"] == "correlated"][0]]
+    not_correlated_importance_val = importances["importance"][importances.index[importances["feature"] == "not correlated"][0]]
+    assert correlated_importance_val > not_correlated_importance_val
+
+
 def test_graph_permutation_importances(X_y, test_pipeline):
     go = pytest.importorskip('plotly.graph_objects', reason='Skipping plotting test because plotly not installed')
     X, y = X_y
@@ -126,7 +190,7 @@ def test_graph_permutation_importances(X_y, test_pipeline):
     assert np.array_equal(fig_dict['data'][0]['y'][::-1], perm_importance_data['feature'])
 
 
-@patch('evalml.pipelines.utils.calculate_permutation_importances')
+@patch('evalml.pipelines.graph_utils.calculate_permutation_importances')
 def test_graph_permutation_importances_show_all_features(mock_perm_importances):
     go = pytest.importorskip('plotly.graph_objects', reason='Skipping plotting test because plotly not installed')
     mock_perm_importances.return_value = pd.DataFrame({"feature": ["f1", "f2"], "importance": [0.0, 0.6]})