Unpin spark requirements (#2661)

Author: tamargrey

.github/workflows/auto_approve_dependency_PRs.yaml

@@ -4,7 +4,7 @@ on:
       - cron: '*/30 * * * *'
   workflow_dispatch:
   workflow_run:
-    workflows: ["Unit Tests - Latest Dependencies", "Unit Tests - 3.8 Minimum Dependencies"]
+    workflows: ["Unit Tests - Latest Dependencies", "Unit Tests - 3.9 Minimum Dependencies"]
     branches:
       - 'latest-dep-update-[a-f0-9]+'
       - 'min-dep-update-[a-f0-9]+'

.github/workflows/build_docs.yaml

@@ -16,7 +16,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.8", "3.10"]
+        python_version: ["3.9", "3.10", "3.11"]
     steps:
       - name: Checkout repository
         uses: actions/checkout@v3

.github/workflows/install_test.yaml

@@ -14,7 +14,7 @@ jobs:
       fail-fast: false
       matrix:
         os: [ubuntu-latest, macos-latest, windows-latest]
-        python_version: ["3.8", "3.9", "3.10"]
+        python_version: ["3.9", "3.10"]
         exclude:
           - python_version: "3.10"
             os: macos-latest

.github/workflows/latest_dependency_checker.yaml

@@ -19,7 +19,7 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Update dependencies
         run: |
           python -m pip install --upgrade pip

.github/workflows/looking_glass_airflow_performance_tests.yaml

@@ -66,7 +66,7 @@ jobs:
                     "datasets-bucket-name-s3": "featuretools-performance-tests",
                     "results-bucket-name-s3": "featuretools-performance-results"
                   },
-                  "python_version": "3.8",
+                  "python_version": "3.9",
                   "scenarios_yaml": "entityset_scenarios_${{ matrix.test_type }}.yaml",
                   "featuretools_branch_previous": "${{ env.PREVIOUS_HASH }}",
                   "featuretools_branch_new": "${{ env.CURRENT_HASH }}",

.github/workflows/tests_with_latest_deps.yaml

@@ -13,7 +13,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.8", "3.9", "3.10", "3.11"]
+        python_version: ["3.9", "3.10", "3.11"]
         libraries: ["core", "spark/dask - misc", "spark/dask - computational", "spark/dask - entityset_1", "spark/dask - entityset_2", "spark/dask - primitives"]
 
     steps:
@@ -45,7 +45,7 @@ jobs:
           python -m pip install -e unpacked_sdist/[dask]
           python -m pip install -e unpacked_sdist/[spark]
           python -m pip install -e unpacked_sdist/[test]
-      - if: ${{ matrix.python_version == 3.8 && startsWith(matrix.libraries, 'spark/dask') }}
+      - if: ${{ matrix.python_version == 3.9 && startsWith(matrix.libraries, 'spark/dask') }}
         name: Generate coverage args
         run: echo "coverage_args=--cov=featuretools --cov-config=../pyproject.toml --cov-report=xml:../coverage.xml" >> $GITHUB_ENV
       - if: ${{ env.coverage_args }}
@@ -99,7 +99,7 @@ jobs:
     strategy:
       fail-fast: false
       matrix:
-        python_version: ["3.8", "3.9", "3.10", "3.11"]
+        python_version: ["3.9", "3.10", "3.11"]
     steps:
       - name: Download miniconda
         shell: pwsh

.github/workflows/tests_with_minimum_deps.yaml

@@ -8,7 +8,7 @@ on:
   workflow_dispatch:
 jobs:
   py38_tests_minimum_dependencies:
-    name: Tests - 3.8 Minimum Dependencies
+    name: Tests - 3.9 Minimum Dependencies
     runs-on: ubuntu-latest
     strategy:
       fail-fast: false
@@ -22,7 +22,7 @@ jobs:
           repository: ${{ github.event.pull_request.head.repo.full_name }}
       - uses: actions/setup-python@v4
         with:
-          python-version: 3.8
+          python-version: 3.9
       - name: Config pip, upgrade pip, and install graphviz
         run: |
           sudo apt update

.readthedocs.yaml

@@ -15,7 +15,7 @@ formats: []
 build:
   os: "ubuntu-22.04"
   tools:
-    python: "3.8"
+    python: "3.9"
   apt_packages:
     - graphviz
     - openjdk-11-jre-headless

docs/notebook_version_standardizer.py

@@ -75,7 +75,7 @@ def _fix_execution_and_output(notebook):
         json.dump(source, f, ensure_ascii=False, indent=1)
 
 
-def _get_notebooks_with_executions_and_empty(notebooks, default_version="3.8.2"):
+def _get_notebooks_with_executions_and_empty(notebooks, default_version="3.9.2"):
     executed = []
     empty_last_cell = []
     versions = []
@@ -89,7 +89,7 @@ def _get_notebooks_with_executions_and_empty(notebooks, default_version="3.8.2")
     return (executed, empty_last_cell, versions)
 
 
-def _fix_versions(notebooks, default_version="3.8.2"):
+def _fix_versions(notebooks, default_version="3.9.2"):
     for notebook in notebooks:
         _fix_python_version(notebook, default_version)
 

docs/source/api_reference.rst

@@ -28,17 +28,6 @@ Deep Feature Synthesis
     dfs
     get_valid_primitives
 
-Wrappers
-~~~~~~~~
-.. currentmodule:: featuretools
-
-scikit-learn (BETA)
--------------------
-.. autosummary::
-    :toctree: generated/
-
-    wrappers.DFSTransformer
-
 Timedelta
 ~~~~~~~~~
 .. currentmodule:: featuretools

docs/source/getting_started/afe.ipynb

@@ -249,7 +249,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/getting_started/handling_time.ipynb

@@ -710,7 +710,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/getting_started/primitives.ipynb

@@ -426,7 +426,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/getting_started/using_entitysets.ipynb

@@ -354,7 +354,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/getting_started/woodwork_types.ipynb

@@ -377,7 +377,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/guides/advanced_custom_primitives.ipynb

@@ -254,7 +254,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/guides/deployment.ipynb

@@ -202,7 +202,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/guides/feature_descriptions.ipynb

@@ -384,7 +384,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/guides/feature_selection.ipynb

@@ -394,7 +394,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/guides/performance.ipynb

@@ -210,7 +210,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/guides/specifying_primitive_options.ipynb

@@ -249,7 +249,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/guides/sql_database_integration.ipynb

@@ -137,7 +137,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   },
   "vscode": {
    "interpreter": {

docs/source/guides/time_series.ipynb

@@ -273,7 +273,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/guides/tuning_dfs.ipynb

@@ -245,7 +245,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/guides/using_dask_entitysets.ipynb

@@ -248,7 +248,10 @@
     "\n",
     "The use of `featuretools.remove_low_information_features()` cannot currently be used with a Dask feature matrix.\n",
     "\n",
-    "When manually defining a `Feature`, the `use_previous` parameter cannot be used if this feature will be applied to calculate a feature matrix from a Dask `EntitySet`."
+    "When manually defining a `Feature`, the `use_previous` parameter cannot be used if this feature will be applied to calculate a feature matrix from a Dask `EntitySet`.\n",
+    "\n",
+    "### Dask `string[pyarrow]`\n",
+    "Featuretools may have issues with the new string storage model used by Dask.  To workaround this, add `dask.config.set({'dataframe.convert-string': False})`, prior to running dask operations."
    ]
   }
  ],
@@ -269,7 +272,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/guides/using_spark_entitysets.ipynb

@@ -269,7 +269,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/index.ipynb

@@ -364,7 +364,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

docs/source/install.md

@@ -1,6 +1,6 @@
 # Install
 
-Featuretools is available for Python 3.8 - 3.11. It can be installed from [pypi](https://pypi.org/project/featuretools/), [conda-forge](https://anaconda.org/conda-forge/featuretools), or from [source](https://github.com/alteryx/featuretools).
+Featuretools is available for Python 3.9 - 3.11. It can be installed from [pypi](https://pypi.org/project/featuretools/), [conda-forge](https://anaconda.org/conda-forge/featuretools), or from [source](https://github.com/alteryx/featuretools).
 
 To install Featuretools, run the following command:
 
@@ -201,7 +201,7 @@ You can do so by installing it as a package inside a container (following the no
 creating a new image with Featuretools pre-installed, using the following commands in your `Dockerfile`:
 
 ```dockerfile
-FROM --platform=linux/x86_64 python:3.8-slim-buster
+FROM --platform=linux/x86_64 python:3.9-slim-buster
 RUN apt update && apt -y update
 RUN apt install -y build-essential
 RUN pip3 install --upgrade --quiet pip

docs/source/release_notes.rst

@@ -5,12 +5,16 @@ Release Notes
 
 Future Release
 ==============
+    .. warning::
+        This release of Featuretools will not support Python 3.8
+
     * Enhancements
     * Fixes
         * Fix dependency issues (:pr:`2644`, :pr:`2656`)
-        * Add workaround for pandas 2.2.0 bug with nunique and unpin pandas (:pr:`2657`)
+        * Add workaround for pandas 2.2.0 bug with nunique and unpin pandas deps (:pr:`2657`)
     * Changes
         * Fix deprecation warnings with is_categorical_dtype (:pr:`2641`)
+        * Remove woodwork, pyarrow, numpy, and pandas pins for spark installation (:pr:`2661`)
     * Documentation Changes
         * Update Featuretools logo to display properly in dark mode (:pr:`2632`)
     * Testing Changes

docs/source/resources/frequently_asked_questions.ipynb

@@ -2210,7 +2210,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   },
   "mimetype": "text/x-python",
   "name": "python",

docs/source/resources/transition_to_ft_v1.0.ipynb

@@ -861,7 +861,7 @@
    "name": "python",
    "nbconvert_exporter": "python",
    "pygments_lexer": "ipython3",
-   "version": "3.8.2"
+   "version": "3.9.2"
   }
  },
  "nbformat": 4,

featuretools/entityset/entityset.py

@@ -1269,11 +1269,15 @@ def add_last_time_indexes(self, updated_dataframes=None):
                         lti_df = pd.Series([], dtype="object")
                     else:
                         if lti_is_spark:
-                            lti_df["last_time"] = ps.to_datetime(lti_df["last_time"])
-                            lti_df["last_time_old"] = ps.to_datetime(
-                                lti_df["last_time_old"],
-                            )
                             # TODO: Figure out a workaround for fillna and replace
+                            if lti_df["last_time_old"].dtype != "datetime64[ns]":
+                                lti_df["last_time_old"] = ps.to_datetime(
+                                    lti_df["last_time_old"],
+                                )
+                            if lti_df["last_time"].dtype != "datetime64[ns]":
+                                lti_df["last_time"] = ps.to_datetime(
+                                    lti_df["last_time"],
+                                )
                             lti_df = lti_df.max(axis=1)
                         else:
                             lti_df["last_time"] = lti_df["last_time"].astype(

featuretools/primitives/standard/aggregation/num_unique.py

@@ -60,6 +60,6 @@ def finalize(s):
 
             return dd.Aggregation(self.name, chunk=chunk, agg=agg, finalize=finalize)
 
-        if self.use_string_for_pd_calc:
+        if self.use_string_for_pd_calc or agg_type == Library.SPARK:
             return "nunique"
         return pd.Series.nunique

featuretools/tests/conftest.py

@@ -173,6 +173,8 @@ def pd_latlong_df():
 
 @pytest.fixture
 def dask_latlong_df(pd_latlong_df):
+    dask = pytest.importorskip("dask", reason="Dask not installed, skipping")
+    dask.config.set({"dataframe.convert-string": False})
     dd = pytest.importorskip("dask.dataframe", reason="Dask not installed, skipping")
     return dd.from_pandas(pd_latlong_df.reset_index(drop=True), npartitions=4)
 

featuretools/tests/entityset_tests/test_ww_es.py

@@ -748,6 +748,8 @@ def latlong_df(request):
 
 
 def test_replace_dataframe_data_transformation(latlong_df):
+    dask = pytest.importorskip("dask", reason="Dask not installed, skipping")
+    dask.config.set({"dataframe.convert-string": False})
     initial_df = latlong_df.copy()
     initial_df.ww.init(
         name="latlongs",