Revert "Revert "Unpin pyspark"" (mlflow#10148)

Signed-off-by: harupy <[email protected]>
Signed-off-by: B-Step62 <[email protected]>
Co-authored-by: Yuki Watanabe <[email protected]>

Author: harupy

.github/actions/py-cache-key/action.yml

@@ -22,5 +22,5 @@ runs:
         # Refresh cache daily
         DATE=$(date -u "+%Y%m%d")
         # Change this value to force a cache refresh
-        N=0
+        N=1
         echo "value=$RUNNER_IMAGE-$PYTHON_VERSION-$DATE-$REQUIREMENTS_HASH-$N" >> $GITHUB_OUTPUT

.github/workflows/master.yml

@@ -109,8 +109,6 @@ jobs:
           python -m venv .venv
           source .venv/bin/activate
           source ./dev/install-common-deps.sh --ml
-          # pyspark 3.5 is incompatible with delta 2.4
-          pip install 'pyspark<3.5'
       - uses: ./.github/actions/pipdeptree
       - name: Import check
         run: |
@@ -353,8 +351,7 @@ jobs:
           pip install -r requirements/test-requirements.txt
           pip install --no-dependencies tests/resources/mlflow-test-plugin
           pip install -e .[extras]
-          # pyspark 3.5 is incompatible with delta 2.4
-          pip install 'pyspark<3.5'
+          pip install pyspark
           pip install mleap
           # Install Hugging Face datasets to test Hugging Face usage with MLflow dataset tracking
           pip install datasets

.github/workflows/recipe.yml

@@ -42,9 +42,7 @@ jobs:
       - name: Install dependencies
         run: |
           source ./dev/install-common-deps.sh
-          pip install -e .
-          # pyspark 3.5 is incompatible with delta 2.4
-          pip install 'pyspark<3.5'
+          pip install pyspark
       - name: Run tests
         run: |
           pytest tests/recipes
@@ -70,8 +68,7 @@ jobs:
           pip install -r requirements/test-requirements.txt
           pip install --no-dependencies tests/resources/mlflow-test-plugin
           pip install -e .
-          # pyspark 3.5 is incompatible with delta 2.4
-          pip install 'pyspark<3.5'
+          pip install pyspark
           # TODO: Importing datasets in a pandas UDF (created by mlflow.pyfunc.spark_udf) crashes
           # the Python worker. To avoid this, uninstall `datasets`. This is a temporary workaround.
           pip uninstall -y datasets

mlflow/utils/_spark_utils.py

@@ -46,7 +46,7 @@ def _create_local_spark_session_for_recipes():
     _prepare_subprocess_environ_for_creating_local_spark_session()
     return (
         SparkSession.builder.master("local[*]")
-        .config("spark.jars.packages", "io.delta:delta-core_2.12:2.4.0")
+        .config("spark.jars.packages", "io.delta:delta-spark_2.12:3.0.0")
         .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
         .config(
             "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog"

tests/data/test_delta_dataset_source.py

@@ -14,7 +14,7 @@ def spark_session():
 
     with (
         SparkSession.builder.master("local[*]")
-        .config("spark.jars.packages", "io.delta:delta-core_2.12:2.4.0")
+        .config("spark.jars.packages", "io.delta:delta-spark_2.12:3.0.0")
         .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
         .config(
             "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog"

tests/data/test_pandas_dataset.py

@@ -24,7 +24,7 @@ def spark_session():
 
     with (
         SparkSession.builder.master("local[*]")
-        .config("spark.jars.packages", "io.delta:delta-core_2.12:2.4.0")
+        .config("spark.jars.packages", "io.delta:delta-spark_2.12:3.0.0")
         .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
         .config(
             "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog"

tests/data/test_spark_dataset.py

@@ -21,7 +21,7 @@ def spark_session(tmp_path):
 
     with (
         SparkSession.builder.master("local[*]")
-        .config("spark.jars.packages", "io.delta:delta-core_2.12:2.4.0")
+        .config("spark.jars.packages", "io.delta:delta-spark_2.12:3.0.0")
         .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
         .config(
             "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog"

tests/data/test_spark_dataset_source.py

@@ -14,7 +14,7 @@ def spark_session():
 
     with (
         SparkSession.builder.master("local[*]")
-        .config("spark.jars.packages", "io.delta:delta-core_2.12:2.4.0")
+        .config("spark.jars.packages", "io.delta:delta-spark_2.12:3.0.0")
         .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
         .config(
             "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog"

tests/recipes/test_ingest_step.py

@@ -35,7 +35,7 @@ def spark_session():
     with tempfile.TemporaryDirectory() as tmpdir:
         with (
             SparkSession.builder.master("local[*]")
-            .config("spark.jars.packages", "io.delta:delta-core_2.12:2.4.0")
+            .config("spark.jars.packages", "io.delta:delta-spark_2.12:3.0.0")
             .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
             .config(
                 "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog"

tests/recipes/test_predict_step.py

@@ -28,7 +28,7 @@ def spark_session():
     with tempfile.TemporaryDirectory() as tmpdir:
         with (
             SparkSession.builder.master("local[*]")
-            .config("spark.jars.packages", "io.delta:delta-core_2.12:2.4.0")
+            .config("spark.jars.packages", "io.delta:delta-spark_2.12:3.0.0")
             .config("spark.sql.extensions", "io.delta.sql.DeltaSparkSessionExtension")
             .config(
                 "spark.sql.catalog.spark_catalog", "org.apache.spark.sql.delta.catalog.DeltaCatalog"