add to ci.

.github/workflows/nightly_tests.yml

@@ -525,118 +525,172 @@
           pip install slack_sdk tabulate
           python utils/log_reports.py >> $GITHUB_STEP_SUMMARY
 
+  run_nightly_pipeline_level_quantization_tests:
+    name: Torch quantization nightly tests
+    strategy:
+      fail-fast: false
+      max-parallel: 2
+    runs-on:
+      group: aws-g6e-xlarge-plus
+    container:
+      image: diffusers/diffusers-pytorch-cuda
+      options: --shm-size "20gb" --ipc host --gpus 0
+    steps:
+      - name: Checkout diffusers
+        uses: actions/checkout@v3
+        with:
+          fetch-depth: 2
+      - name: NVIDIA-SMI
+        run: nvidia-smi
+      - name: Install dependencies
+        run: |
+          python -m venv /opt/venv && export PATH="/opt/venv/bin:$PATH"
+          python -m uv pip install -e [quality,test]
+          python -m uv pip install -U bitsandbytes optimum_quanto
+          python -m uv pip install pytest-reportlog
+      - name: Environment
+        run: |
+          python utils/print_env.py
+      - name: Pipeline-level quantization tests on GPU
+        env:
+          HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
+          # https://pytorch.org/docs/stable/notes/randomness.html#avoiding-nondeterministic-algorithms
+          CUBLAS_WORKSPACE_CONFIG: :16:8
+          BIG_GPU_MEMORY: 40
+        run: |
+          python -m pytest -n 1 --max-worker-restart=0 --dist=loadfile \
+            --make-reports=tests_pipeline_level_quant_torch_cuda \
+            --report-log=tests_pipeline_level_quant_torch_cuda.log \
+            tests/quantization/test_pipeline_level_quantization.py
+      - name: Failure short reports
+        if: ${{ failure() }}
+        run: |
+          cat reports/tests_pipeline_level_quant_torch_cuda_stats.txt
+          cat reports/tests_pipeline_level_quant_torch_cuda_failures_short.txt
+      - name: Test suite reports artifacts
+        if: ${{ always() }}
+        uses: actions/upload-artifact@v4
+        with:
+          name: torch_cuda_pipeline_level_quant_reports
+          path: reports
+      - name: Generate Report and Notify Channel
+        if: always()
+        run: |
+          pip install slack_sdk tabulate
+          python utils/log_reports.py >> $GITHUB_STEP_SUMMARY
+
 # M1 runner currently not well supported
 # TODO: (Dhruv) add these back when we setup better testing for Apple Silicon
 #  run_nightly_tests_apple_m1:
 #    name: Nightly PyTorch MPS tests on MacOS
 #    runs-on: [ self-hosted, apple-m1 ]
 #    if: github.event_name == 'schedule'
 #
 #    steps:
 #      - name: Checkout diffusers
 #        uses: actions/checkout@v3
 #        with:
 #          fetch-depth: 2
 #
 #      - name: Clean checkout
 #        shell: arch -arch arm64 bash {0}
 #        run: |
 #          git clean -fxd
 #      - name: Setup miniconda
 #        uses: ./.github/actions/setup-miniconda
 #        with:
 #          python-version: 3.9
 #
 #      - name: Install dependencies
 #        shell: arch -arch arm64 bash {0}
 #        run: |
 #          ${CONDA_RUN} python -m pip install --upgrade pip uv
 #          ${CONDA_RUN} python -m uv pip install -e [quality,test]
 #          ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
 #          ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate
 #          ${CONDA_RUN} python -m uv pip install pytest-reportlog
 #      - name: Environment
 #        shell: arch -arch arm64 bash {0}
 #        run: |
 #          ${CONDA_RUN} python utils/print_env.py
 #      - name: Run nightly PyTorch tests on M1 (MPS)
 #        shell: arch -arch arm64 bash {0}
 #        env:
 #          HF_HOME: /System/Volumes/Data/mnt/cache
 #          HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
 #        run: |
 #          ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \
 #            --report-log=tests_torch_mps.log \
 #            tests/
 #      - name: Failure short reports
 #        if: ${{ failure() }}
 #        run: cat reports/tests_torch_mps_failures_short.txt
 #
 #      - name: Test suite reports artifacts
 #        if: ${{ always() }}
 #        uses: actions/upload-artifact@v4
 #        with:
 #          name: torch_mps_test_reports
 #          path: reports
 #
 #      - name: Generate Report and Notify Channel
 #        if: always()
 #        run: |
 #          pip install slack_sdk tabulate
 #          python utils/log_reports.py >> $GITHUB_STEP_SUMMARY  run_nightly_tests_apple_m1:
 #    name: Nightly PyTorch MPS tests on MacOS
 #    runs-on: [ self-hosted, apple-m1 ]
 #    if: github.event_name == 'schedule'
 #
 #    steps:
 #      - name: Checkout diffusers
 #        uses: actions/checkout@v3
 #        with:
 #          fetch-depth: 2
 #
 #      - name: Clean checkout
 #        shell: arch -arch arm64 bash {0}
 #        run: |
 #          git clean -fxd
 #      - name: Setup miniconda
 #        uses: ./.github/actions/setup-miniconda
 #        with:
 #          python-version: 3.9
 #
 #      - name: Install dependencies
 #        shell: arch -arch arm64 bash {0}
 #        run: |
 #          ${CONDA_RUN} python -m pip install --upgrade pip uv
 #          ${CONDA_RUN} python -m uv pip install -e [quality,test]
 #          ${CONDA_RUN} python -m uv pip install torch torchvision torchaudio --extra-index-url https://download.pytorch.org/whl/cpu
 #          ${CONDA_RUN} python -m uv pip install accelerate@git+https://github.com/huggingface/accelerate
 #          ${CONDA_RUN} python -m uv pip install pytest-reportlog
 #      - name: Environment
 #        shell: arch -arch arm64 bash {0}
 #        run: |
 #          ${CONDA_RUN} python utils/print_env.py
 #      - name: Run nightly PyTorch tests on M1 (MPS)
 #        shell: arch -arch arm64 bash {0}
 #        env:
 #          HF_HOME: /System/Volumes/Data/mnt/cache
 #          HF_TOKEN: ${{ secrets.DIFFUSERS_HF_HUB_READ_TOKEN }}
 #        run: |
 #          ${CONDA_RUN} python -m pytest -n 1 -s -v --make-reports=tests_torch_mps \
 #            --report-log=tests_torch_mps.log \
 #            tests/
 #      - name: Failure short reports
 #        if: ${{ failure() }}
 #        run: cat reports/tests_torch_mps_failures_short.txt
 #
 #      - name: Test suite reports artifacts
 #        if: ${{ always() }}
 #        uses: actions/upload-artifact@v4
 #        with:
 #          name: torch_mps_test_reports
 #          path: reports
 #
 #      - name: Generate Report and Notify Channel
 #        if: always()
 #        run: |
 #          pip install slack_sdk tabulate
 #          python utils/log_reports.py >> $GITHUB_STEP_SUMMARY