Skip to content

Commit ebc7bed

Browse files
DN6DN6
authored
Add tests fetcher (huggingface#5848)
* add tests fetcher to utils * add test fetcher * update * update * remove unused dependency version check script * update * fix mistake * update * update * update * update * update * update * update * remove concurrency params * update * update * update * update * update * update * move test fetcher to dedicated workflow
1 parent ee519cf commit ebc7bed

File tree

2 files changed

+1278
-0
lines changed

2 files changed

+1278
-0
lines changed

‎.github/workflows/pr_test_fetcher.yml‎

Lines changed: 171 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,171 @@
1+
name: Fast tests for PRs
2+
3+
on:
4+
pull_request:
5+
branches:
6+
- main
7+
push:
8+
branches:
9+
- ci-*
10+
11+
env:
12+
DIFFUSERS_IS_CI: yes
13+
OMP_NUM_THREADS: 4
14+
MKL_NUM_THREADS: 4
15+
PYTEST_TIMEOUT: 60
16+
17+
jobs:
18+
setup_pr_tests:
19+
name: Setup PR Tests
20+
runs-on: docker-cpu
21+
container:
22+
image: diffusers/diffusers-pytorch-cpu
23+
options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/
24+
defaults:
25+
run:
26+
shell: bash
27+
outputs:
28+
matrix: ${{ steps.set_matrix.outputs.matrix }}
29+
test_map: ${{ steps.set_matrix.outputs.test_map }}
30+
steps:
31+
- name: Checkout diffusers
32+
uses: actions/checkout@v3
33+
with:
34+
fetch-depth: 2
35+
- name: Install dependencies
36+
run: |
37+
apt-get update && apt-get install libsndfile1-dev libgl1 -y
38+
python -m pip install -e .
39+
- name: Environment
40+
run: |
41+
python utils/print_env.py
42+
- name: Fetch Tests
43+
run: |
44+
python utils/tests_fetcher.py | tee test_preparation.txt
45+
- name: Report fetched tests
46+
uses: actions/upload-artifact@v3
47+
with:
48+
name: test_fetched
49+
path: test_preparation.txt
50+
- id: set_matrix
51+
name: Create Test Matrix
52+
# The `keys` is used as GitHub actions matrix for jobs, i.e. `models`, `pipelines`, etc.
53+
# The `test_map` is used to get the actual identified test files under each key.
54+
# If no test to run (so no `test_map.json` file), create a dummy map (empty matrix will fail)
55+
run: |
56+
if [ -f test_map.json ]; then
57+
keys=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); d = list(test_map.keys()); print(json.dumps(d))')
58+
test_map=$(python3 -c 'import json; fp = open("test_map.json"); test_map = json.load(fp); fp.close(); print(json.dumps(test_map))')
59+
else
60+
keys=$(python3 -c 'keys = ["dummy"]; print(keys)')
61+
test_map=$(python3 -c 'test_map = {"dummy": []}; print(test_map)')
62+
fi
63+
echo $keys
64+
echo $test_map
65+
echo "matrix=$keys" >> $GITHUB_OUTPUT
66+
echo "test_map=$test_map" >> $GITHUB_OUTPUT
67+
68+
run_pr_tests:
69+
name: Run PR Tests
70+
needs: setup_pr_tests
71+
if: contains(fromJson(needs.setup_pr_tests.outputs.matrix), 'dummy') != true
72+
strategy:
73+
fail-fast: false
74+
max-parallel: 2
75+
matrix:
76+
modules: ${{ fromJson(needs.setup_pr_tests.outputs.matrix) }}
77+
runs-on: docker-cpu
78+
container:
79+
image: diffusers/diffusers-pytorch-cpu
80+
options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/
81+
defaults:
82+
run:
83+
shell: bash
84+
steps:
85+
- name: Checkout diffusers
86+
uses: actions/checkout@v3
87+
with:
88+
fetch-depth: 2
89+
90+
- name: Install dependencies
91+
run: |
92+
apt-get update && apt-get install libsndfile1-dev libgl1 -y
93+
python -m pip install -e .[quality,test]
94+
python -m pip install accelerate
95+
96+
- name: Environment
97+
run: |
98+
python utils/print_env.py
99+
100+
- name: Run all selected tests on CPU
101+
run: |
102+
python -m pytest -n 2 --dist=loadfile -v --make-reports=${{ matrix.modules }}_tests_cpu ${{ fromJson(needs.setup_pr_tests.outputs.test_map)[matrix.modules] }}
103+
104+
- name: Failure short reports
105+
if: ${{ failure() }}
106+
continue-on-error: true
107+
run: |
108+
cat reports/${{ matrix.modules }}_tests_cpu_stats.txt
109+
cat reports/${{ matrix.modules }}_tests_cpu/failures_short.txt
110+
111+
- name: Test suite reports artifacts
112+
if: ${{ always() }}
113+
uses: actions/upload-artifact@v3
114+
with:
115+
name: ${{ matrix.modules }}_test_reports
116+
path: reports
117+
118+
run_staging_tests:
119+
strategy:
120+
fail-fast: false
121+
matrix:
122+
config:
123+
- name: Hub tests for models, schedulers, and pipelines
124+
framework: hub_tests_pytorch
125+
runner: docker-cpu
126+
image: diffusers/diffusers-pytorch-cpu
127+
report: torch_hub
128+
129+
name: ${{ matrix.config.name }}
130+
runs-on: ${{ matrix.config.runner }}
131+
container:
132+
image: ${{ matrix.config.image }}
133+
options: --shm-size "16gb" --ipc host -v /mnt/hf_cache:/mnt/cache/
134+
135+
defaults:
136+
run:
137+
shell: bash
138+
139+
steps:
140+
- name: Checkout diffusers
141+
uses: actions/checkout@v3
142+
with:
143+
fetch-depth: 2
144+
145+
- name: Install dependencies
146+
run: |
147+
apt-get update && apt-get install libsndfile1-dev libgl1 -y
148+
python -m pip install -e .[quality,test]
149+
150+
- name: Environment
151+
run: |
152+
python utils/print_env.py
153+
154+
- name: Run Hub tests for models, schedulers, and pipelines on a staging env
155+
if: ${{ matrix.config.framework == 'hub_tests_pytorch' }}
156+
run: |
157+
HUGGINGFACE_CO_STAGING=true python -m pytest \
158+
-m "is_staging_test" \
159+
--make-reports=tests_${{ matrix.config.report }} \
160+
tests
161+
162+
- name: Failure short reports
163+
if: ${{ failure() }}
164+
run: cat reports/tests_${{ matrix.config.report }}_failures_short.txt
165+
166+
- name: Test suite reports artifacts
167+
if: ${{ always() }}
168+
uses: actions/upload-artifact@v2
169+
with:
170+
name: pr_${{ matrix.config.report }}_test_reports
171+
path: reports

0 commit comments

Comments
 (0)