Skip to content

Commit 1d60f61

Browse files
yuwenzhoyuwenzho
authored
Migrate onnxrt RTN WOQ to 3.x API (#1544)
Signed-off-by: yuwenzho <[email protected]> Signed-off-by: chensuyue <[email protected]> Signed-off-by: xin3he <[email protected]>
1 parent 0c4d4bd commit 1d60f61

File tree

30 files changed

+2775
-10
lines changed

30 files changed

+2775
-10
lines changed

‎.azure-pipelines/scripts/install_nc.sh‎

Lines changed: 7 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -5,15 +5,19 @@ cd /neural-compressor
55
if [[ $1 = *"3x_pt" ]]; then
66
python -m pip install --no-cache-dir -r requirements_pt.txt
77
python setup.py pt bdist_wheel
8-
pip install dist/neural_compressor*.whl
8+
pip install dist/neural_compressor*.whl --force-reinstall
99
elif [[ $1 = *"3x_tf" ]]; then
1010
python -m pip install --no-cache-dir -r requirements_tf.txt
1111
python setup.py tf bdist_wheel
12-
pip install dist/neural_compressor*.whl
12+
pip install dist/neural_compressor*.whl --force-reinstall
13+
elif [[ $1 = *"3x_ort" ]]; then
14+
python -m pip install --no-cache-dir -r requirements_ort.txt
15+
python setup.py ort bdist_wheel
16+
pip install dist/neural_compressor*.whl --force-reinstall
1317
else
1418
python -m pip install --no-cache-dir -r requirements.txt
1519
python setup.py 2x bdist_wheel
16-
pip install dist/neural_compressor*.whl
20+
pip install dist/neural_compressor*.whl --force-reinstall
1721
fi
1822

1923
echo -e "\n pip list after install Neural Compressor ... "

‎.azure-pipelines/scripts/ut/3x/collect_log_3x.sh‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -1,5 +1,6 @@
11
source /neural-compressor/.azure-pipelines/scripts/change_color.sh
22

3+
set -xe
34
pip install coverage
45
export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.${1}
56
coverage_log="/neural-compressor/log_dir/coverage_log"
@@ -22,7 +23,7 @@ cd /neural-compressor
2223
git config --global --add safe.directory /neural-compressor
2324
git fetch
2425
git checkout master
25-
echo y | pip uninstall neural-compressor
26+
echo y | pip uninstall neural_compressor_${1}
2627
cd /neural-compressor/.azure-pipelines/scripts && bash install_nc.sh ${1}
2728

2829
coverage erase

‎.azure-pipelines/scripts/ut/3x/coverage.3x_ort‎

Lines changed: 15 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,15 @@
1+
[run]
2+
branch = True
3+
4+
[report]
5+
include =
6+
*/neural_compressor/common/*
7+
*/neural_compressor/onnxrt/*
8+
exclude_lines =
9+
pragma: no cover
10+
raise NotImplementedError
11+
raise TypeError
12+
if self.device == "gpu":
13+
if device == "gpu":
14+
except ImportError:
15+
except Exception as e:

‎.azure-pipelines/scripts/ut/3x/run_3x_ort.sh‎

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,34 @@
1+
#!/bin/bash
2+
python -c "import neural_compressor as nc"
3+
test_case="run 3x ONNXRT"
4+
echo "${test_case}"
5+
6+
# install requirements
7+
echo "set up UT env..."
8+
pip install -r /neural-compressor/test/3x/onnxrt/requirements.txt
9+
pip install coverage
10+
pip list
11+
12+
export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.3x_ort
13+
inc_path=$(python -c 'import neural_compressor; print(neural_compressor.__path__[0])')
14+
cd /neural-compressor/test || exit 1
15+
find ./3x/onnxrt/* -name "test*.py" | sed 's,\.\/,coverage run --source='"${inc_path}"' --append ,g' | sed 's/$/ --verbose/'> run.sh
16+
17+
LOG_DIR=/neural-compressor/log_dir
18+
mkdir -p ${LOG_DIR}
19+
ut_log_name=${LOG_DIR}/ut_3x_ort.log
20+
21+
echo "cat run.sh..."
22+
sort run.sh -o run.sh
23+
cat run.sh | tee ${ut_log_name}
24+
echo "------UT start-------"
25+
bash -x run.sh 2>&1 | tee -a ${ut_log_name}
26+
cp .coverage ${LOG_DIR}/.coverage
27+
28+
echo "------UT end -------"
29+
30+
if [ $(grep -c "FAILED" ${ut_log_name}) != 0 ] || [ $(grep -c "core dumped" ${ut_log_name}) != 0 ] || [ $(grep -c "ModuleNotFoundError:" ${ut_log_name}) != 0 ] || [ $(grep -c "OK" ${ut_log_name}) == 0 ];then
31+
echo "Find errors in UT test, please check the output..."
32+
exit 1
33+
fi
34+
echo "UT finished successfully! "

‎.azure-pipelines/scripts/ut/3x/run_3x_pt.sh‎

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -5,9 +5,8 @@ echo "${test_case}"
55

66
# install requirements
77
echo "set up UT env..."
8-
pip install transformers
8+
pip install -r /neural-compressor/test/3x/torch/requirements.txt
99
pip install coverage
10-
pip install pytest
1110
pip list
1211

1312
export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.3x_pt

‎.azure-pipelines/scripts/ut/3x/run_3x_tf.sh‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -5,8 +5,8 @@ echo "${test_case}"
55

66
# install requirements
77
echo "set up UT env..."
8+
pip install -r /neural-compressor/test/3x/tensorflow/requirements.txt
89
pip install coverage
9-
pip install pytest
1010
pip list
1111

1212
export COVERAGE_RCFILE=/neural-compressor/.azure-pipelines/scripts/ut/3x/coverage.3x_tf

‎.azure-pipelines/ut-3x-ort.yml‎

Lines changed: 106 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,106 @@
1+
trigger: none
2+
3+
pr:
4+
autoCancel: true
5+
drafts: false
6+
branches:
7+
include:
8+
- master
9+
paths:
10+
include:
11+
- neural_compressor/common
12+
- neural_compressor/onnxrt
13+
- test/3x/onnxrt
14+
- setup.py
15+
- requirements_ort.txt
16+
17+
pool: ICX-16C
18+
19+
variables:
20+
IMAGE_NAME: "neural-compressor"
21+
IMAGE_TAG: "py310"
22+
UPLOAD_PATH: $(Build.SourcesDirectory)/log_dir
23+
DOWNLOAD_PATH: $(Build.SourcesDirectory)/log_dir
24+
ARTIFACT_NAME: "UT_coverage_report_3x_ort"
25+
REPO: $(Build.Repository.Uri)
26+
27+
stages:
28+
- stage: ONNXRT
29+
displayName: Unit Test 3x ONNXRT
30+
dependsOn: []
31+
jobs:
32+
- job:
33+
displayName: Unit Test 3x ONNXRT
34+
steps:
35+
- template: template/ut-template.yml
36+
parameters:
37+
dockerConfigName: "commonDockerConfig"
38+
utScriptFileName: "3x/run_3x_ort"
39+
uploadPath: $(UPLOAD_PATH)
40+
utArtifact: "ut_coverage_3x"
41+
42+
43+
- stage: ONNXRT_baseline
44+
displayName: Unit Test 3x ONNXRT baseline
45+
dependsOn: []
46+
jobs:
47+
- job:
48+
displayName: Unit Test 3x ONNXRT baseline
49+
steps:
50+
- template: template/ut-template.yml
51+
parameters:
52+
dockerConfigName: "gitCloneDockerConfig"
53+
utScriptFileName: "3x/run_3x_ort"
54+
uploadPath: $(UPLOAD_PATH)
55+
utArtifact: "ut_coverage_3x_baseline"
56+
repo: $(REPO)
57+
58+
- stage: Coverage
59+
displayName: "Coverage Combine"
60+
pool:
61+
vmImage: "ubuntu-latest"
62+
dependsOn: [ONNXRT, ONNXRT_baseline]
63+
jobs:
64+
- job: CollectDatafiles
65+
steps:
66+
- script: |
67+
if [[ ! $(docker images | grep -i ${IMAGE_NAME}:${IMAGE_TAG}) ]]; then
68+
docker build -f ${BUILD_SOURCESDIRECTORY}/.azure-pipelines/docker/Dockerfile.devel -t ${IMAGE_NAME}:${IMAGE_TAG} .
69+
fi
70+
docker images | grep -i ${IMAGE_NAME}
71+
if [[ $? -ne 0 ]]; then
72+
echo "NO Such Repo"
73+
exit 1
74+
fi
75+
displayName: "Build develop docker image"
76+
77+
- task: DownloadPipelineArtifact@2
78+
inputs:
79+
artifact:
80+
path: $(DOWNLOAD_PATH)
81+
82+
- script: |
83+
echo "--- create container ---"
84+
docker run -d -it --name="collectLogs" -v ${BUILD_SOURCESDIRECTORY}:/neural-compressor ${IMAGE_NAME}:${IMAGE_TAG} /bin/bash
85+
echo "--- docker ps ---"
86+
docker ps
87+
echo "--- collect logs ---"
88+
docker exec collectLogs /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts \
89+
&& bash install_nc.sh 3x_ort \
90+
&& bash ut/3x/collect_log_3x.sh 3x_ort"
91+
displayName: "Collect UT Coverage"
92+
93+
- task: PublishPipelineArtifact@1
94+
condition: succeededOrFailed()
95+
inputs:
96+
targetPath: $(UPLOAD_PATH)
97+
artifact: $(ARTIFACT_NAME)
98+
publishLocation: "pipeline"
99+
100+
- task: Bash@3
101+
condition: always()
102+
inputs:
103+
targetType: "inline"
104+
script: |
105+
docker exec collectLogs bash -c "rm -fr /neural-compressor/* && rm -fr /neural-compressor/.* || true"
106+
displayName: "Docker clean up"

‎.azure-pipelines/ut-3x-pt.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ stages:
8888
docker exec collectLogs /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts \
8989
&& bash install_nc.sh 3x_pt \
9090
&& bash ut/3x/collect_log_3x.sh 3x_pt"
91-
displayName: "collect logs"
91+
displayName: "Collect UT Coverage"
9292
9393
- task: PublishPipelineArtifact@1
9494
condition: succeededOrFailed()

‎.azure-pipelines/ut-3x-tf.yml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -88,7 +88,7 @@ stages:
8888
docker exec collectLogs /bin/bash +x -c "cd /neural-compressor/.azure-pipelines/scripts \
8989
&& bash install_nc.sh 3x_tf \
9090
&& bash ut/3x/collect_log_3x.sh 3x_tf"
91-
displayName: "collect logs"
91+
displayName: "Collect UT Coverage"
9292
9393
- task: PublishPipelineArtifact@1
9494
condition: succeededOrFailed()

‎.azure-pipelines/ut-basic-no-cover.yml‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -19,6 +19,7 @@ pr:
1919
- neural_compressor/common
2020
- neural_compressor/torch
2121
- neural_compressor/tensorflow
22+
- neural_compressor/onnxrt
2223

2324
pool: ICX-16C
2425

0 commit comments

Comments
 (0)