Update README.md (#117) #143
This file contains hidden or bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| name: Regression Test | |
| on: | |
| push: | |
| branches: | |
| - main | |
| pull_request: | |
| branches: | |
| - main | |
| jobs: | |
| run_all_tests: | |
| runs-on: ubuntu-latest | |
| #if: "!contains(github.event.pull_request.title, '[NO-REGRESSION-TEST]')" | |
| env: | |
| LANGS: "go rust python typescript" | |
| # ignore package version for Go e.g. 'a.b/c@506fb8ece467f3a71c29322169bef9b0bc92d554' | |
| DIFFJSON_IGNORE: > | |
| ['id'] | |
| ['Path'] | |
| ['Modules']['a.b/c']['Dependencies']['a.b/c'] | |
| ['Modules']['a.b/c/cmdx']['Dependencies']['a.b/c/cmdx'] | |
| steps: | |
| - name: Checkout pull request code | |
| uses: actions/checkout@v4 | |
| with: | |
| path: 'pr_repo' | |
| - name: Checkout main branch code | |
| uses: actions/checkout@v4 | |
| with: | |
| ref: 'main' | |
| path: 'main_repo' | |
| - name: Setup Go environment | |
| uses: actions/setup-go@v5 | |
| with: | |
| go-version: '1.22' | |
| cache-dependency-path: | | |
| main_repo/go.sum | |
| pr_repo/go.sum | |
| - name: Setup Rust toolchain | |
| uses: dtolnay/rust-toolchain@stable | |
| with: | |
| toolchain: stable | |
| components: rust-analyzer | |
| - name: Setup Python environment | |
| uses: actions/setup-python@v5 | |
| with: | |
| python-version: '3.11' | |
| - name: Setup JDK 21 | |
| uses: actions/setup-java@v4 | |
| with: | |
| java-version: '21' | |
| distribution: 'temurin' | |
| - name: Setup Node.js | |
| uses: actions/setup-node@v4 | |
| with: | |
| node-version: '22' | |
| - name: Compile both binaries | |
| run: | | |
| (cd main_repo && go build -o ../abcoder_old) | |
| (cd pr_repo && go build -o ../abcoder_new) | |
| - name: Install evaluation dependencies | |
| run: pip install -r ./pr_repo/script/requirements.txt | |
| ############################################################################## | |
| - name: Install dependencies for old | |
| run: | | |
| # HACK: auto installation uses the published version, not our local version | |
| (cd ./main_repo/ts-parser && npm install && npm run build && npm install -g .) | |
| OUTDIR=out_old ABCEXE=./abcoder_old ./main_repo/script/run_testdata.sh first | |
| # avoid wasting time install a new jdtls | |
| echo "JDTLS_ROOT_PATH=$(realpath ./main_repo/lang/java/lsp/jdtls/jdt-language-server-*)" >> $GITHUB_ENV | |
| - name: Run OLD abcoder | |
| run: | |
| OUTDIR=out_old ABCEXE=./abcoder_old ./main_repo/script/run_testdata.sh all | |
| - name: Reset dependencies | |
| run: | | |
| npm uninstall -g abcoder-ts-parser | |
| - name: Install dependencies for new | |
| run: | | |
| (cd ./pr_repo/ts-parser && npm install && npm run build && npm install -g .) | |
| OUTDIR=out_new ABCEXE=./abcoder_new ./pr_repo/script/run_testdata.sh first | |
| - name: Run NEW abcoder | |
| run: | |
| OUTDIR=out_new ABCEXE=./abcoder_new ./pr_repo/script/run_testdata.sh all | |
| - name: Upload output directories | |
| uses: actions/upload-artifact@v4 | |
| if: always() | |
| with: | |
| name: regression-outputs | |
| path: | | |
| out_old | |
| out_new | |
| retention-days: 3 | |
| - name: Compare outputs and check for regression | |
| run: ./pr_repo/script/diffjson.py out_old out_new $COMPARE_IGNORE |