1+ name : nightly-ci
2+
3+ on :
4+ schedule :
5+ - cron : ' 00 08 * * *' # early morning 08:00 AM UTC, which is 1 AM PST/4 AM EST.
6+
7+ # concurrency is currently broken, see details https://github.com/actions/runner/issues/1532
8+ # concurrency:
9+ # group: pr-integration-tests-${{ github.event.pull_request.number }}
10+ # cancel-in-progress: true
11+
12+ jobs :
13+ check_date :
14+ runs-on : ubuntu-latest
15+ name : Check latest commit
16+ outputs :
17+ WAS_EDITED : ${{ steps.check_date.outputs.WAS_EDITED }}
18+ steps :
19+ - uses : actions/checkout@v2
20+ with :
21+ ref : develop
22+ - id : check_date
23+ name : Check if there were commits in the last day
24+ if : ${{ github.event_name == 'schedule' }}
25+ run : echo '::set-output name=WAS_EDITED::'$(test -n "$(git log --format=%H --since='24 hours ago')" && echo 'true' || echo 'false')
26+
27+ integration-test-python :
28+ needs : [check_date]
29+ runs-on : ${{ matrix.os }}
30+ strategy :
31+ fail-fast : false
32+ matrix :
33+ python-version : [ "3.8" ]
34+ os : [ ubuntu-latest ]
35+ env :
36+ OS : ${{ matrix.os }}
37+ PYTHON : ${{ matrix.python-version }}
38+ services :
39+ redis :
40+ image : redis
41+ ports :
42+ - 6379:6379
43+ options : >-
44+ --health-cmd "redis-cli ping"
45+ --health-interval 10s
46+ --health-timeout 5s
47+ --health-retries 5
48+ steps :
49+ - uses : actions/checkout@v2
50+ with :
51+ # pull_request_target runs the workflow in the context of the base repo
52+ # as such actions/checkout needs to be explicit configured to retrieve
53+ # code from the PR.
54+ ref : refs/pull/${{ github.event.pull_request.number }}/merge
55+ submodules : recursive
56+ - name : Setup Python
57+ uses : actions/setup-python@v2
58+ id : setup-python
59+ with :
60+ python-version : ${{ os.PYTHON }}
61+ architecture : x64
62+ - name : Setup Go
63+ id : setup-go
64+ uses : actions/setup-go@v2
65+ with :
66+ go-version : 1.18.0
67+ - name : Set up gcloud SDK
68+ uses : google-github-actions/setup-gcloud@v0
69+ with :
70+ project_id : ${{ secrets.GCP_PROJECT_ID }}
71+ service_account_key : ${{ secrets.GCP_SA_KEY }}
72+ export_default_credentials : true
73+ - name : Use gcloud CLI
74+ run : gcloud info
75+ - name : Set up AWS SDK
76+ uses : aws-actions/configure-aws-credentials@v1
77+ with :
78+ aws-access-key-id : ${{ secrets.AWS_ACCESS_KEY_ID }}
79+ aws-secret-access-key : ${{ secrets.AWS_SECRET_ACCESS_KEY }}
80+ aws-region : us-west-2
81+ - name : Use AWS CLI
82+ run : aws sts get-caller-identity
83+ - name : Upgrade pip version
84+ run : |
85+ pip install --upgrade "pip>=21.3.1,<22.1"
86+ name : Get pip cache dir
87+ id : pip-cache
88+ run : |
89+ echo "::set-output name=dir::$(pip cache dir)"
90+ name : pip cache
91+ uses : actions/cache@v2
92+ with :
93+ path : |
94+ ${{ steps.pip-cache.outputs.dir }}
95+ /opt/hostedtoolcache/Python
96+ /Users/runner/hostedtoolcache/Python
97+ key : ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-${{ hashFiles(format('**/py{0}-ci-requirements.txt', env.PYTHON)) }}
98+ restore-keys : |
99+ ${{ runner.os }}-${{ steps.setup-python.outputs.python-version }}-pip-
100+ name : Install pip-tools
101+ run : pip install pip-tools
102+ - name : Install apache-arrow on ubuntu
103+ if : matrix.os == 'ubuntu-latest'
104+ run : |
105+ sudo apt update
106+ sudo apt install -y -V ca-certificates lsb-release wget
107+ wget https://apache.jfrog.io/artifactory/arrow/$(lsb_release --id --short | tr 'A-Z' 'a-z')/apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
108+ sudo apt install -y -V ./apache-arrow-apt-source-latest-$(lsb_release --codename --short).deb
109+ sudo apt update
110+ sudo apt install -y -V libarrow-dev
111+ name : Install apache-arrow on macos
112+ if : matrix.os == 'macOS-latest'
113+ run : brew install apache-arrow
114+ - name : Install dependencies
115+ run : make install-python-ci-dependencies
116+ - name : Setup Redis Cluster
117+ run : |
118+ docker pull vishnunair/docker-redis-cluster:latest
119+ docker run -d -p 6001:6379 -p 6002:6380 -p 6003:6381 -p 6004:6382 -p 6005:6383 -p 6006:6384 --name redis-cluster vishnunair/docker-redis-cluster
120+ name : Test python
121+ if : ${{ always() }} # this will guarantee that step won't be canceled and resources won't leak
122+ env :
123+ FEAST_SERVER_DOCKER_IMAGE_TAG : ${{ needs.build-docker-image.outputs.DOCKER_IMAGE_TAG }}
124+ FEAST_USAGE : " False"
125+ IS_TEST : " True"
126+ SNOWFLAKE_CI_DEPLOYMENT : ${{ secrets.SNOWFLAKE_CI_DEPLOYMENT }}
127+ SNOWFLAKE_CI_USER : ${{ secrets.SNOWFLAKE_CI_USER }}
128+ SNOWFLAKE_CI_PASSWORD : ${{ secrets.SNOWFLAKE_CI_PASSWORD }}
129+ SNOWFLAKE_CI_ROLE : ${{ secrets.SNOWFLAKE_CI_ROLE }}
130+ SNOWFLAKE_CI_WAREHOUSE : ${{ secrets.SNOWFLAKE_CI_WAREHOUSE }}
131+ run : pytest -n 8 --cov=./ --cov-report=xml --color=yes sdk/python/tests --integration --durations=5 --timeout=1200 --timeout_method=thread
132+ - name : Upload coverage to Codecov
133+ uses : codecov/codecov-action@v1
134+ with :
135+ token : ${{ secrets.CODECOV_TOKEN }}
136+ files : ./coverage.xml
137+ flags : integrationtests
138+ env_vars : OS,PYTHON
139+ fail_ci_if_error : true
140+ verbose : true
0 commit comments