Skip to content

Commit 02e9996

Browse files
davidheryantodavidheryanto
authored
Merge pull request feast-dev#183 from davidheryanto/ci-cleanup
Continuous integration and deployment (CI/CD) update
2 parents 78e65bb + d715b2c commit 02e9996

File tree

114 files changed

+885
-1724
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

114 files changed

+885
-1724
lines changed

‎.gitignore‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -4,6 +4,9 @@ scratch*
44
### Local Environment ###
55
*local*.env
66

7+
### Secret ###
8+
**/service_account.json
9+
710
### Gradle ###
811
**/.gradle
912
!gradle/wrapper/gradle-wrapper.jar

‎.travis.yml‎

Lines changed: 117 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,117 @@
1+
cache:
2+
directories:
3+
- $HOME/.m2
4+
5+
matrix:
6+
include:
7+
- stage: unit test
8+
name: test ingestion
9+
language: java
10+
jdk: openjdk8
11+
install: echo "skip mvn install"
12+
script: mvn --projects ingestion --batch-mode test
13+
14+
- stage: unit test
15+
name: test core
16+
language: java
17+
jdk: openjdk8
18+
install: echo "skip mvn install"
19+
script: mvn --projects core --batch-mode test
20+
21+
- stage: unit test
22+
name: test serving
23+
language: java
24+
jdk: openjdk8
25+
install: echo "skip mvn install"
26+
script: mvn --projects serving --batch-mode test
27+
28+
- stage: unit test
29+
name: test cli
30+
language: go
31+
go: 1.12.x
32+
env: GO111MODULE=on
33+
script: go test ./cli/feast/...
34+
35+
- stage: build
36+
name: build core
37+
language: java
38+
jdk: openjdk8
39+
services: docker
40+
before_install: .travis/decrypt_secrets.sh
41+
install: . .travis/install_google_cloud_sdk.sh
42+
before_script: .travis/prepare_maven_cache_for_docker.sh
43+
script:
44+
- docker build --tag=us.gcr.io/kf-feast/feast-core:${TRAVIS_COMMIT} --build-arg=REVISION=${TRAVIS_COMMIT} --file Dockerfiles/core/Dockerfile .
45+
- docker push us.gcr.io/kf-feast/feast-core:${TRAVIS_COMMIT}
46+
if: type != pull_request
47+
48+
- stage: build
49+
name: build serving
50+
language: java
51+
jdk: openjdk8
52+
services: docker
53+
before_install: .travis/decrypt_secrets.sh
54+
install: . .travis/install_google_cloud_sdk.sh
55+
before_script: .travis/prepare_maven_cache_for_docker.sh
56+
script:
57+
- docker build --tag=us.gcr.io/kf-feast/feast-serving:${TRAVIS_COMMIT} --build-arg=REVISION=${TRAVIS_COMMIT} --file Dockerfiles/serving/Dockerfile .
58+
- docker push us.gcr.io/kf-feast/feast-serving:${TRAVIS_COMMIT}
59+
if: type != pull_request
60+
61+
- stage: build
62+
name: build cli
63+
language: go
64+
go: 1.12.x
65+
env:
66+
- GO111MODULE=on
67+
- FEAST_CLI_GCS_URI=gs://feast-templocation-kf-feast/build_${TRAVIS_BUILD_NUMBER}/cli/feast
68+
before_install: .travis/decrypt_secrets.sh
69+
install: . .travis/install_google_cloud_sdk.sh
70+
script:
71+
- go build -o ./cli/build/feast ./cli/feast
72+
- gsutil cp ./cli/build/feast ${FEAST_CLI_GCS_URI}
73+
if: type != pull_request
74+
75+
- stage: integration test
76+
name: test batch and streaming import job
77+
language: python
78+
python: 3.6
79+
env:
80+
- BATCH_IMPORT_DATA_LOCAL_PATH=${TRAVIS_BUILD_DIR}/integration-tests/testdata/feature_values/ingestion_1.csv
81+
- BATCH_IMPORT_DATA_GCS_PATH=gs://feast-templocation-kf-feast/build_${TRAVIS_BUILD_NUMBER}/integration-tests/testdata/feature_values/ingestion_1.csv
82+
- FEAST_IMAGE_TAG=${TRAVIS_COMMIT}
83+
- FEAST_CLI_GCS_URI=gs://feast-templocation-kf-feast/build_${TRAVIS_BUILD_NUMBER}/cli/feast
84+
- FEAST_WAREHOUSE_DATASET=feast_build_${TRAVIS_BUILD_NUMBER}
85+
- FEAST_CORE_URI=localhost:6565
86+
- FEAST_SERVING_URI=localhost:6566
87+
- KAFKA_BROKERS=localhost:9092
88+
- KAFKA_TOPICS=feast-topic
89+
before_install: .travis/decrypt_secrets.sh
90+
install:
91+
- . .travis/install_google_cloud_sdk.sh
92+
- . .travis/install_feast_sdk.sh
93+
before_script:
94+
- .travis/start_local_feast.sh
95+
- .travis/prepare_testdata.sh
96+
script:
97+
- .travis/run_batch_import_and_validate.sh
98+
- .travis/run_streaming_import_and_validate.sh
99+
after_script: .travis/cleanup_testdata.sh
100+
if: type != pull_request
101+
102+
- stage: deployment test
103+
name: test helm deployment
104+
language: minimal
105+
env:
106+
- BUILD_NUMBER=${TRAVIS_BUILD_NUMBER}
107+
- FEAST_IMAGE_TAG=${TRAVIS_COMMIT}
108+
- FEAST_WAREHOUSE_DATASET=feast_build_${TRAVIS_BUILD_NUMBER}
109+
- RELEASE_NAME=feast-build-${TRAVIS_BUILD_NUMBER}
110+
before_install: .travis/decrypt_secrets.sh
111+
install:
112+
- . .travis/install_google_cloud_sdk.sh
113+
- . .travis/install_helm.sh
114+
before_script: envsubst < ${TRAVIS_BUILD_DIR}/integration-tests/feast-helm-values.yaml.template > ${TRAVIS_BUILD_DIR}/integration-tests/feast-helm-values.yaml
115+
script: helm install --name ${RELEASE_NAME} --wait --timeout 300 ./charts/feast -f ${TRAVIS_BUILD_DIR}/integration-tests/feast-helm-values.yaml
116+
after_script: helm delete --purge ${RELEASE_NAME}
117+
if: type != pull_request

‎.travis/cleanup_testdata.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/usr/bin/env bash
2+
3+
bq --project_id=kf-feast rm -rf --dataset ${FEAST_WAREHOUSE_DATASET}
4+
gsutil rm -r ${BATCH_IMPORT_DATA_GCS_PATH}

‎.travis/decrypt_secrets.sh‎

Lines changed: 11 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,11 @@
1+
#!/usr/bin/env bash
2+
3+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
4+
5+
# Decrypt Google Cloud service account JSON key for BigQuery and GKE access
6+
openssl aes-256-cbc \
7+
-K $encrypted_deb1d355a9cd_key \
8+
-iv $encrypted_deb1d355a9cd_iv \
9+
-in ${SCRIPT_DIR}/service_account.json.enc \
10+
-out ${SCRIPT_DIR}/service_account.json \
11+
-d

‎.travis/install_feast_sdk.sh‎

Lines changed: 13 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,13 @@
1+
#!/usr/bin/env bash
2+
3+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
4+
5+
# This assumes Feast CLI has been built and pushed to GCS in the previous build step
6+
gsutil cp ${FEAST_CLI_GCS_URI} ${TRAVIS_BUILD_DIR}/cli/build/feast
7+
chmod +x ${TRAVIS_BUILD_DIR}/cli/build/feast
8+
export PATH=${TRAVIS_BUILD_DIR}/cli/build:$PATH
9+
feast config set coreURI ${FEAST_CORE_URI}
10+
11+
pip install -qe ${TRAVIS_BUILD_DIR}/sdk/python
12+
pip install -qr ${TRAVIS_BUILD_DIR}/integration-tests/testutils/requirements.txt
13+
export GOOGLE_APPLICATION_CREDENTIALS=${SCRIPT_DIR}/service_account.json

‎.travis/install_google_cloud_sdk.sh‎

Lines changed: 12 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,12 @@
1+
#!/usr/bin/env bash
2+
3+
SCRIPT_DIR="$( cd "$( dirname "${BASH_SOURCE[0]}" )" >/dev/null 2>&1 && pwd )"
4+
GOOGLE_CLOUD_SDK_ARCHIVE_URL=https://dl.google.com/dl/cloudsdk/channels/rapid/downloads/google-cloud-sdk-244.0.0-linux-x86_64.tar.gz
5+
6+
wget -qO- ${GOOGLE_CLOUD_SDK_ARCHIVE_URL} | tar xz
7+
export PATH=$PWD/google-cloud-sdk/bin:$PATH
8+
gcloud -q components install kubectl
9+
gcloud config set project kf-feast
10+
gcloud -q auth activate-service-account --key-file=${SCRIPT_DIR}/service_account.json
11+
gcloud -q auth configure-docker
12+
gcloud -q container clusters get-credentials feast-test-cluster --zone us-central1-a --project kf-feast

‎.travis/install_helm.sh‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
#!/usr/bin/env bash
2+
3+
wget -qO- https://storage.googleapis.com/kubernetes-helm/helm-v2.13.1-linux-amd64.tar.gz | tar xz
4+
sudo mv linux-amd64/helm /usr/local/bin/helm

‎.travis/prepare_maven_cache_for_docker.sh‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,5 @@
1+
#!/usr/bin/env bash
2+
3+
# This works because Dockerfile for core and serving assumes Maven repository
4+
# is located at .m2 directory inside Feast repository root directory
5+
cp -r $HOME/.m2 ${TRAVIS_BUILD_DIR}/.m2

‎.travis/prepare_testdata.sh‎

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,8 @@
1+
#!/usr/bin/env bash
2+
3+
cd ${TRAVIS_BUILD_DIR}/integration-tests/testdata/import_specs
4+
envsubst < batch_from_gcs.yaml.template > batch_from_gcs.yaml
5+
envsubst < stream_from_kafka.yaml.template > stream_from_kafka.yaml
6+
7+
bq --project_id=kf-feast mk --dataset $FEAST_WAREHOUSE_DATASET
8+
gsutil cp ${BATCH_IMPORT_DATA_LOCAL_PATH} ${BATCH_IMPORT_DATA_GCS_PATH}

‎.travis/run_batch_import_and_validate.sh‎

Lines changed: 16 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,16 @@
1+
#!/usr/bin/env bash
2+
3+
cd ${TRAVIS_BUILD_DIR}/integration-tests
4+
5+
feast apply entity testdata/entity_specs/entity_1.yaml
6+
feast apply feature testdata/feature_specs/entity_1.feature_*.yaml
7+
feast jobs run testdata/import_specs/batch_from_gcs.yaml --wait
8+
9+
python -m testutils.validate_feature_values \
10+
--entity_spec_file=testdata/entity_specs/entity_1.yaml \
11+
--feature_spec_files=testdata/feature_specs/entity_1*.yaml \
12+
--expected-warehouse-values-file=testdata/feature_values/ingestion_1.csv \
13+
--expected-serving-values-file=testdata/feature_values/serving_1.csv \
14+
--bigquery-dataset-for-warehouse=${FEAST_WAREHOUSE_DATASET} \
15+
--feast-serving-url=${FEAST_SERVING_URI} \
16+
--project=kf-feast

0 commit comments

Comments
 (0)