Skip to content

Commit 44e3d6d

Browse files
davidheryantodavidheryanto
committed
Add .drone.yml
1 parent e46fb9e commit 44e3d6d

File tree

3 files changed

+268
-1
lines changed

3 files changed

+268
-1
lines changed

‎.drone.yml‎

Lines changed: 265 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,265 @@
1+
kind: pipeline
2+
name: default
3+
4+
steps:
5+
6+
# Retrieve cache for Maven and Go dependencies for faster build
7+
# ============================================================
8+
9+
- name: restore_cache
10+
image: homerovalle/drone-gcs-cache
11+
settings:
12+
pull: true
13+
bucket: feast-templocation-kf-feast
14+
path: feast-templocation-kf-feast/feast-mvn-cache
15+
restore: true
16+
json_key:
17+
from_secret: google_credentials
18+
19+
# Run unit tests for Feast components
20+
# ============================================================
21+
22+
- name: test_feast_core
23+
image: maven:3.6-jdk-8-slim
24+
environment:
25+
MAVEN_OPTS: "-Dmaven.repo.local=.m2 -DdependencyLocationsEnabled=false"
26+
commands:
27+
- mvn --projects core test
28+
depends_on: [ restore_cache ]
29+
30+
- name: test_feast_serving
31+
image: maven:3.6-jdk-8-slim
32+
environment:
33+
MAVEN_OPTS: "-Dmaven.repo.local=.m2 -DdependencyLocationsEnabled=false"
34+
commands:
35+
- mvn --projects serving test
36+
depends_on: [ restore_cache ]
37+
38+
- name: test_feast_ingestion
39+
image: maven:3.6-jdk-8-slim
40+
environment:
41+
MAVEN_OPTS: "-Dmaven.repo.local=.m2 -DdependencyLocationsEnabled=false"
42+
commands:
43+
- mvn --projects ingestion test
44+
depends_on: [ restore_cache ]
45+
46+
- name: test_feast_cli
47+
image: golang:1.12
48+
environment:
49+
GOCACHE: /drone/src/.gocache
50+
commands:
51+
- go test ./cli/feast/...
52+
depends_on: [ restore_cache ]
53+
54+
# Build and publish Docker images for Feast components
55+
# ============================================================
56+
57+
- name: build_feast_core
58+
image: plugins/gcr:18
59+
settings:
60+
dockerfile: Dockerfiles/core/Dockerfile
61+
build_args:
62+
- REVISION=${DRONE_COMMIT}
63+
registry: us.gcr.io
64+
repo: kf-feast/feast-core
65+
tags: ${DRONE_COMMIT}
66+
target: production
67+
json_key:
68+
from_secret: google_credentials
69+
depends_on: [ restore_cache ]
70+
71+
- name: build_feast_serving
72+
image: plugins/gcr:18
73+
settings:
74+
dockerfile: Dockerfiles/serving/Dockerfile
75+
build_args:
76+
- REVISION=${DRONE_COMMIT}
77+
registry: us.gcr.io
78+
repo: kf-feast/feast-serving
79+
tags: ${DRONE_COMMIT}
80+
target: production
81+
json_key:
82+
from_secret: google_credentials
83+
depends_on: [ restore_cache ]
84+
85+
- name: build_feast_cli
86+
image: golang:1.12
87+
environment:
88+
GOCACHE: /drone/src/.gocache
89+
commands:
90+
- go build -o ./cli/build/feast ./cli/feast
91+
depends_on: [ restore_cache ]
92+
93+
# Run integration tests
94+
# ============================================================
95+
96+
- name: integration_test_prepare
97+
image: devth/helm:v2.13.1
98+
environment:
99+
FEAST_IMAGE_TAG: ${DRONE_COMMIT}
100+
FEAST_WAREHOUSE_DATASET: feast_build_${DRONE_BUILD_NUMBER}
101+
FEAST_CORE_URL: build-${DRONE_BUILD_NUMBER}.feast.dragonin.me:80
102+
FEAST_SERVING_URL: build-${DRONE_BUILD_NUMBER}.feast.dragonin.me:80
103+
FEAST_RELEASE_NAME: feast-${DRONE_BUILD_NUMBER}
104+
BATCH_IMPORT_DATA_GCS_PATH: gs://feast-templocation-kf-feast/build_${DRONE_BUILD_NUMBER}/integration-tests/testdata/feature_values/ingestion_1.csv
105+
KAFKA_BROKERS: 10.128.0.55:31090
106+
KAFKA_TOPICS: feast_build_${DRONE_BUILD_NUMBER}
107+
JSON_KEY:
108+
from_secret: google_credentials
109+
commands:
110+
- FEAST_HOME=$(pwd)
111+
112+
# Authenticate to Google Cloud and Kube cluster
113+
- printf "%s\n" "$JSON_KEY" > /etc/service-account.json
114+
- gcloud auth activate-service-account --key-file /etc/service-account.json
115+
- gcloud container clusters get-credentials feast-test-cluster --zone us-central1-a --project kf-feast
116+
- unset JSON_KEY
117+
118+
# Install Feast with Helm using Feast images built from previous build steps
119+
- cd $FEAST_HOME/integration-tests
120+
- envsubst < feast-helm-values.yaml.template > feast-helm-values.yaml
121+
- helm install --name $FEAST_RELEASE_NAME --wait --timeout 210 ../charts/feast -f feast-helm-values.yaml
122+
123+
# Ensure unique import jobs for every CI build by parameterizing job properties with CI environment
124+
- cd $FEAST_HOME/integration-tests/testdata/import_specs
125+
- envsubst < batch_from_gcs.yaml.template > batch_from_gcs.yaml
126+
- envsubst < stream_from_kafka.yaml.template > stream_from_kafka.yaml
127+
128+
# Prepare BigQuery dataset and remote batch import data in Cloud Storage
129+
- bq mk --dataset $FEAST_WAREHOUSE_DATASET
130+
- gsutil cp $FEAST_HOME/integration-tests/testdata/feature_values/ingestion_1.csv $BATCH_IMPORT_DATA_GCS_PATH
131+
depends_on: [ build_feast_core, build_feast_serving, build_feast_cli ]
132+
133+
- name: integration_test_batch
134+
image: python:3.7
135+
environment:
136+
FEAST_CORE_URL: build-${DRONE_BUILD_NUMBER}.feast.dragonin.me:80
137+
FEAST_SERVING_URL: build-${DRONE_BUILD_NUMBER}.feast.dragonin.me:80
138+
FEAST_WAREHOUSE_DATASET: feast_build_${DRONE_BUILD_NUMBER}
139+
commands:
140+
- FEAST_HOME=$(pwd)
141+
142+
# Install and use latest Feast SDK in the integration test
143+
- pip install -qe $FEAST_HOME/sdk/python
144+
- pip install -qr $FEAST_HOME/integration-tests/testutils/requirements.txt
145+
146+
# Use Feast CLI built from "build_feast_cli" step
147+
- cp ./cli/build/feast /usr/bin/feast
148+
- feast config set coreURI $FEAST_CORE_URL
149+
150+
# Register entity and feature specs, then start a batch import job
151+
- cd $FEAST_HOME/integration-tests/testdata
152+
- feast apply entity entity_specs/entity_1.yaml
153+
- feast apply feature feature_specs/entity_1*.yaml
154+
- feast jobs run import_specs/batch_from_gcs.yaml --wait
155+
156+
# Ensure correctness of warehouse and serving data
157+
- cd $FEAST_HOME/integration-tests
158+
- |
159+
python -m testutils.validate_feature_values \
160+
--entity_spec_file=testdata/entity_specs/entity_1.yaml \
161+
--feature_spec_files=testdata/feature_specs/entity_1*.yaml \
162+
--expected-warehouse-values-file=testdata/feature_values/ingestion_1.csv \
163+
--expected-serving-values-file=testdata/feature_values/serving_1.csv \
164+
--bigquery-dataset-for-warehouse=$FEAST_WAREHOUSE_DATASET \
165+
--feast-serving-url=$FEAST_SERVING_URL
166+
depends_on: [ integration_test_prepare ]
167+
168+
- name: integration_test_streaming
169+
image: python:3.7
170+
environment:
171+
FEAST_CORE_URL: build-${DRONE_BUILD_NUMBER}.feast.dragonin.me:80
172+
FEAST_SERVING_URL: build-${DRONE_BUILD_NUMBER}.feast.dragonin.me:80
173+
KAFKA_BROKERS: 10.128.0.55:31090
174+
KAFKA_TOPICS: feast_build_${DRONE_BUILD_NUMBER}
175+
commands:
176+
- FEAST_HOME=$(pwd)
177+
178+
# Install and use latest Feast SDK in the integration test
179+
- pip install -qe $FEAST_HOME/sdk/python
180+
- pip install -qr $FEAST_HOME/integration-tests/testutils/requirements.txt
181+
182+
# Use Feast CLI built from "build_feast_cli" step
183+
- cp ./cli/build/feast /usr/bin/feast
184+
- feast config set coreURI $FEAST_CORE_URL
185+
186+
# Register entity and feature specs, then start a streaming import job
187+
- cd $FEAST_HOME/integration-tests/testdata
188+
- feast apply entity entity_specs/entity_2.yaml
189+
- feast apply feature feature_specs/entity_2*.yaml
190+
# Assume Feast Core is configured to use DirectRunner
191+
# Hence we run "feast jobs" in background (because process will not return
192+
# unlike when using DataflowRunner), then wait about 20 seconds
193+
# which is normally long enough for Feast job to start listening to Kafka streams
194+
- feast jobs run import_specs/stream_from_kafka.yaml &
195+
- sleep 20
196+
197+
# Produce streaming feature values for ingestion
198+
# Normally it will take ~20 seconds for the values to appear in Feast Serving
199+
- cd $FEAST_HOME/integration-tests
200+
- |
201+
python -m testutils.kafka_producer \
202+
--bootstrap_servers=$KAFKA_BROKERS \
203+
--topic=$KAFKA_TOPICS \
204+
--entity_spec_file=testdata/entity_specs/entity_2.yaml \
205+
--feature_spec_files=testdata/feature_specs/entity_2*.yaml \
206+
--feature_values_file=testdata/feature_values/ingestion_2.csv
207+
- sleep 20
208+
209+
# Ensure correctness of serving data
210+
#
211+
# NOTE: Ignore checking correctness of warehouse data because with streaming
212+
# data it can take ~20 minutes for the data to be avaiable in warehouse,
213+
# a bit too long for the CI build job
214+
#
215+
# TODO: Consider a way to shorten the wait duration in Feast Ingestion
216+
# before loading the feature values to BigQuery so we can validate serving data too here.
217+
- cd $FEAST_HOME/integration-tests
218+
- |
219+
python -m testutils.validate_feature_values \
220+
--entity_spec_file=testdata/entity_specs/entity_2.yaml \
221+
--feature_spec_files=testdata/feature_specs/entity_2*.yaml \
222+
--expected-serving-values-file=testdata/feature_values/serving_2.csv \
223+
--feast-serving-url=$FEAST_SERVING_URL
224+
depends_on: [ integration_test_batch ]
225+
226+
- name: integration_test_cleanup
227+
image: devth/helm:v2.13.1
228+
environment:
229+
FEAST_WAREHOUSE_DATASET: feast_build_${DRONE_BUILD_NUMBER}
230+
JSON_KEY:
231+
from_secret: google_credentials
232+
commands:
233+
# Authentication to Google Cloud and Kube cluster
234+
- printf "%s\n" "$JSON_KEY" > /etc/service-account.json
235+
- gcloud auth activate-service-account --key-file /etc/service-account.json
236+
- gcloud container clusters get-credentials feast-test-cluster --zone us-central1-a --project kf-feast
237+
238+
- helm delete --purge feast-$DRONE_BUILD_NUMBER
239+
- bq rm -rf --dataset $FEAST_WAREHOUSE_DATASET
240+
depends_on: [ integration_test_batch, integration_test_streaming ]
241+
when:
242+
status: [ success, failure ]
243+
244+
# Save build dependencies to cache for faster future builds
245+
# ============================================================
246+
247+
- name: rebuild_cache
248+
image: homerovalle/drone-gcs-cache
249+
settings:
250+
pull: true
251+
bucket: feast-templocation-kf-feast
252+
path: feast-templocation-kf-feast/feast-mvn-cache
253+
rebuild: true
254+
json_key:
255+
from_secret: google_credentials
256+
mount:
257+
- .m2
258+
- .gocache
259+
depends_on:
260+
- test_feast_core
261+
- test_feast_serving
262+
- test_feast_ingestion
263+
- test_feast_cli
264+
when:
265+
status: [ success, failure ]

‎charts/feast/templates/core-deploy.yaml‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -76,7 +76,7 @@ spec:
7676
valueFrom:
7777
secretKeyRef:
7878
name: {{ template "postgresql.fullname" . }}
79-
key: postgres-password
79+
key: postgresql-password
8080
- name: LOG_TYPE
8181
value: {{ .Values.core.logType }}
8282
- name: PROJECT_ID

‎integration-tests/feast-helm-values.yaml.template‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -95,6 +95,8 @@ postgresql:
9595
provision: true
9696
persistence:
9797
enabled: false
98+
# Hardcoded password here because this Feast is only for testing
99+
postgresqlPassword: HaeYeeMa7eD1
98100

99101
redis:
100102
provision: true

0 commit comments

Comments
 (0)