Skip to content

Commit a99005c

Browse files
oavdeevoavdeev
authored
docs, fixes and scripts to run e2e tests in minikube (feast-dev#1254)
Signed-off-by: Oleg Avdeev <[email protected]>
1 parent 77f7c1f commit a99005c

File tree

15 files changed

+353
-30
lines changed

15 files changed

+353
-30
lines changed

‎.dockerignore‎

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -2,3 +2,4 @@ docs
22
!docs/coverage
33
charts
44
env
5+
**/.terraform

‎.gitignore‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -184,3 +184,8 @@ sdk/python/docs/html
184184
*_pb2.py
185185
*_pb2.pyi
186186
*_pb2_grpc.py
187+
188+
# VSCode
189+
.bloop
190+
.metals
191+
*.code-workspace

‎Makefile‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -156,6 +156,9 @@ build-ci-docker:
156156
build-jupyter-docker:
157157
docker build -t $(REGISTRY)/feast-jupyter:$(VERSION) -f infra/docker/jupyter/Dockerfile .
158158

159+
build-local-test-docker:
160+
docker build -t feast:local -f infra/docker/tests/Dockerfile .
161+
159162
# Documentation
160163

161164
install-dependencies-proto-docs:

‎infra/docker/tests/Dockerfile‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,24 @@
1+
ARG BASE_IMAGE=gcr.io/kf-feast/feast-ci:latest
2+
3+
FROM ${BASE_IMAGE}
4+
5+
RUN mkdir -p /src/sdk /src/spark/ingestion
6+
7+
COPY sdk/python /src/sdk/python
8+
9+
COPY README.md /src/README.md
10+
11+
WORKDIR /src
12+
13+
RUN pip install -r sdk/python/requirements-ci.txt
14+
15+
RUN git init .
16+
RUN pip install -e sdk/python -U
17+
RUN pip install "s3fs" "boto3" "urllib3>=1.25.4"
18+
19+
COPY tests /src/tests
20+
21+
RUN pip install -r tests/requirements.txt
22+
23+
COPY infra/scripts /src/infra/scripts
24+
COPY spark/ingestion/target /src/spark/ingestion/target

‎infra/scripts/azure-runner.sh‎

Lines changed: 6 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -12,7 +12,7 @@ GIT_REMOTE_URL=https://github.com/feast-dev/feast.git
1212
echo "########## Starting e2e tests for ${GIT_REMOTE_URL} ${GIT_TAG} ###########"
1313

1414
# Note requires running in root feast directory
15-
source infra/scripts/runner-helper.sh
15+
source infra/scripts/k8s-common-functions.sh
1616

1717
# Workaround for COPY command in core docker image that pulls local maven repo into the image
1818
# itself.
@@ -34,8 +34,12 @@ RELEASE=sparkop
3434
# Delete old helm release and PVCs
3535
k8s_cleanup "$RELEASE" "$NAMESPACE"
3636

37+
wait_for_images "${DOCKER_REPOSITORY}" "${GIT_TAG}"
38+
3739
# Helm install everything in a namespace
38-
helm_install "$RELEASE" "${DOCKER_REPOSITORY}" "${GIT_TAG}" --namespace "$NAMESPACE"
40+
helm_install "$RELEASE" "${DOCKER_REPOSITORY}" "${GIT_TAG}" "$NAMESPACE" \
41+
--set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_NAME=${AZURE_BLOB_ACCOUNT_NAME}" \
42+
--set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_ACCESS_KEY=${AZURE_BLOB_ACCOUNT_ACCESS_KEY}"
3943

4044
# Delete old test running pod if it exists
4145
kubectl delete pod -n "$NAMESPACE" ci-test-runner 2>/dev/null || true

‎infra/scripts/runner-helper.sh‎ renamed to ‎infra/scripts/k8s-common-functions.sh‎

Lines changed: 23 additions & 12 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,13 @@
1-
#!/usr/bin/env bash
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
function wait_for_images {
6+
local DOCKER_REPOSITORY=$1
7+
local GIT_TAG=$2
8+
# Wait for images to be available in the docker repository; ci is the last image built
9+
timeout 15m bash -c "while ! gcloud container images list-tags ${DOCKER_REPOSITORY}/feast-ci --format=json | jq -e \".[] | select(.tags[] | contains (\\\"${GIT_TAG}\\\"))\" > /dev/null; do sleep 10s; done"
10+
}
211

312
function k8s_cleanup {
413
local RELEASE=$1
@@ -17,8 +26,12 @@ function k8s_cleanup {
1726

1827
# Set a new postgres password. Note that the postgres instance is not available outside
1928
# the k8s cluster anyway so it doesn't have to be super secure.
20-
echo "${STEP_BREADCRUMB} Setting PG password"
21-
PG_PASSWORD=$(head -c 59 /dev/urandom | md5sum | head -c 16)
29+
echo "${STEP_BREADCRUMB:-} Setting PG password"
30+
31+
# use either shasum or md5sum, whichever exists
32+
SUM=$(which md5sum shasum | grep -v "not found" | tail -n1 || true )
33+
34+
PG_PASSWORD=$(head -c 59 /dev/urandom | $SUM | head -c 16)
2235
kubectl delete secret feast-postgresql -n "$NAMESPACE" || true
2336
kubectl create secret generic feast-postgresql --from-literal=postgresql-password="$PG_PASSWORD" -n "$NAMESPACE"
2437
}
@@ -30,36 +43,34 @@ function helm_install {
3043
# $RELEASE is helm release name
3144
# $DOCKER_REPOSITORY is the docker repo containing feast images tagged with $GIT_TAG
3245
# ... you can pass additional args to this function that are passed on to helm install
46+
# $NAMESPACE is the namespace name
3347

3448
local RELEASE=$1
3549
local DOCKER_REPOSITORY=$2
3650
local GIT_TAG=$3
51+
local NAMESPACE=$4
3752

38-
shift 3
39-
40-
# Wait for images to be available in the docker repository; ci is the last image built
41-
timeout 15m bash -c "while ! gcloud container images list-tags ${DOCKER_REPOSITORY}/feast-ci --format=json | jq -e \".[] | select(.tags[] | contains (\\\"${GIT_TAG}\\\"))\" > /dev/null; do sleep 10s; done"
53+
shift 4
4254

4355
# We skip statsd exporter and other metrics stuff since we're not using it anyway, and it
4456
# has some issues with unbound PVCs (that cause kubectl delete pvc to hang).
45-
echo "${STEP_BREADCRUMB} Helm installing feast"
57+
echo "${STEP_BREADCRUMB:-} Helm installing feast"
4658

47-
if ! time helm install --wait "$RELEASE" infra/charts/feast \
59+
if ! time helm install --wait "$RELEASE" ./infra/charts/feast \
4860
--timeout 15m \
4961
--set "feast-jupyter.image.repository=${DOCKER_REPOSITORY}/feast-jupyter" \
5062
--set "feast-jupyter.image.tag=${GIT_TAG}" \
5163
--set "feast-online-serving.image.repository=${DOCKER_REPOSITORY}/feast-serving" \
5264
--set "feast-online-serving.image.tag=${GIT_TAG}" \
5365
--set "feast-jobservice.image.repository=${DOCKER_REPOSITORY}/feast-jobservice" \
5466
--set "feast-jobservice.image.tag=${GIT_TAG}" \
55-
--set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_NAME=${AZURE_BLOB_ACCOUNT_NAME}" \
56-
--set "feast-jobservice.envOverrides.FEAST_AZURE_BLOB_ACCOUNT_ACCESS_KEY=${AZURE_BLOB_ACCOUNT_ACCESS_KEY}" \
5767
--set "feast-core.image.repository=${DOCKER_REPOSITORY}/feast-core" \
5868
--set "feast-core.image.tag=${GIT_TAG}" \
5969
--set "prometheus-statsd-exporter.enabled=false" \
6070
--set "prometheus.enabled=false" \
6171
--set "grafana.enabled=false" \
6272
--set "feast-jobservice.enabled=false" \
73+
--namespace "$NAMESPACE" \
6374
"$@" ; then
6475

6576
echo "Error during helm install. "
@@ -105,4 +116,4 @@ subjects:
105116
- kind: ServiceAccount
106117
name: default
107118
EOF
108-
}
119+
}

‎infra/scripts/run-minikube-test.sh‎

Lines changed: 38 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,38 @@
1+
#!/bin/bash
2+
3+
set -euo pipefail
4+
5+
NAMESPACE=sparkop
6+
JOB_NAME=test-runner
7+
8+
# Delete all sparkapplication resources that may be left over from the previous test runs.
9+
kubectl delete sparkapplication --all -n sparkop || true
10+
11+
JOB_SPEC=$(dirname $0)/test_job.yaml
12+
13+
# Delete previous instance of the job if it exists
14+
kubectl delete -n ${NAMESPACE} "job/$JOB_NAME" 2>/dev/null || true
15+
16+
# Create the job
17+
kubectl apply -n ${NAMESPACE} -f "$JOB_SPEC"
18+
19+
# Wait for job to have a pod.
20+
for i in {1..10}
21+
do
22+
POD=$(kubectl get pods -n ${NAMESPACE} --selector=job-name=$JOB_NAME --output=jsonpath='{.items[0].metadata.name}')
23+
if [ ! -z "$POD" ]; then
24+
break
25+
else
26+
sleep 1
27+
fi
28+
done
29+
30+
echo "Waiting for pod to be ready:"
31+
kubectl wait -n ${NAMESPACE} --for=condition=ContainersReady "pod/$POD" --timeout=60s || true
32+
33+
echo "Job output:"
34+
kubectl logs -n ${NAMESPACE} -f "job/$JOB_NAME"
35+
36+
# Can't wait for both conditions at once, so wait for complete first then wait for failure
37+
kubectl wait -n ${NAMESPACE} --for=condition=complete "job/$JOB_NAME" --timeout=60s && exit 0
38+
kubectl wait -n ${NAMESPACE} --for=condition=failure "job/$JOB_NAME" --timeout=60s && exit 1

‎infra/scripts/setup-e2e-local.sh‎

Lines changed: 26 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,26 @@
1+
#!/bin/bash
2+
set -euo pipefail
3+
4+
STEP_BREADCRUMB='~~~~~~~~'
5+
6+
pushd "$(dirname $0)"
7+
source k8s-common-functions.sh
8+
9+
# spark k8s test - runs in sparkop namespace (so it doesn't interfere with a concurrently
10+
# running EMR test).
11+
NAMESPACE=sparkop
12+
RELEASE=sparkop
13+
14+
# Clean up old release
15+
k8s_cleanup "$RELEASE" "$NAMESPACE"
16+
17+
# Helm install everything in a namespace
18+
helm_install "$RELEASE" "${DOCKER_REPOSITORY}" "${GIT_TAG}" "$NAMESPACE" --create-namespace
19+
20+
# Delete all sparkapplication resources that may be left over from the previous test runs.
21+
kubectl delete sparkapplication --all -n "$NAMESPACE" || true
22+
23+
# Make sure the test pod has permissions to create sparkapplication resources
24+
setup_sparkop_role
25+
26+
echo "DONE"

‎infra/scripts/test-end-to-end-local.sh‎

Lines changed: 69 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,69 @@
1+
#!/usr/bin/env bash
2+
3+
set -euo pipefail
4+
5+
export DISABLE_FEAST_SERVICE_FIXTURES=1
6+
export DISABLE_SERVICE_FIXTURES=1
7+
8+
export FEAST_SPARK_K8S_NAMESPACE=sparkop
9+
export FEAST_S3_ENDPOINT_URL=http://minio.minio.svc.cluster.local:9000
10+
11+
# Used by tests
12+
export AWS_S3_ENDPOINT_URL=http://minio.minio.svc.cluster.local:9000
13+
14+
cat << SPARK_CONF_END >/tmp/spark_conf.yml
15+
apiVersion: "sparkoperator.k8s.io/v1beta2"
16+
kind: SparkApplication
17+
metadata:
18+
namespace: default
19+
spec:
20+
type: Scala
21+
mode: cluster
22+
image: "gcr.io/kf-feast/spark-py:v3.0.1"
23+
imagePullPolicy: Always
24+
sparkVersion: "3.0.1"
25+
timeToLiveSeconds: 3600
26+
pythonVersion: "3"
27+
sparkConf:
28+
"spark.hadoop.fs.s3a.endpoint": http://minio.minio.svc.cluster.local:9000
29+
"spark.hadoop.fs.s3a.path.style.access": "true"
30+
"spark.hadoop.fs.s3a.access.key": ${AWS_ACCESS_KEY_ID}
31+
"spark.hadoop.fs.s3a.secret.key": ${AWS_SECRET_ACCESS_KEY}
32+
restartPolicy:
33+
type: Never
34+
volumes:
35+
- name: "test-volume"
36+
hostPath:
37+
path: "/tmp"
38+
type: Directory
39+
driver:
40+
cores: 1
41+
coreLimit: "1200m"
42+
memory: "512m"
43+
labels:
44+
version: 3.0.1
45+
serviceAccount: spark
46+
volumeMounts:
47+
- name: "test-volume"
48+
mountPath: "/tmp"
49+
executor:
50+
cores: 1
51+
instances: 1
52+
memory: "512m"
53+
labels:
54+
version: 3.0.1
55+
volumeMounts:
56+
- name: "test-volume"
57+
mountPath: "/tmp"
58+
SPARK_CONF_END
59+
export FEAST_SPARK_K8S_JOB_TEMPLATE_PATH=/tmp/spark_conf.yml
60+
61+
PYTHONPATH=sdk/python pytest tests/e2e/ \
62+
--feast-version develop \
63+
--core-url sparkop-feast-core:6565 \
64+
--serving-url sparkop-feast-online-serving:6566 \
65+
--env k8s \
66+
--staging-path s3a://feast-staging \
67+
--redis-url sparkop-redis-master.sparkop.svc.cluster.local:6379 \
68+
--kafka-brokers sparkop-kafka.sparkop.svc.cluster.local:9092 \
69+
-m "not bq"

‎infra/scripts/test_job.yaml‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
apiVersion: batch/v1
2+
kind: Job
3+
metadata:
4+
name: test-runner
5+
namespace: sparkop
6+
spec:
7+
backoffLimit: 1
8+
template:
9+
spec:
10+
containers:
11+
- name: ubuntu
12+
image: feast:local
13+
command: ["bash", "-c", "./infra/scripts/test-end-to-end-local.sh"]
14+
imagePullPolicy: Never
15+
args:
16+
- bash
17+
stdin: true
18+
stdinOnce: true
19+
tty: true
20+
env:
21+
- name: AWS_ACCESS_KEY_ID
22+
valueFrom:
23+
secretKeyRef:
24+
name: minio
25+
key: accesskey
26+
- name: AWS_SECRET_ACCESS_KEY
27+
valueFrom:
28+
secretKeyRef:
29+
name: minio
30+
key: secretkey
31+
- name: AWS_DEFAULT_REGION
32+
value: us-east-1
33+
- name: AWS_S3_SIGNATURE_VERSION
34+
value: s3v4
35+
restartPolicy: Never

0 commit comments

Comments
 (0)