Skip to content

Commit 16f64fe

Browse files
khorshuhengkhorshuheng
authored
Terraform module to demonstrate Feast setup on GCP (feast-dev#1148)
* Terraform module to demonstrate Feast setup on GCP Signed-off-by: Khor Shu Heng <[email protected]> * Use local variable instead of template for helm values Signed-off-by: Khor Shu Heng <[email protected]> Co-authored-by: Khor Shu Heng <[email protected]>
1 parent a3e6e53 commit 16f64fe

File tree

8 files changed

+428
-0
lines changed

8 files changed

+428
-0
lines changed

‎infra/terraform/gcp/README.md‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,35 @@
1+
# Terraform config for feast on GCP
2+
3+
This serves as a guide on how to deploy Feast on GCP. At the end of this guide, we will have provisioned:
4+
1. GKE cluster
5+
2. Feast services running on GKE
6+
3. Google Memorystore (Redis) as online store
7+
4. Dataproc cluster
8+
4. Kafka running on GKE, exposed to the dataproc cluster via internal load balancer.
9+
10+
# Steps
11+
12+
1. Create a tfvars file, e.g. `my.tfvars`. A sample configuration is as below:
13+
14+
```
15+
gcp_project_name = "kf-feast"
16+
name_prefix = "feast-0-8"
17+
region = "asia-east1"
18+
gke_machine_type = "n1-standard-2"
19+
network = "default"
20+
subnetwork = "default"
21+
dataproc_staging_bucket = "kf-feast-dataproc-staging-test"
22+
```
23+
24+
3. Configure tf state backend, e.g.:
25+
```
26+
terraform {
27+
backend "gcs" {
28+
bucket = "<your bucket name>"
29+
prefix = "terraform/feast"
30+
}
31+
}
32+
```
33+
34+
3. Use `terraform apply -var-file="my.tfvars"` to deploy.
35+

‎infra/terraform/gcp/dataproc.tf‎

Lines changed: 68 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,68 @@
1+
resource "google_storage_bucket" "dataproc_staging_bucket" {
2+
name = var.dataproc_staging_bucket
3+
project = var.gcp_project_name
4+
location = var.region
5+
force_destroy = true
6+
}
7+
8+
resource "google_dataproc_autoscaling_policy" "feast_dataproc_cluster_asp" {
9+
policy_id = var.name_prefix
10+
location = var.region
11+
project = var.gcp_project_name
12+
13+
worker_config {
14+
min_instances = var.min_dataproc_worker_count
15+
max_instances = var.max_dataproc_worker_count
16+
}
17+
18+
basic_algorithm {
19+
yarn_config {
20+
graceful_decommission_timeout = "3600s"
21+
scale_down_factor = 0.5
22+
scale_up_factor = 0.5
23+
}
24+
}
25+
}
26+
27+
resource "google_dataproc_cluster" "feast_dataproc_cluster" {
28+
project = var.gcp_project_name
29+
name = var.name_prefix
30+
region = var.region
31+
32+
cluster_config {
33+
staging_bucket = google_storage_bucket.dataproc_staging_bucket.name
34+
35+
master_config {
36+
num_instances = 1
37+
machine_type = var.dataproc_master_instance_type
38+
disk_config {
39+
boot_disk_type = var.dataproc_master_disk_type
40+
boot_disk_size_gb = var.dataproc_master_disk_size
41+
}
42+
}
43+
44+
worker_config {
45+
num_instances = var.min_dataproc_worker_count
46+
machine_type = var.dataproc_worker_instance_type
47+
disk_config {
48+
boot_disk_type = var.dataproc_worker_disk_type
49+
boot_disk_size_gb = var.dataproc_worker_disk_size
50+
}
51+
}
52+
53+
gce_cluster_config {
54+
subnetwork = var.subnetwork
55+
service_account = google_service_account.feast_sa.email
56+
57+
internal_ip_only = true
58+
}
59+
60+
software_config {
61+
image_version = var.dataproc_image_version
62+
}
63+
64+
autoscaling_config {
65+
policy_uri = google_dataproc_autoscaling_policy.feast_dataproc_cluster_asp.name
66+
}
67+
}
68+
}

‎infra/terraform/gcp/feast.tf‎

Lines changed: 121 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,121 @@
1+
locals {
2+
feast_postgres_secret_name = "${var.name_prefix}-postgres-secret"
3+
feast_helm_values = {
4+
redis = {
5+
enabled = false
6+
}
7+
8+
kafka = {
9+
enabled = true
10+
externalAccess = {
11+
enabled = true
12+
service = {
13+
types = "LoadBalancer"
14+
port = 9094
15+
loadBalancerIPs = [google_compute_address.kafka_broker.address]
16+
loadBalancerSourceRanges = ["10.0.0.0/8"]
17+
annotations = {
18+
"cloud.google.com/load-balancer-type" = "Internal"
19+
}
20+
}
21+
}
22+
}
23+
24+
grafana = {
25+
enabled = false
26+
}
27+
28+
29+
postgresql = {
30+
existingSecret = local.feast_postgres_secret_name
31+
}
32+
33+
feast-core = {
34+
postgresql = {
35+
existingSecret = local.feast_postgres_secret_name
36+
}
37+
}
38+
39+
feast-online-serving = {
40+
enabled = true
41+
"application-override.yaml" = {
42+
feast = {
43+
core-host = "${var.name_prefix}-feast-core"
44+
core-grpc-port = 6565
45+
active_store = "online_store"
46+
stores = [
47+
{
48+
name = "online_store"
49+
type = "REDIS"
50+
config = {
51+
host = google_redis_instance.online_store.host
52+
port = 6379
53+
subscriptions = [
54+
{
55+
name = "*"
56+
project = "*"
57+
}
58+
]
59+
}
60+
}
61+
]
62+
}
63+
}
64+
}
65+
66+
feast-jupyter = {
67+
enabled = true
68+
envOverrides = {
69+
feast_redis_host = google_redis_instance.online_store.host,
70+
feast_redis_port = 6379,
71+
feast_spark_launcher = "dataproc"
72+
feast_dataproc_cluster_name = google_dataproc_cluster.feast_dataproc_cluster.name
73+
feast_dataproc_project = var.gcp_project_name
74+
feast_dataproc_region = var.region
75+
feast_spark_staging_location = "gs://${var.dataproc_staging_bucket}/artifacts/"
76+
feast_historical_feature_output_location : "gs://${var.dataproc_staging_bucket}/out/"
77+
feast_historical_feature_output_format : "parquet"
78+
demo_kafka_brokers : "${google_compute_address.kafka_broker.address}:9094"
79+
demo_data_location : "gs://${var.dataproc_staging_bucket}/test-data/"
80+
}
81+
gcpServiceAccount = {
82+
enabled = true
83+
name = var.feast_sa_secret_name
84+
key = "credentials.json"
85+
}
86+
}
87+
}
88+
}
89+
90+
resource "random_password" "feast-postgres-password" {
91+
length = 16
92+
special = false
93+
}
94+
95+
resource "kubernetes_secret" "feast-postgres-secret" {
96+
metadata {
97+
name = local.feast_postgres_secret_name
98+
}
99+
data = {
100+
postgresql-password = random_password.feast-postgres-password.result
101+
}
102+
}
103+
104+
resource "google_compute_address" "kafka_broker" {
105+
project = var.gcp_project_name
106+
region = var.region
107+
subnetwork = var.subnetwork
108+
name = "${var.name_prefix}-kafka"
109+
address_type = "INTERNAL"
110+
}
111+
112+
resource "helm_release" "feast" {
113+
depends_on = [kubernetes_secret.feast-postgres-secret, kubernetes_secret.feast_sa_secret]
114+
115+
name = var.name_prefix
116+
chart = "../../charts/feast"
117+
118+
values = [
119+
yamlencode(local.feast_helm_values)
120+
]
121+
}

‎infra/terraform/gcp/gke.tf‎

Lines changed: 19 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,19 @@
1+
resource "google_container_cluster" "feast_gke_cluster" {
2+
name = "${var.name_prefix}-cluster"
3+
location = var.region
4+
network = var.network
5+
subnetwork = var.subnetwork
6+
7+
initial_node_count = var.gke_node_count
8+
node_config {
9+
machine_type = var.gke_machine_type
10+
}
11+
12+
ip_allocation_policy {
13+
}
14+
}
15+
16+
data "google_container_cluster" "feast_gke_cluster" {
17+
location = var.region
18+
name = google_container_cluster.feast_gke_cluster.name
19+
}

‎infra/terraform/gcp/iam.tf‎

Lines changed: 37 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
resource "google_service_account" "feast_sa" {
2+
account_id = var.name_prefix
3+
display_name = var.name_prefix
4+
project = var.gcp_project_name
5+
}
6+
7+
resource "google_service_account_key" "feast_sa" {
8+
service_account_id = google_service_account.feast_sa.name
9+
}
10+
11+
resource "google_project_iam_member" "feast_dataproc_worker" {
12+
project = var.gcp_project_name
13+
role = "roles/dataproc.worker"
14+
member = "serviceAccount:${google_service_account.feast_sa.email}"
15+
}
16+
17+
resource "google_project_iam_member" "feast_dataproc_editor" {
18+
project = var.gcp_project_name
19+
role = "roles/dataproc.editor"
20+
member = "serviceAccount:${google_service_account.feast_sa.email}"
21+
}
22+
23+
resource "google_project_iam_member" "feast_batch_ingestion_storage" {
24+
project = var.gcp_project_name
25+
role = "roles/storage.admin"
26+
member = "serviceAccount:${google_service_account.feast_sa.email}"
27+
}
28+
29+
resource "kubernetes_secret" "feast_sa_secret" {
30+
metadata {
31+
name = var.feast_sa_secret_name
32+
}
33+
data = {
34+
"credentials.json" = base64decode(google_service_account_key.feast_sa.private_key)
35+
}
36+
}
37+

‎infra/terraform/gcp/online_store.tf‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,18 @@
1+
resource "google_redis_instance" "online_store" {
2+
project = var.gcp_project_name
3+
region = var.region
4+
name = "${var.name_prefix}-online-store"
5+
tier = var.redis_tier
6+
memory_size_gb = var.redis_memory_size_gb
7+
8+
authorized_network = data.google_compute_network.redis-network.id
9+
10+
redis_version = "REDIS_5_0"
11+
display_name = "Feast Online Store"
12+
13+
}
14+
15+
data "google_compute_network" "redis-network" {
16+
project = var.gcp_project_name
17+
name = var.network
18+
}

‎infra/terraform/gcp/provider.tf‎

Lines changed: 27 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,27 @@
1+
provider "google" {
2+
version = "~> 3.46"
3+
project = var.gcp_project_name
4+
}
5+
6+
data "google_client_config" "gcp_client" {
7+
provider = google
8+
}
9+
10+
provider "kubernetes" {
11+
version = "~> 1.13.3"
12+
13+
host = google_container_cluster.feast_gke_cluster.endpoint
14+
token = data.google_client_config.gcp_client.access_token
15+
cluster_ca_certificate = base64decode(google_container_cluster.feast_gke_cluster.master_auth.0.cluster_ca_certificate)
16+
load_config_file = false
17+
}
18+
19+
provider "helm" {
20+
version = "~> 1.3.2"
21+
kubernetes {
22+
host = google_container_cluster.feast_gke_cluster.endpoint
23+
token = data.google_client_config.gcp_client.access_token
24+
cluster_ca_certificate = base64decode(google_container_cluster.feast_gke_cluster.master_auth.0.cluster_ca_certificate)
25+
load_config_file = false
26+
}
27+
}

0 commit comments

Comments
 (0)