integration tests: test deploying as restricted admin (sourcegraph#570)

uwedeportivo · uwedeportivo · commit b51b7184211b · 2020-04-06T08:32:40.000-07:00
* integration tests: test deploying as restricted admin

* hook it in for a try

* try again

* try again

* try again

* try again

* try again

* try again

* try again

* fixed bug revealed by test

* comment out restricted test for now

* add to pipeline

* add to pipeline 2

* add to pipeline 3

* test ray

* forgot namespace

* no rollout for service

* not https

* code review keegan

* integration test label

* migrate doc calls out required kustomize

* migrate doc calls out required kustomize 2

* testing prune

* revert prune testing

* fresh create needs different overlay chain

* lsif-server to precise code intel rename
diff --git a/.buildkite/integration-restricted-test.sh b/.buildkite/integration-restricted-test.sh
@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+set -ex
+
+cd $(dirname "${BASH_SOURCE[0]}")/..
+
+export DEPLOY_SOURCEGRAPH_ROOT=$(pwd)
+export TEST_GCP_PROJECT=sourcegraph-server
+export TEST_GCP_ZONE=us-central1-a
+export TEST_GCP_USERNAME=buildkite@sourcegraph-dev.iam.gserviceaccount.com
+export BUILD_CREATOR=$BUILDKITE_BUILD_CREATOR
+export BUILD_UUID=$BUILDKITE_BUILD_ID
+export BUILD_BRANCH=$BUILDKITE_BRANCH
+
+${DEPLOY_SOURCEGRAPH_ROOT}/tests/integration/restricted/test.sh
+
diff --git a/.buildkite/integration-test.sh b/.buildkite/integration-test.sh
@@ -15,3 +15,4 @@ export GENERATED_BASE=`mktemp -d`
 kustomize build overlays/non-root-create-cluster -o ${GENERATED_BASE}
 
 go test ./... -v -timeout 25m ${maybe_short_flag}
+
diff --git a/.buildkite/pipeline.yml b/.buildkite/pipeline.yml
@@ -15,6 +15,11 @@ steps:
     concurrency_group: deploy-sourcegraph/integration
     command: .buildkite/integration-test.sh
 
+  - label: ":k8s:"
+    concurrency: 5
+    concurrency_group: deploy-sourcegraph/integration-restricted
+    command: .buildkite/integration-restricted-test.sh
+
   - wait
   
   - label: ":gcompute: :floppy_disk: 🧹"
diff --git a/base/kustomization.yaml b/base/kustomization.yaml
@@ -4,12 +4,10 @@ resources:
   - searcher/searcher.Service.yaml
   - searcher/searcher.Deployment.yaml
   - backend.Service.yaml
-  - frontend/sourcegraph-frontend.Role.yaml
   - frontend/sourcegraph-frontend-internal.Service.yaml
   - frontend/sourcegraph-frontend.Deployment.yaml
   - frontend/sourcegraph-frontend.ServiceAccount.yaml
   - frontend/sourcegraph-frontend.Service.yaml
-  - frontend/sourcegraph-frontend.RoleBinding.yaml
   - frontend/sourcegraph-frontend.Ingress.yaml
   - redis/redis-store.Service.yaml
   - redis/redis-store.Deployment.yaml
@@ -33,11 +31,9 @@ resources:
   - pgsql/pgsql.Service.yaml
   - pgsql/pgsql.Deployment.yaml
   - prometheus/prometheus.ServiceAccount.yaml
-  - prometheus/prometheus.ClusterRoleBinding.yaml
   - prometheus/prometheus.ConfigMap.yaml
   - prometheus/prometheus.Deployment.yaml
   - prometheus/prometheus.PersistentVolumeClaim.yaml
-  - prometheus/prometheus.ClusterRole.yaml
   - prometheus/prometheus.Service.yaml
   - query-runner/query-runner.Service.yaml
   - query-runner/query-runner.Deployment.yaml
diff --git a/base/rbac-roles/kustomization.yaml b/base/rbac-roles/kustomization.yaml
@@ -0,0 +1,7 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+  - sourcegraph-frontend.Role.yaml
+  - sourcegraph-frontend.RoleBinding.yaml
+  - prometheus.ClusterRoleBinding.yaml
+  - prometheus.ClusterRole.yaml
diff --git a/base/rbac-roles/prometheus.ClusterRole.yaml b/base/rbac-roles/prometheus.ClusterRole.yaml
diff --git a/base/rbac-roles/prometheus.ClusterRoleBinding.yaml b/base/rbac-roles/prometheus.ClusterRoleBinding.yaml
diff --git a/base/rbac-roles/sourcegraph-frontend.Role.yaml b/base/rbac-roles/sourcegraph-frontend.Role.yaml
diff --git a/base/rbac-roles/sourcegraph-frontend.RoleBinding.yaml b/base/rbac-roles/sourcegraph-frontend.RoleBinding.yaml
diff --git a/docs/migrate.md b/docs/migrate.md
@@ -6,6 +6,18 @@ version you are upgrading to should be applied (unless otherwise noted).
 
 ## 3.14 (Unreleased)
 
+The `kubectl-apply-all.sh` command now uses `kustomize` and requires `kubectl` client version >= 1.14. 
+
+If your kubectl client version is older and doesn't support `apply -k` you need to 
+install the standalone [kustomize](https://kustomize.io/) binary, generate the YAML files with `kustomize build` and
+then use the built YAML with `kubectl apply -f`. For example use:
+
+```shell script
+kustomize build base | kubectl apply -f -
+kustomize build base/rbac-roles | kubectl apply -f -
+```
+in your version of `kubectl-apply-all.sh` if you cannot upgrade `kubectl` to a client version >= 1.14.
+
 ### Existing installations: Migrating the container user from root to non-root
 
 Version 3.14 changes the security context of the installation by switching to a non-root user for all containers.
diff --git a/overlays/migrate-to-nonroot/kustomization.yaml b/overlays/migrate-to-nonroot/kustomization.yaml
@@ -2,6 +2,7 @@ apiVersion: kustomize.config.k8s.io/v1beta1
 kind: Kustomization
 bases:
   - ../../base
+  - ../../base/rbac-roles
 patchesStrategicMerge:
   - gitserver/gitserver.StatefulSet.yaml
   - grafana/grafana.Deployment.yaml
diff --git a/overlays/minikube/kustomization.yaml b/overlays/minikube/kustomization.yaml
@@ -3,6 +3,7 @@ kind: Kustomization
 namespace: ns-sourcegraph
 bases:
   - ../../base
+  - ../../base/rbac-roles
 patchesJson6902:
   - target:
       kind: Deployment
diff --git a/overlays/namespaced/kustomization.yaml b/overlays/namespaced/kustomization.yaml
@@ -3,3 +3,4 @@ kind: Kustomization
 namespace: ns-sourcegraph
 bases:
   - ../../base
+  - ../../base/rbac-roles
diff --git a/overlays/non-privileged-create-cluster/README.md b/overlays/non-privileged-create-cluster/README.md
@@ -0,0 +1,10 @@
+This kustomization is for Sourcegraph installations in clusters with security restrictions.
+It avoids creating `Roles` and does all the rolebinding in a namespace. It configures Prometheus to work in the namespace
+and not require ClusterRole wide privileges when doing service discovery for scraping targets.
+
+This version and `non-privileged` need to stay in sync. This version is only used for cluster creation.
+
+```shell script
+cd overlays/non-privileged
+kubectl -n ns-sourcegraph apply -l deploy=sourcegraph,rbac-admin!=escalated -k .
+```
diff --git a/overlays/non-privileged-create-cluster/frontend/sourcegraph-frontend.RoleBinding.yaml b/overlays/non-privileged-create-cluster/frontend/sourcegraph-frontend.RoleBinding.yaml
@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    category: rbac
+    deploy: sourcegraph
+  name: sourcegraph-frontend-nonprivileged
+roleRef:
+  apiGroup: ""
+  kind: ClusterRole
+  name: view
+subjects:
+  - kind: ServiceAccount
+    name: sourcegraph-frontend
+    namespace: ns-sourcegraph
diff --git a/overlays/non-privileged-create-cluster/kustomization.yaml b/overlays/non-privileged-create-cluster/kustomization.yaml
@@ -0,0 +1,10 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+namespace: ns-sourcegraph
+bases:
+  - ../non-root-create-cluster
+resources:
+  - frontend/sourcegraph-frontend.RoleBinding.yaml
+  - prometheus/prometheus.RoleBinding.yaml
+patchesStrategicMerge:
+  - prometheus/prometheus.ConfigMap.yaml
diff --git a/overlays/non-privileged-create-cluster/prometheus/prometheus.ConfigMap.yaml b/overlays/non-privileged-create-cluster/prometheus/prometheus.ConfigMap.yaml
@@ -0,0 +1,74 @@
+apiVersion: v1
+data:
+  prometheus.yml: |
+    global:
+      scrape_interval:     30s
+      evaluation_interval: 30s
+
+    alerting:
+      alertmanagers:
+        - kubernetes_sd_configs:
+          - role: endpoints
+            namespaces:
+              names:
+                - ns-sourcegraph
+          relabel_configs:
+            - source_labels: [__meta_kubernetes_service_name]
+              regex: alertmanager
+              action: keep
+
+    rule_files:
+      - '*_rules.yml'
+      - "/sg_config_prometheus/*_rules.yml"
+      - "/sg_prometheus_add_ons/*_rules.yml"
+
+    scrape_configs:
+    - job_name: 'kubernetes-service-endpoints'
+
+      kubernetes_sd_configs:
+      - role: endpoints
+        namespaces:
+          names:
+           - ns-sourcegraph
+
+      relabel_configs:
+      - source_labels: [__meta_kubernetes_service_annotation_sourcegraph_prometheus_scrape]
+        action: keep
+        regex: true
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_scheme]
+        action: replace
+        target_label: __scheme__
+        regex: (https?)
+      - source_labels: [__meta_kubernetes_service_annotation_prometheus_io_path]
+        action: replace
+        target_label: __metrics_path__
+        regex: (.+)
+      - source_labels: [__address__, __meta_kubernetes_service_annotation_prometheus_io_port]
+        action: replace
+        target_label: __address__
+        regex: (.+)(?::\d+);(\d+)
+        replacement: $1:$2
+      - action: labelmap
+        regex: __meta_kubernetes_service_label_(.+)
+      - source_labels: [__meta_kubernetes_namespace]
+        action: replace
+        # Sourcegraph specific customization. We want a more convenient to type label.
+        # target_label: kubernetes_namespace
+        target_label: ns
+      - source_labels: [__meta_kubernetes_service_name]
+        action: replace
+        target_label: kubernetes_name
+      # Sourcegraph specific customization. We want a nicer name for job
+      - source_labels: [app]
+        action: replace
+        target_label: job
+      # Sourcegraph specific customization. We want a nicer name for instance
+      - source_labels: [__meta_kubernetes_pod_name]
+        action: replace
+        target_label: instance
+  node_rules.yml: ""
+kind: ConfigMap
+metadata:
+  labels:
+    deploy: sourcegraph
+  name: prometheus
diff --git a/overlays/non-privileged-create-cluster/prometheus/prometheus.RoleBinding.yaml b/overlays/non-privileged-create-cluster/prometheus/prometheus.RoleBinding.yaml
@@ -0,0 +1,15 @@
+apiVersion: rbac.authorization.k8s.io/v1
+kind: RoleBinding
+metadata:
+  labels:
+    category: rbac
+    deploy: sourcegraph
+  name: prometheus-nonprivileged
+roleRef:
+  apiGroup: ""
+  kind: ClusterRole
+  name: view
+subjects:
+  - kind: ServiceAccount
+    name: prometheus
+    namespace: ns-sourcegraph
diff --git a/overlays/non-privileged/frontend/sourcegraph-frontend.RoleBinding.yaml b/overlays/non-privileged/frontend/sourcegraph-frontend.RoleBinding.yaml
@@ -12,3 +12,4 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: sourcegraph-frontend
+    namespace: ns-sourcegraph
diff --git a/overlays/non-privileged/prometheus/prometheus.RoleBinding.yaml b/overlays/non-privileged/prometheus/prometheus.RoleBinding.yaml
@@ -12,3 +12,4 @@ roleRef:
 subjects:
   - kind: ServiceAccount
     name: prometheus
+    namespace: ns-sourcegraph
diff --git a/tests/integration/restricted/nonroot-policy.yaml b/tests/integration/restricted/nonroot-policy.yaml
@@ -0,0 +1,27 @@
+apiVersion: policy/v1beta1
+kind: PodSecurityPolicy
+metadata:
+  name: nonroot-policy
+spec:
+  privileged: false
+  allowPrivilegeEscalation: false
+  # The rest fills in some required fields.
+  seLinux:
+    rule: RunAsAny
+  supplementalGroups:
+    rule: 'MustRunAs'
+    ranges:
+      # Forbid adding the root group.
+      - min: 1
+        max: 65535
+  runAsUser:
+    rule: 'MustRunAsNonRoot'
+  fsGroup:
+    rule: 'MustRunAs'
+    ranges:
+      # Forbid adding the root group.
+      - min: 1
+        max: 65535
+  volumes:
+    - '*'
+  readOnlyRootFilesystem: true
diff --git a/tests/integration/restricted/sourcegraph.StorageClass.yaml b/tests/integration/restricted/sourcegraph.StorageClass.yaml
@@ -0,0 +1,9 @@
+kind: StorageClass
+apiVersion: storage.k8s.io/v1
+metadata:
+  name: sourcegraph
+  labels:
+    deploy: sourcegraph-storage
+provisioner: kubernetes.io/gce-pd
+parameters:
+  type: pd-ssd # This configures SSDs (recommended).
diff --git a/tests/integration/restricted/test.sh b/tests/integration/restricted/test.sh
@@ -0,0 +1,56 @@
+#!/usr/bin/env bash
+
+set -ex
+
+CLUSTER_NAME_SUFFIX=`echo ${BUILD_UUID} | head -c 8`
+
+CLUSTER_NAME="ds-test-restricted-${CLUSTER_NAME_SUFFIX}"
+
+cd $(dirname "${BASH_SOURCE[0]}")
+
+# set up the cluster, set up the fake user and restricted policy and then deploy the non-privileged overlay as that user
+
+gcloud container clusters create ${CLUSTER_NAME} --zone ${TEST_GCP_ZONE} --num-nodes 3 --machine-type n1-standard-16 --disk-type pd-ssd --project ${TEST_GCP_PROJECT} --labels=cost-category=build,build-creator=${BUILD_CREATOR},build-branch=${BUILD_BRANCH},integration-test=fresh
+
+gcloud container clusters get-credentials ${CLUSTER_NAME} --zone ${TEST_GCP_ZONE} --project ${TEST_GCP_PROJECT}
+
+kubectl create clusterrolebinding cluster-admin-binding --clusterrole cluster-admin --user ${TEST_GCP_USERNAME}
+
+kubectl apply -f sourcegraph.StorageClass.yaml
+
+kubectl apply -f nonroot-policy.yaml
+
+kubectl create namespace ns-sourcegraph
+
+kubectl create serviceaccount -n ns-sourcegraph fake-user
+
+kubectl create rolebinding -n ns-sourcegraph fake-admin --clusterrole=admin --serviceaccount=ns-sourcegraph:fake-user
+
+kubectl create role -n ns-sourcegraph nonroot:unprivileged --verb=use --resource=podsecuritypolicy --resource-name=nonroot-policy
+
+kubectl create rolebinding -n ns-sourcegraph fake-user:nonroot:unprivileged --role=nonroot:unprivileged --serviceaccount=ns-sourcegraph:fake-user
+
+kubectl --as=system:serviceaccount:ns-sourcegraph:fake-user -n ns-sourcegraph apply -k ${DEPLOY_SOURCEGRAPH_ROOT}/overlays/non-privileged-create-cluster
+
+kubectl  -n ns-sourcegraph expose deployment sourcegraph-frontend --type=NodePort --name sourcegraph --type=LoadBalancer
+
+# wait for it all to finish (we list out the ones with persistent volume claim because they take longer)
+
+kubectl -n ns-sourcegraph rollout status -w statefulset/indexed-search
+kubectl -n ns-sourcegraph rollout status -w deployment/precise-code-intel-bundle-manager
+kubectl -n ns-sourcegraph rollout status -w deployment/prometheus
+kubectl -n ns-sourcegraph rollout status -w deployment/redis-cache
+kubectl -n ns-sourcegraph rollout status -w deployment/redis-store
+kubectl -n ns-sourcegraph rollout status -w statefulset/gitserver
+kubectl -n ns-sourcegraph rollout status -w deployment/sourcegraph-frontend
+
+# hit it with one request
+
+SOURCEGRAPH_IP=`kubectl -n ns-sourcegraph describe service sourcegraph | grep "LoadBalancer Ingress:" | cut -d ":" -f 2 | tr -d " "`
+
+curl -m 10 http://${SOURCEGRAPH_IP}:3080
+
+# delete cluster
+
+gcloud container clusters delete ${CLUSTER_NAME} --zone ${TEST_GCP_ZONE} --project ${TEST_GCP_PROJECT} --quiet
+

Original file line number	Diff line number	Diff line change
@@ -15,3 +15,4 @@ export GENERATED_BASE=`mktemp -d`
`15`	`15`	`kustomize build overlays/non-root-create-cluster -o ${GENERATED_BASE}`
`16`	`16`
`17`	`17`	`go test ./... -v -timeout 25m ${maybe_short_flag}`
	`18`	`+`