feat: fix tests, move to controlplane/netassert image

Author: sublimino

Dockerfile

@@ -4,6 +4,8 @@ WORKDIR /code
 ENTRYPOINT ["/usr/local/bin/entrypoint.sh"]
 CMD ["npm", "test"]
 
+ENV GOSU_VERSION="1.10"
+
 RUN \
   bash -euxo pipefail -c "curl -sL https://deb.nodesource.com/setup_9.x | bash -x" \
   && DEBIAN_FRONTEND=noninteractive \
@@ -18,11 +20,31 @@ RUN \
       ssh \
       wget \
   \
-  && rm -rf /var/lib/apt/lists/*
+  && rm -rf /var/lib/apt/lists/* \
+  \
+  && ARCH="$(dpkg --print-architecture | awk -F- '{ print $NF }')" \
+  \
+  && wget -O /usr/local/bin/gosu "https://github.com/tianon/gosu/releases/download/${GOSU_VERSION}/gosu-${ARCH}" \
+  && chmod +x /usr/local/bin/gosu \
+  && gosu nobody true
+
+RUN \
+  adduser \
+    --shell /bin/bash \
+    --uid 30000 \
+    --gecos "" \
+    --disabled-password \
+    netassert \
+  && \
+  CACHE_DIR=/code/node_modules/.cache \
+  && mkdir -p "${CACHE_DIR}" \
+  && chown netassert -R "${CACHE_DIR}"
 
 COPY package.json /code/
 RUN npm install
 
+COPY node_modules/node-nmap/ /code/node_modules/node-nmap/
+
 # TODO(ajm) netassert doesn't run in the container yet
 COPY test/ /code/test/
 COPY entrypoint.sh yj netassert /usr/local/bin/

Jenkinsfile

@@ -7,10 +7,14 @@ pipeline {
     GIT_CREDENTIALS = "ssh-key-jenkins-bot"
   }
 
+  // stages is "all pipeline stages"
   stages {
+    // the name of this stage, represented in the stage view e.g. https://jenkins.ctlplane.io/job/netassert/
     stage('Build') {
+      // defines the "agent" aka "jenkins slave"
       agent {
         docker {
+          // always run in this image, it's got latest kubectl and is based from a google-managed image
           image 'docker.io/controlplane/gcloud-sdk:latest'
           args '-v /var/run/docker.sock:/var/run/docker.sock ' +
             '--user=root ' +
@@ -19,6 +23,7 @@ pipeline {
         }
       }
 
+      // here is the actual build for this stage
       steps {
         ansiColor('xterm') {
           sh 'make build CONTAINER_TAG="${CONTAINER_TAG}"'

Makefile

@@ -16,13 +16,20 @@ endif
 
 CONTAINER_TAG ?= $(GIT_TAG)
 CONTAINER_NAME := $(REGISTRY)/$(NAME):$(CONTAINER_TAG)
+TEST_CONTAINER_TAG := "testing"
+CONTAINER_NAME_TESTING := $(REGISTRY)/$(NAME):$(TEST_CONTAINER_TAG)
 
 TEST_FILE := "test/test-localhost-remote.yaml"
 
 export NAME REGISTRY BUILD_DATE GIT_MESSAGE GIT_SHA GIT_TAG CONTAINER_TAG CONTAINER_NAME
 
+.PHONY: all test
+.SILENT:
+
+all: help
+
 .PHONY: cluster
-cluster: ## builds a test cluster
+cluster: ## creates a test GKE cluster
 	@echo "+ $@"
 	gcloud container clusters create \
 	--zone europe-west2-a \
@@ -32,7 +39,7 @@ cluster: ## builds a test cluster
 	--num-nodes 1 \
 	--preemptible \
 	--enable-network-policy \
-	netassert
+	netassert-test
 
 .PHONY: build
 build: ## builds a docker image
@@ -52,9 +59,15 @@ push: ## pushes a docker image
 .PHONY: run-in-docker
 run-in-docker: ## runs the last build docker image inside docker
 	@echo "+ $@"
-	docker run -i \
+	set -x ;	docker run -i \
 		--net=host \
+		--cap-add NET_ADMIN \
+		--cap-add NET_RAW \
 		${DOCKER_ARGS} \
+		-v ~/.config/gcloud/:/root/.config/gcloud/ \
+		-v ~/.ssh/:/tmp/.ssh/:ro \
+		-v ~/.kube/:/root/.kube:ro \
+		-v $(shell readlink -f ~/.ssh/config):/tmp/ssh-config:ro \
 		-v /var/run/docker.sock:/var/run/docker.sock:ro \
 		"${CONTAINER_NAME}" ${ARGS}
 
@@ -71,16 +84,54 @@ rollcage-test: ## build, test, and push container, then run local tests
 	make rollcage && ./netassert test/test-all.yaml
 
 .PHONY: test
-test: ## build, test, and push container, then run local tests
+test: test-deploy ## build, test, and push container, then run local tests
 	@echo "+ $@"
-	make rollcage && ./netassert test/test-all.yaml
+	make build push CONTAINER_TAG="$(TEST_CONTAINER_TAG)" \
+		&& ./netassert \
+			--image ${CONTAINER_NAME_TESTING} \
+			test/test-all.yaml \
+		&& make run-in-docker \
+			CONTAINER_NAME=$(CONTAINER_NAME_TESTING) \
+			ARGS='netassert --image ${CONTAINER_NAME_TESTING} test/test-all.yaml'
+
+.PHONY: test-local
+test-local: test-deploy ## test from the local machine
+	@echo "+ $@"
+	./netassert \
+		--image ${CONTAINER_NAME_TESTING} \
+		test/test-all.yaml
+
+.PHONY: test-deploy
+test-deploy: ## deploy test services
+	@echo "+ $@"
+	set -x; for DEPLOYMENT_TYPE in \
+    frontend \
+    microservice \
+    database \
+    ; do \
+  	\
+    DEPLOYMENT="test-$${DEPLOYMENT_TYPE}"; \
+    kubectl run "$${DEPLOYMENT}" \
+      --image=busybox \
+      --labels=app=web,role="$${DEPLOYMENT_TYPE}" \
+      --requests='cpu=10m,memory=32Mi' \
+      --expose \
+      --port 80 \
+      -- sh -c "while true; do { printf 'HTTP/1.1 200 OK\r\n\n I am a $${DEPLOYMENT_TYPE}\n'; } | nc -l -p  80; done"; \
+  	\
+    kubectl scale deployment "$${DEPLOYMENT}" --replicas=3; \
+  done; \
+  \
+  kubectl apply -f resource/net-pol/web-deny-all.yaml -f resource/net-pol/test-services-allow.yaml;
+
 
 .PHONY: rollcage
 rollcage: ## build, test, and push the container
 	@echo "+ $@"
 	rollcage build run push \
 		--interactive false \
-	  --tag sublimino/scratch:dev --pull=false "npm test"  \
+	  --tag controlplane/netassert:none \
+	  --pull=false "npm test"  \
 		-- \
 	  --net=host \
 		--env DEBUG="" \
@@ -91,7 +142,8 @@ rollcage-docker: ## experimental, does not currently work with gcloud
 	@echo "+ $@"
 	rollcage build run push \
 		--interactive false \
-		--tag sublimino/scratch:dev --pull=false "npm test" \
+		--tag controlplane/netassert:none \
+		--pull=false "npm test" \
 		-- \
 	  --net=host \
 		--env DEBUG=1 \
@@ -125,7 +177,6 @@ $${ACTION}: \#\# help\n\
 .PHONY: help
 help: ## parse jobs and descriptions from this Makefile
 	@grep -E '^[ a-zA-Z0-9_-]+:([^=]|$$)' $(MAKEFILE_LIST) \
-    | grep -Ev '^help\b[[:space:]]*:' \
+    | grep -Ev '^(all|help)\b[[:space:]]*:' \
     | sort \
     | awk 'BEGIN {FS = ":.*?##"}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
-

entrypoint.sh

@@ -1,11 +1,31 @@
 #!/bin/bash
 
 set -euo pipefail
+DEBUG=1
 
 if [[ "${DEBUG:-}" != "" ]]; then
   set -x
 fi
 
+if [[ -e /var/run/docker.sock ]]; then
+  groupadd docker
+
+  # get gid of docker socket file
+  DOCKER_SOCK_GID=$(ls -ng /var/run/docker.sock | cut -f3 -d' ')
+
+  # get group of docker inside container
+  DOCKER_GID=$(getent group docker | cut -f3 -d: || true)
+
+  # if they don't match, adjust
+  if [[ ! -z "${DOCKER_SOCK_GID}" && "${DOCKER_SOCK_GID}" != "${DOCKER_GID}" ]]; then
+    groupmod -g "${DOCKER_SOCK_GID}" docker
+  fi
+
+  if ! groups netassert | grep -q docker; then
+    usermod -aG docker netassert
+  fi
+fi
+
 if [[ "${TEST_YAML:-}" != "" ]]; then
   echo "${TEST_YAML}" | base64 -d >/code/test/test.yaml
 fi
@@ -18,12 +38,38 @@ fi
 if [[ "${DEBUG:-}" != "" ]]; then
   pwd
   id
-  ls -lasp /root/ /root/.ssh/ || true
+  ls -lasp \
+    /home/netassert/ \
+    /home/netassert/.ssh/ || true
   echo "/code/test/test.yaml:"
   cat /code/test/test.yaml
 fi
 
+
 [[ -d ${HOME}/.parallel ]] || mkdir -p ${HOME}/.parallel || true
 [[ -f ${HOME}/.parallel/will-cite ]] || touch ~/.parallel/will-cite
 
+gosu netassert bash -c "$(cat << EOF
+[[ -d \${HOME}/.parallel ]] || mkdir -p \${HOME}/.parallel || true
+[[ -f \${HOME}/.parallel/will-cite ]] || touch ~/.parallel/will-cite
+EOF
+)"
+
+if [[ -d /tmp/.ssh ]]; then
+  cp -a /tmp/.ssh /home/netassert/
+fi
+if [[ -L /home/netassert/.ssh/config ]]; then
+ rm -f /home/netassert/.ssh/config
+fi
+if [[ -d /tmp/ssh-config ]]; then
+  cp -af /tmp/ssh-config /home/netassert/.ssh/config
+fi
+
+chown netassert -R /home/netassert
+
+# TODO(AJM) run without root
+#exec gosu netassert "${@}"
+pwd
+ls -lasp
+
 exec "${@}"

netassert

@@ -38,7 +38,7 @@ declare -a ARGUMENTS
 EXPECTED_NUM_ARGUMENTS=0
 ARGUMENTS=()
 FILENAME="test/test.yaml"
-NETASSERT_IMAGE_NAME="sublimino/scratch:dev"
+NETASSERT_IMAGE_NAME="controlplane/netassert:none"
 CONTAINER_BASE_NAME="netassert"
 CONTAINER_NAME="${CONTAINER_BASE_NAME}-$(cut -d- -f1 </proc/sys/kernel/random/uuid)"
 CONFIG=''
@@ -71,7 +71,6 @@ main() {
   iterate_k8s
 
   iterate_host
-
 }
 
 count_expected_tests() {
@@ -187,7 +186,7 @@ configure_parallel() {
   mkdir -p "${TMPDIR}" &>/dev/null || true
 
   (parallel --record-env)
-  export -f _iterate_k8s_worker _iterate_host_worker jqc success info debug error warning ssh_to_node log_message_prefix is_gke to_yaml
+  export -f _iterate_k8s_worker _iterate_host_worker jqc success info debug error warning ssh_to_node log_message_prefix is_gke to_yaml wait_safe
 
   export CONFIG TEMP_DIR THIS_SCRIPT DIR NETASSERT_IMAGE_NAME DEBUG IS_OFFLINE CONTAINER_BASE_NAME SSH_USER GCLOUD_SSH_OPTIONS
 }

package.json

@@ -4,7 +4,7 @@
   "description": "",
   "main": "index.js",
   "scripts": {
-    "test": "bash -c 'APP=\"ava --tap --concurrency=2\"; if [[ $(whoami) != root ]]; then echo \"WARNING: UDP requires root (currently $(whoami)). Requesting sudo\"; sudo ${APP}; else ${APP}; fi'"
+    "test": "bash -c 'APP=\"ava --tap --concurrency=2\"; if false; then echo \"WARNING: UDP requires root (currently $(whoami)). Requesting sudo\"; sudo ${APP}; else ${APP}; fi'"
   },
   "repository": {
     "type": "git",

test/test-localhost.yaml

@@ -2,7 +2,8 @@ host:
   localhost:
     localhost:
       - TCP:22
-      - -999
-      - -UDP:1234
-      - -UDP:555
+      - 39111
+#      - -999
+#      - -UDP:1234
+#      - -UDP:555