Initial commit

Author: cgsimmons

.circleci/config.yml

@@ -0,0 +1,32 @@
+version: 2
+jobs:
+  publish-image:
+    machine: true
+    steps:
+      - checkout
+      - run: |
+          echo $JSON_KEYFILE | docker login -u _json_key --password-stdin https://gcr.io
+          ./build.sh
+  send_notification:
+    machine: true
+    steps:
+      - checkout
+      - run: |
+          CONTAINER_NAME=gcr.io/$GCLOUD_PROJECT/kubernetes-deploy:$(utils/get_build_tag.sh)
+          curl -X POST --data-urlencode \
+          "payload={\"channel\": \"#container-builds\", \"username\": \"Bob the Builder\", \"text\": \"Built and published kubernetes-deploy: $CONTAINER_NAME\", \"icon_emoji\": \":construction_worker:\"}" \
+          $SLACK_HOOK_URL;
+
+workflows:
+  version: 2
+  build:
+    jobs:
+      - publish-image:
+          context: org-global
+          filters:
+            branches:
+              only: master
+      - send_notification:
+          context: org-global
+          requires:
+            - publish-image

.flake8

@@ -0,0 +1,2 @@
+[flake8]
+max-line-length = 140

.gitignore

@@ -0,0 +1,6 @@
+*.pyc
+env
+venv
+.env
+.venv
+.vscode

Dockerfile

@@ -0,0 +1,14 @@
+FROM python:3.7-stretch
+
+WORKDIR /run/app
+
+RUN echo "deb [signed-by=/usr/share/keyrings/cloud.google.gpg] http://packages.cloud.google.com/apt cloud-sdk main" | tee -a /etc/apt/sources.list.d/google-cloud-sdk.list && curl https://packages.cloud.google.com/apt/doc/apt-key.gpg | apt-key --keyring /usr/share/keyrings/cloud.google.gpg  add - && apt-get update -y && apt-get install google-cloud-sdk -y
+
+COPY requirements.txt ./
+
+RUN pip install -r requirements.txt
+
+COPY . .
+COPY ./run.sh /usr/local/bin
+
+ENTRYPOINT ["run.sh"]

LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2020 Adgorithmics Inc
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

README.md

@@ -0,0 +1,59 @@
+# kubernetes-deploy
+
+A script to deploy projects to kubernetes. Designed to run as a kubernetes job.
+
+## Setup
+1. Give all of your deployements the same PROJECT label as well as a fitting TIER label. Every deployment (and cronjob) with the PROJECT label will be updated in the deployment. The TIER label will determine the order in which to scale down, update, scale up deployments in the case of a cold database migration. Deployments can use different images, but they must all use the same TAG. See `Features` and `Optional` env variables below for more details.
+
+## Features
+-   Migration Job - If you would like to trigger database migrations, setup a command with on one of your deployment images that can be used to run the database migration process. Provide this deployment name as APP_MIGRATOR_SOURCE env variable as well as pass the command and args via APP_MIGRATOR_COMMAND and APP_MIGRATOR_ARGS env variables. You will also need to define the DATABASE_* env variables to perform the necessary backup to Google Storage. If the `migration` option is set to `1` (hot migration - no scale down), or `2` (cold migration - scale down and up deployments) then the deployment script will first backup the database, scale down deployments (if cold migration), fetch the APP_MIGRATOR_SOURCE deployment and update the image tag, command and args, run the migration, update all other deployment images, scale back up deployments (if cold migration).
+-   Trello list cleanup - If you pass the necessary trello and mailgun env variables (with TRELLO_SEND_NOTIFICATION flag is True) the deployment script will collect all cards in the trello list, send out a notification email with their details, and archive the cards.
+-   Cronjob support - If you give cronjobs the same PROJECT label, they will also be updated in the final stage of the deployment.
+
+## Environment Variables
+
+### Required
+
+-   SLACK_TOKEN - For slack notifications
+-   PROJECT - Matches the `project` label on the deployments
+
+### Required if performing migrations (`migration` option set above 0)
+-   DATABASE_INSTANCE_NAME - Cloud SQL instance name
+-   DATABASE_NAME - Cloud SQL database name
+-   DATABASE_BACKUP_BUCKET - GS URI (gs://bucket/directory/etc) database backup location (will append `/PROJECT`)
+-   APP_MIGRATOR_SOURCE - The name of the deployment to use as the base configuration for migration jobs
+
+### Required if TRELLO_SEND_NOTIFICATION flag is True
+
+-   TRELLO_KEY - Trello api key
+-   TRELLO_TOKEN - Trello api token
+-   TRELLO_LIST_ID - Target trello list for release notification generation
+-   MAILGUN_DOMAIN - Mailgun domain
+-   MAILGUN_KEY - Mailgun api key
+-   MAILGUN_TO - Recipients for release notification email
+
+### Optional (with defaults for development cinnamon deployment)
+
+-   GOOGLE_APPLICATION_CREDENTIALS - Path to gcloud authentication json key file. Not needed if you have local gcloud auth initialized
+-   DEBUG [`False`] - Will authorize kubeApi with local gcloud when `True`
+-   DISABLED [`False`] - Exits process without deploying when `True`
+-   APP_ENV [`development`] - App environment (production, development) to construct slack notification
+-   HOSTNAME [`localhost`] - Host running this process (provided by Kubernetes)
+-   NAMESPACE [`default`] - Pod namespace
+-   SLACK_CHANNEL [`dev-null`] - Target channel for slack notifications
+-   TIERS [`frontend,scheduler,worker,gateway,apiserver`] - Comma separated list of deployments (in scale down order)
+-   TRELLO_SEND_NOTIFICATION [`False`] - Cleanup trello list and send release notification via email
+-   APP_MIGRATOR_COMMAND = [`npm`] - A comma separated list of commands to on the migration job container
+-   APP_MIGRATOR_ARGS = [`run,--prefix,/app,migration:run`] - A comma separated list of args to set on the migration job container
+
+## Required Arguments
+
+-   -t, --tag - The new monolith image tag to roll out (`dev-20.02.18-36b17ee`)
+-   -m, --migration - The migration level: 0=None, 1=Hot, 2=Cold
+
+## Running Locally
+
+Just add all of the necessary env variables and run `deploy.py` and pass the tag and migration options.
+
+-   `pip install -r requirements.txt`
+-   `SLACK_TOKEN=<token> PROJECT=<your project> DEBUG=True python deploy.py --tag=<image tag> --migration=<migration level (0, 1, 2)>`

build.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+
+# Takes image build tag from script argument or generates from date and short sha
+TAG_ARG=$1
+BUILD_TAG=${TAG_ARG:-$(utils/get_build_tag.sh)}
+BASE_IMAGE=gcr.io/$GCLOUD_PROJECT/kubernetes-deploy
+
+if [ "${BOOTLEG}" = 'true' ]; then
+    random=$(
+        head /dev/urandom | tr -dc A-Za-z0-9 | head -c 13
+        echo ''
+    )
+    prefix="bootleg-$random-"
+    TAG=$BASE_IMAGE:$prefix$BUILD_TAG
+    docker build . --tag=$TAG
+    docker push "$TAG"
+    echo "BOOTLEG IMAGE PUSHED: $TAG"
+else
+    LATEST_TAG=$BASE_IMAGE:latest
+    TAG=$BASE_IMAGE:$BUILD_TAG
+    docker build . --tag=$TAG --tag=$LATEST_TAG
+    docker push "$TAG"
+    docker push $LATEST_TAG
+    echo "IMAGE PUSHED: $TAG"
+fi

config.py

@@ -0,0 +1,54 @@
+import os
+
+# -------- ENV --------
+#  Debug will authorize kube api with local gcloud auth when True
+DEBUG = os.getenv("DEBUG", False) in ["true", "True"]
+DISABLED = os.getenv("DISABLED", False) in ["true", "True"]
+# Determines the slack notification info
+APP_ENV = os.getenv("APP_ENV", "development")
+
+# -------- Project, Pod, Cluster --------
+PROJECT = os.getenv("PROJECT")
+HOST_NAME = os.getenv("HOSTNAME", "localhost")
+NAMESPACE = os.getenv("NAMESPACE", "default")
+
+# -------- Slack --------
+SLACK_TOKEN = os.getenv("SLACK_TOKEN")
+SLACK_CHANNEL = os.getenv("SLACK_CHANNEL", "dev-null")
+
+# -------- Trello --------
+TRELLO_KEY = os.getenv("TRELLO_KEY")
+TRELLO_TOKEN = os.getenv("TRELLO_TOKEN")
+TRELLO_LIST_ID = os.getenv("TRELLO_LIST_ID")
+TRELLO_SEND_NOTIFICATION = os.getenv("TRELLO_SEND_NOTIFICATION", False) in [
+    "true",
+    "True",
+]
+
+# -------- Mailgun --------
+MAILGUN_DOMAIN = os.getenv("MAILGUN_DOMAIN")
+MAILGUN_KEY = os.getenv("MAILGUN_KEY")
+MAILGUN_TO = os.getenv("MAILGUN_TO")
+
+# -------- Database backup --------
+DATABASE_INSTANCE_NAME = os.getenv("DATABASE_INSTANCE_NAME")
+DATABASE_NAME = os.getenv("DATABASE_NAME")
+DATABASE_BACKUP_BUCKET = f"{os.getenv('DATABASE_BACKUP_BUCKET')}/{DATABASE_NAME}"
+
+# -------- Migrate job --------
+APP_MIGRATOR_SOURCE = os.getenv("APP_MIGRATOR_SOURCE")
+# comma separated list
+APP_MIGRATOR_COMMAND = os.getenv("APP_MIGRATOR_COMMAND", "npm").split(",")
+# comma separated list
+APP_MIGRATOR_ARGS = os.getenv(
+    "APP_MIGRATOR_ARGS", "run,--prefix,/app,migration:run"
+).split(",")
+
+# -------- Deployment Tiers --------
+# comma separated listed in scale down order
+TIERS = os.getenv("TIERS", "frontend,scheduler,worker,gateway,apiserver").split(",")
+# frontend - public facing frontend
+# scheduler - service scheduler
+# worker - service queue workers
+# gateway - public facing api gateway
+# apiserver - service apiserver / internal gateway