feat: Add webhook to create an editor site

Signed-off-by: Andrew Burnes <[email protected]>

Author: apburnes

.cloudgov/manifest.yml

@@ -30,29 +30,31 @@ applications:
       - mailer
       - slack
     env:
-      NODE_ENV: ((node_env))
-      APP_ENV: ((env))
       ADMIN_HOSTNAME: https://admin.((domain))
+      APP_ENV: ((env))
       APP_HOSTNAME: https://((domain))
+      CF_CDN_SPACE_NAME: ((cf_cdn_space_name))
+      CF_DOMAIN_WITH_CDN_PLAN_GUID: 440cce96-8989-428b-a60e-91c1393bf3f2
       CLOUD_FOUNDRY_API_HOST: https://api.fr.cloud.gov
       CLOUD_FOUNDRY_OAUTH_TOKEN_URL: https://login.fr.cloud.gov/oauth/token
       DOMAIN: ((domain))
+      FEATURE_FILE_STORAGE_SERVICE: ((feature_file_storage_service))
       HOMEPAGE_URL: https://cloud.gov/pages
       LOG_LEVEL: ((log_level))
-      NPM_CONFIG_PRODUCTION: true
-      NODE_MODULES_CACHE: false
-      USER_AUDITOR: federalist
-      S3_SERVICE_PLAN_ID: F36820DC-FDB6-496C-9D96-68861F5D0D95
-      UAA_HOST: ((uaa_host))
       NEW_RELIC_APP_NAME: ((new_relic_app_name))
-      PROXY_DOMAIN: ((proxy_domain))
-      PRODUCT: pages
-      CF_CDN_SPACE_NAME: ((cf_cdn_space_name))
-      CF_DOMAIN_WITH_CDN_PLAN_GUID: 440cce96-8989-428b-a60e-91c1393bf3f2
       NEW_RELIC_ERROR_COLLECTOR_ENABLED: false
+      NODE_ENV: ((node_env))
+      NODE_MODULES_CACHE: false
+      NPM_CONFIG_PRODUCTION: true
+      OPS_EMAIL: [email protected]
+      PAGES_EDITOR_HOST: https://pages-editor((env_postfix)).app.cloud.gov
+      PRODUCT: pages
+      PROXY_DOMAIN: ((proxy_domain))
       QUEUES_BUILD_TASKS_CONCURRENCY: ((queues_build_tasks_concurrency))
       QUEUES_SITE_BUILDS_CONCURRENCY: ((queues_site_builds_concurrency))
-      FEATURE_FILE_STORAGE_SERVICE: ((feature_file_storage_service))
+      S3_SERVICE_PLAN_ID: F36820DC-FDB6-496C-9D96-68861F5D0D95
+      UAA_HOST: ((uaa_host))
+      USER_AUDITOR: federalist
   - name: pages-admin((env_postfix))
     buildpack: staticfile_buildpack
     path: ../admin-client

.cloudgov/vars/pages-dev.yml

@@ -1,16 +1,15 @@
-product: pages
-name: pages-dev
+cf_cdn_space_name: sites-dev
 domain: pages-dev.cloud.gov
-instances: 2
 env: dev
 env_postfix: -dev
+feature_file_storage_service: "true"
+instances: 2
 log_level: verbose
-uaa_host: https://uaa.fr-stage.cloud.gov
-uaa_login_host: https://login.fr-stage.cloud.gov
+name: pages-dev
 new_relic_app_name: web-pages-dev
-proxy_domain: sites.pages-dev.cloud.gov
-cf_cdn_space_name: sites-dev
 node_env: development
+proxy_domain: sites.pages-dev.cloud.gov
 queues_build_tasks_concurrency: 2
 queues_site_builds_concurrency: 3
-feature_file_storage_service: "true"
+uaa_host: https://uaa.fr-stage.cloud.gov
+uaa_login_host: https://login.fr-stage.cloud.gov

.cloudgov/vars/pages-production.yml

@@ -1,16 +1,15 @@
-product: pages
-name: pages-production
+cf_cdn_space_name: sites
 domain: pages.cloud.gov
-instances: 4
 env: production
 env_postfix: -production
+feature_file_storage_service: "false"
+instances: 4
 log_level: info
-uaa_host: https://uaa.fr.cloud.gov
-uaa_login_host: https://login.fr.cloud.gov
+name: pages-production
 new_relic_app_name: web-pages-production
-proxy_domain: sites.pages.cloud.gov
-cf_cdn_space_name: sites
 node_env: production
+proxy_domain: sites.pages.cloud.gov
 queues_build_tasks_concurrency: 10
 queues_site_builds_concurrency: 10
-feature_file_storage_service: "false"
+uaa_host: https://uaa.fr.cloud.gov
+uaa_login_host: https://login.fr.cloud.gov

.cloudgov/vars/pages-staging.yml

@@ -1,16 +1,15 @@
-product: pages
-name: pages-staging
+cf_cdn_space_name: sites-staging
 domain: pages-staging.cloud.gov
-instances: 2
 env: staging
 env_postfix: -staging
+feature_file_storage_service: "true"
+instances: 2
 log_level: verbose
-uaa_host: https://uaa.fr-stage.cloud.gov
-uaa_login_host: https://login.fr-stage.cloud.gov
+name: pages-staging
 new_relic_app_name: web-pages-staging
-proxy_domain: sites.pages-staging.cloud.gov
-cf_cdn_space_name: sites-staging
 node_env: production
+proxy_domain: sites.pages-staging.cloud.gov
 queues_build_tasks_concurrency: 4
 queues_site_builds_concurrency: 4
-feature_file_storage_service: "true"
+uaa_host: https://uaa.fr-stage.cloud.gov
+uaa_login_host: https://login.fr-stage.cloud.gov

api/bull-board/app.js

@@ -10,6 +10,7 @@ const slowDown = require('express-slow-down');
 const {
   ArchiveBuildLogsQueue,
   BuildTasksQueue,
+  CreateEditorSiteQueue,
   DomainQueue,
   FailStuckBuildsQueue,
   MailQueue,
@@ -38,6 +39,7 @@ createBullBoard({
     new BullMQAdapter(new SiteBuildsQueue(connection)),
     new BullMQAdapter(new ArchiveBuildLogsQueue(connection)),
     new BullMQAdapter(new BuildTasksQueue(connection)),
+    new BullMQAdapter(new CreateEditorSiteQueue(connection)),
     new BullMQAdapter(new DomainQueue(connection)),
     new BullMQAdapter(new FailStuckBuildsQueue(connection)),
     new BullMQAdapter(new MailQueue(connection)),

api/controllers/webhook.js

@@ -1,5 +1,12 @@
 const { wrapHandlers } = require('../utils');
 const Webhooks = require('../services/Webhooks');
+const { decrypt } = require('../services/Encryptor');
+const { encryption } = require('../../config');
+const QueueJobs = require('../queue-jobs');
+const { createQueueConnection } = require('../utils/queues');
+
+const connection = createQueueConnection();
+const queueJob = new QueueJobs(connection);
 
 module.exports = wrapHandlers({
   async github(req, res) {
@@ -17,4 +24,24 @@ module.exports = wrapHandlers({
 
     res.ok();
   },
+
+  async site(req, res) {
+    const { body } = req;
+
+    const userEmail = decrypt(body.userEmail, encryption.key);
+    const apiKey = decrypt(body.apiKey, encryption.key);
+    const siteId = decrypt(body.siteId, encryption.key);
+    const siteName = decrypt(body.siteName, encryption.key);
+    const orgName = decrypt(body.org, encryption.key);
+
+    await queueJob.startCreateEditorSiteTask({
+      userEmail,
+      apiKey,
+      siteId,
+      siteName,
+      orgName,
+    });
+
+    return res.ok();
+  },
 });

api/queue-jobs/index.js

@@ -1,6 +1,11 @@
 const moment = require('moment');
 const PromisePool = require('@supercharge/promise-pool');
-const { BuildTasksQueue, MailQueue, SiteBuildsQueue } = require('../queues');
+const {
+  BuildTasksQueue,
+  CreateEditorSiteQueue,
+  MailQueue,
+  SiteBuildsQueue,
+} = require('../queues');
 const Templates = require('../services/mailer/templates');
 const { truncateString } = require('../utils');
 const {
@@ -11,6 +16,7 @@ class QueueJobs {
   constructor(connection) {
     this.siteBuildsQueue = new SiteBuildsQueue(connection);
     this.buildTasksQueue = new BuildTasksQueue(connection);
+    this.createEditorSiteQueue = new CreateEditorSiteQueue(connection);
     this.mailQueue = new MailQueue(connection);
   }
 
@@ -27,7 +33,7 @@ class QueueJobs {
     await this.mailQueue.waitUntilReady();
 
     return this.mailQueue.add('alert', {
-      to: ['[email protected]'],
+      to: [process.env.OPS_EMAIL],
       subject: `${appName} ${appEnv} Alert | ${reason}`,
       html: Templates.alert({ errors, reason }),
     });
@@ -147,6 +153,35 @@ class QueueJobs {
     return this.buildTasksQueue.add(name, { buildTaskId }, { priority });
   }
 
+  /**
+   * Adds a create editor site task job to the Create Editor Site Queue
+   * The editor sites's manager email, org name, site name, site id, and bot api key
+   * are used to create the site job added to the queue
+   * @async
+   * @method startCreateEditorSiteTask
+   * @param {Object} editorSite - An instance of the model Build Taks
+   * * @param {number} editorSite.userEmail - The editor site's manager email for the org
+   * * @param {Object} editorSite.orgName - The name of the org
+   * * @param {Object} editorSite.siteName - The name of the site
+   * * @param {Object} editorSite.siteName - The id of the site
+   * * @param {Object} editorSite.apiKey - The bot api key
+   * @return {Promise<{Object}>} The bullmq's queue add job response
+   */
+  async startCreateEditorSiteTask(editorSite) {
+    const { userEmail, orgName, siteName, siteId, apiKey } = editorSite;
+    const jobName = `Creating editor site for org ${orgName}`;
+
+    await this.createEditorSiteQueue.waitUntilReady();
+
+    return this.createEditorSiteQueue.add(jobName, {
+      userEmail,
+      orgName,
+      siteName,
+      siteId,
+      apiKey,
+    });
+  }
+
   /**
    * Adds a site build job to the Site Builds Queue
    * The build's branch, site owner, and site repository attributes

api/queues/CreateEditorSiteQueue.js

@@ -0,0 +1,19 @@
+const { Queue } = require('bullmq');
+
+const CreateEditorSiteQueueName = 'create-editor-site';
+
+class CreateEditorSiteQueue extends Queue {
+  constructor(connection, { attempts = 1 } = {}) {
+    super(CreateEditorSiteQueueName, {
+      connection,
+      defaultJobOptions: {
+        attempts,
+      },
+    });
+  }
+}
+
+module.exports = {
+  CreateEditorSiteQueue,
+  CreateEditorSiteQueueName,
+};

api/queues/index.js

@@ -3,6 +3,10 @@ const {
   ArchiveBuildLogsQueueName,
 } = require('./ArchiveBuildLogsQueue');
 const { BuildTasksQueue, BuildTasksQueueName } = require('./BuildTasksQueue');
+const {
+  CreateEditorSiteQueue,
+  CreateEditorSiteQueueName,
+} = require('./CreateEditorSiteQueue');
 const { DomainQueue, DomainQueueName } = require('./DomainQueue');
 const {
   FailStuckBuildsQueue,
@@ -23,6 +27,8 @@ module.exports = {
   ArchiveBuildLogsQueueName,
   BuildTasksQueue,
   BuildTasksQueueName,
+  CreateEditorSiteQueue,
+  CreateEditorSiteQueueName,
   DomainQueue,
   DomainQueueName,
   FailStuckBuildsQueue,

api/routers/webhook.js

@@ -32,7 +32,32 @@ function verifySignature(req, res, next) {
   next();
 }
 
+function verifySiteRequest(req, res, next) {
+  const { body: payload } = req;
+  const expectedKeys = ['userEmail', 'apiKey', 'siteId', 'siteName', 'org'].sort();
+
+  // ToDo Add additional headers to check if request is legit
+
+  try {
+    const payloadKeys = Object.keys(payload).sort();
+
+    if (payloadKeys.length !== expectedKeys.length) {
+      throw new Error('Invalid request payload');
+    }
+
+    const hasKeys = payloadKeys.every((value, index) => value === expectedKeys[index]);
+
+    if (!hasKeys) throw new Error('Invalid request payload');
+  } catch (err) {
+    res.badRequest();
+    next(err);
+  }
+
+  next();
+}
+
 router.post('/webhook/github', verifySignature, WebhookController.github);
 router.post('/webhook/organization', verifySignature, WebhookController.organization);
+router.post('/webhook/site', verifySiteRequest, WebhookController.site);
 
 module.exports = router;

api/services/SiteCreator.js

@@ -1,6 +1,6 @@
 const GitHub = require('./GitHub');
 const TemplateResolver = require('./TemplateResolver');
-const { Build, Site, User } = require('../models');
+const { Build, Organization, Site, User } = require('../models');
 const { generateS3ServiceName, generateSubdomain } = require('../utils');
 const CloudFoundryAPIClient = require('../utils/cfApiClient');
 const config = require('../../config');
@@ -44,12 +44,18 @@ function ownerIsFederalistUser(owner) {
   });
 }
 
-function checkSiteExists({ owner, repository }) {
+function checkSiteExists({ owner, repository, organizationId }) {
   return Site.findOne({
     where: {
       owner,
       repository,
     },
+    include: [
+      {
+        model: Organization,
+        where: { id: organizationId },
+      },
+    ],
   }).then((existingSite) => {
     if (existingSite) {
       const error = new Error(
@@ -183,11 +189,12 @@ async function saveAndBuildSite({ site, user }) {
 }
 
 async function createSiteFromExistingRepo({ siteParams, user }) {
-  const { owner, repository } = siteParams;
+  const { owner, repository, organizationId } = siteParams;
 
   await checkSiteExists({
     owner,
     repository,
+    organizationId,
   });
   const repo = await checkGithubRepository({
     user,

api/services/Webhooks.js

@@ -6,7 +6,7 @@ const EventCreator = require('./EventCreator');
 const findSiteForWebhookRequest = (payload) => {
   const [owner, repository] = payload.repository.full_name.split('/');
 
-  return Site.findOne({
+  return Site.findAll({
     where: {
       owner: owner.toLowerCase(),
       repository: repository.toLowerCase(),
@@ -101,12 +101,17 @@ const createBuildForWebhookRequest = async (payload, site) => {
 
 const pushWebhookRequest = async (payload) => {
   if (payload.commits && payload.commits.length > 0) {
-    const site = await findSiteForWebhookRequest(payload);
-    if (shouldBuildForSite(site)) {
-      const build = await createBuildForWebhookRequest(payload, site);
-      await build.reload({ include: Site });
-      await GithubBuildHelper.reportBuildStatus(build);
-    }
+    const sites = await findSiteForWebhookRequest(payload);
+
+    await Promise.all(
+      sites.map(async (site) => {
+        if (shouldBuildForSite(site)) {
+          const build = await createBuildForWebhookRequest(payload, site);
+          await build.reload({ include: Site });
+          await GithubBuildHelper.reportBuildStatus(build);
+        }
+      }),
+    );
   }
 };