FEAT: Change cf templates to use vcs for bootstrap

Author: bhosking

cloud/aws-emr/bootstrap/install-hail.sh

@@ -25,6 +25,10 @@ while [ $# -gt 0 ]; do
       shift
       SPARK_VERSION=$1
       ;;
+    --path-prefix)
+      # not used by this script
+      shift
+      ;;
     -*)
       error_msg "unrecognized option: $1"
       ;;

cloud/aws-emr/bootstrap/install-jupyter-noNotebook.sh

@@ -29,6 +29,10 @@ while [ $# -gt 0 ]; do
       shift
       NotebookPath=$1
       ;;
+    --path-prefix)
+      # Passed in by default, but not used here
+      shift
+      ;;
     -*)
       error_msg "unrecognized option: $1"
       ;;

cloud/aws-emr/cf-templates/VariantSpark_Hail_EMR_Notebook.yaml

@@ -1,6 +1,11 @@
 ---
 AWSTemplateFormatVersion: '2010-09-09'
 Description: Variant Spark CF template to spin EMR clusters with Jupyter Notebook.
+Mappings:
+  Constants:
+    GitHub:
+      Repo: "https://github.com/bhosking/VariantSpark"
+      Version: "bootstrap-vcs"
 Parameters:
   # Hardware Group
   MasterNodePricing:
@@ -115,19 +120,33 @@ Resources:
       - Name: Install Jupyter
         ScriptBootstrapAction:
           Args:
+            - "--path-prefix"
+            - !Sub
+              - "${repo}/raw/${version}"
+              - {repo: !FindInMap [Constants, GitHub, Repo], version: !FindInMap [Constants, GitHub, Version]}
+            - "--bootstrap-file"
+            - "/cloud/aws-emr/bootstrap/install-jupyter.sh"
+            - "--"
             - "--notebookPath"
             - Ref: NotebookDir
-          Path: "s3://variant-spark/HailJupyter/install-jupyter.sh"
+          Path: "s3://variant-spark/s3-bootstrap.sh"
       - Name: Install Hail
         ScriptBootstrapAction:
           Args:
+            - "--path-prefix"
+            - !Sub
+              - "${repo}/raw/${version}"
+              - {repo: !FindInMap [Constants, GitHub, Repo], version: !FindInMap [Constants, GitHub, Version]}
+            - "--bootstrap-file"
+            - "/cloud/aws-emr/bootstrap/install-hail.sh"
+            - "--"
             - "--input-path"
             - "s3://variant-spark/HailJupyter/hail/0.1_2.2.1"
             - "--hail-version"
             - "0.1"
             - "--spark-version"
             - "2.2.1"
-          Path: "s3://variant-spark/HailJupyter/install-hail.sh"
+          Path: "s3://variant-spark/s3-bootstrap.sh"
       Configurations:
       - Classification: emrfs-site
         ConfigurationProperties:

cloud/aws-emr/cf-templates/VariantSpark_Hail_EMR_Step.yaml

@@ -1,6 +1,11 @@
 ---
 AWSTemplateFormatVersion: '2010-09-09'
 Description: Variant Spark CF template to spin EMR cluster to process a single VariantSpark job.
+Mappings:
+  Constants:
+    GitHub:
+      Repo: "https://github.com/bhosking/VariantSpark"
+      Version: "bootstrap-vcs"
 Parameters:
   # Random-Forest Parameters Group
   NumTree:
@@ -183,17 +188,31 @@ Resources:
       BootstrapActions:
       - Name: Install Jupyter
         ScriptBootstrapAction:
-          Path: "s3://variant-spark/HailJupyter/install-jupyter-noNotebook.sh"
+            - "--path-prefix"
+            - !Sub
+              - "${repo}/raw/${version}"
+              - {repo: !FindInMap [Constants, GitHub, Repo], version: !FindInMap [Constants, GitHub, Version]}
+            - "--bootstrap-file"
+            - "/cloud/aws-emr/bootstrap/install-jupyter-noNotebook.sh"
+            - "--"
+          Path: "s3://variant-spark/s3-bootstrap.sh"
       - Name: Install Hail
         ScriptBootstrapAction:
           Args:
+            - "--path-prefix"
+            - !Sub
+              - "${repo}/raw/${version}"
+              - {repo: !FindInMap [Constants, GitHub, Repo], version: !FindInMap [Constants, GitHub, Version]}
+            - "--bootstrap-file"
+            - "/cloud/aws-emr/bootstrap/install-hail.sh"
+            - "--"
             - "--input-path"
             - "s3://variant-spark/HailJupyter/hail/0.1_2.2.1"
             - "--hail-version"
             - "0.1"
             - "--spark-version"
             - "2.2.1"
-          Path: "s3://variant-spark/HailJupyter/install-hail.sh"
+          Path: "s3://variant-spark/s3-bootstrap.sh"
       Configurations:
       - Classification: emrfs-site
         ConfigurationProperties: