add ci

Author: kapilt

.github/workflows/ci.yml

@@ -0,0 +1,64 @@
+name: "CI"
+
+on:
+  push:
+    branches:
+      - main
+  pull_request:
+    branches:
+      - main
+
+jobs:
+  Lint:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v2
+      - uses: actions/setup-python@v1
+        with:
+          python-version: 3.8
+      - name: Linting
+        run: |
+          pip install pre-commit
+          pre-commit run --all-files
+
+  Tests:
+    needs: Lint
+    runs-on: ubuntu-latest
+    strategy:
+      matrix:
+        python-version: [3.8, 3.9]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Install poetry
+        shell: bash
+        run: |
+          pip install poetry
+
+      - name: Configure poetry
+        shell: bash
+        run: poetry config virtualenvs.in-project true
+
+      - name: Set up cache
+        uses: actions/cache@v1
+        id: cache
+        with:
+          path: .venv
+          key: venv-${{ runner.os }}-${{ steps.full-python-version.outputs.version }}-${{ hashFiles('**/poetry.lock') }}
+
+      - name: Ensure cache is healthy
+        if: steps.cache.outputs.cache-hit == 'true'
+        shell: bash
+        run: poetry run pip --version >/dev/null 2>&1 || rm -rf .venv
+
+      - name: Install dependencies
+        shell: bash
+        run: poetry install
+
+      - name: Pytest
+        shell: bash
+        run: poetry run pytest -v tests

.pre-commit-config.yaml

@@ -0,0 +1,25 @@
+exclude: ".(json|md)"
+repos:
+  - repo: https://github.com/psf/black
+    rev: 19.10b0
+    hooks:
+      - id: black
+
+  - repo: https://gitlab.com/pycqa/flake8
+    rev: 3.7.8
+    hooks:
+      - id: flake8
+
+  - repo: https://github.com/timothycrosley/isort
+    rev: 4.3.21-2
+    hooks:
+      - id: isort
+
+  - repo: https://github.com/pre-commit/pre-commit-hooks
+    rev: v2.3.0
+    hooks:
+      - id: trailing-whitespace
+        exclude: ^tests/.*/fixtures/.*
+      - id: end-of-file-fixer
+        exclude: ^tests/.*/fixtures/.*
+      - id: debug-statements

readme.md

@@ -1,23 +1,21 @@
 # tfdevops
 
-Terraform support for AWS DevOps Guru.
+Terraform support for AWS DevOps Guru. The service natively only supports Cloudformation stacks.
 https://aws.amazon.com/devops-guru/features/
 
+This project provides support for terraform users by automatically
+converting terraform state to an imported cloudformation stack
+and optionally enabling it with devops guru.
 
-Unfortunately the DevOps Guru service team only supports app level stack enablement via cloudformation, meaning its useless
-to the majority of AWS users.
-
-This project provides support for terraform centric organizations to use it by automatically
-converting terraform state to an imported cloudformation stack and optionally enabling it with devops guru.
-
-Note it only supports roughly 25 resources per the pricing page.
+Note AWS DevOps Guru only supports roughly 25 resources.
 https://aws.amazon.com/devops-guru/pricing/
 
 
-tfdevops also corrects a major usability issue of cloudformation, by providing client side schema validation
-of templates.
+## How it works
 
-Enjoy.
+- Translates terraform state into a cloudformation template with a retain deletion policy
+- Creates a cloudformation stack with imported resources
+- Enrolls the stack into AWS DevOps Guru
 
 ## Usage
 
@@ -69,3 +67,17 @@ path.
 1. Is this a generic terraform to cloudformation converter?
 
 No, while it has some facilities that resemble that, its very targeted at simply producing enough cfn to make devops guru work.
+
+## Supported resources
+
+
+At the moment tfdevops supports the following resources
+
+ - AWS::StepFunctions::StateMachine
+ - AWS::ECS::Service
+ - AWS::SQS::Queue
+ - AWS::SNS::Topic
+ - AWS::RDS::DBInstance
+ - AWS::Lambda::Function
+ - AWS::Events::Rule
+ - AWS::DynamoDB::Table

setup.cfg

@@ -0,0 +1,3 @@
+[flake8]
+max-line-length = 100
+

tests/test_tfdevops.py

@@ -0,0 +1,5 @@
+from tfdevops.cli import Translator, TF_CFN_MAP
+
+
+def test_translator_map():
+    assert set(Translator.get_translator_map()) == set(TF_CFN_MAP)

tfdevops/cli.py

@@ -1,23 +1,26 @@
-import boto3
-from botocore.exceptions import ClientError, WaiterError
-from botocore.waiter import WaiterModel, create_waiter_with_client
-import click
+# Copyright Stacklet, Inc.
+# SPDX-License-Identifier: Apache-2.0
+#
 import json
-import jsonschema
-import jmespath
-import hcl2
 import logging
-from pathlib import Path
 import subprocess
+from pathlib import Path
 from urllib import parse
 
+import boto3
+import click
+import jmespath
+import jsonschema
+from botocore.exceptions import ClientError, WaiterError
+from botocore.waiter import WaiterModel, create_waiter_with_client
 
 __author__ = "Kapil Thangavelu <https://twitter.com/kapilvt>"
 
 log = logging.getLogger("tfdevops")
 
 DEFAULT_STACK_NAME = "GuruStack"
 DEFAULT_CHANGESET_NAME = "GuruImport"
+DEFAULT_S3_ENCRYPT = "AES256"
 
 # manually construct waiter models for change sets since the service
 # team didn't bother to publish one in their smithy models, perhaps
@@ -70,9 +73,10 @@
 
 
 @click.group()
-def cli():
+@click.option("-v", "--verbose", is_flag=True)
+def cli(verbose):
     """Terraform to Cloudformation and AWS DevOps Guru"""
-    logging.basicConfig(level=logging.INFO)
+    logging.basicConfig(level=verbose and logging.DEBUG or logging.INFO)
 
 
 @cli.command()
@@ -102,10 +106,14 @@ def deploy(template, resources, stack_name, guru, template_url, change_name):
         stack_info = client.describe_stacks(StackName=stack_name)["Stacks"][0]
         log.info("Found existing stack, state:%s", stack_info["StackStatus"])
     except ClientError:
-        # somewhat bonkers the service team hasn't put a proper customized exception in place for a common error issue.
-        # ala they have one for client.exceptions.StackNotFoundException but didn't bother
+        # somewhat annoying the service team hasn't put a proper customized
+        # exception in place for a common error issue. ala they have one for
+        # client.exceptions.StackNotFoundException but didn't bother
         # to actually use it for this, or its histerical raison compatibility.
-        # botocore.exceptions.ClientError: An error occurred (ValidationError) when calling the DescribeStacks operation: Stack with id GuruStack does not exist
+        # This unfortunately means we have to catch a very generic client error.
+        # ie. we're trying to catch errors like this.
+        #  botocore.exceptions.ClientError: An error occurred (ValidationError) when
+        #  calling the DescribeStacks operation: Stack with id GuruStack does not exist
         stack_info = None
 
     # so for each stack and each resource we have to deal with the complexity
@@ -123,7 +131,7 @@ def deploy(template, resources, stack_name, guru, template_url, change_name):
     # Its gets worse when you consider the compatibility complexity matrix
     # on the various versions and bugs, like the lack of a proper error code
     # for stack not found above.
-    # Nonetheless, we perserve and try to present a humane interface.
+    # Nonetheless, we persevere and try to present a humane interface.
     #
     # Stack State Enumeration:
     # CREATE_COMPLETE
@@ -184,7 +192,7 @@ def deploy(template, resources, stack_name, guru, template_url, change_name):
         cinfo = client.describe_change_set(
             StackName=stack_name, ChangeSetName=change_name
         )
-    except client.exceptions.ChangeSetNotFoundException:
+    except (client.exceptions.ChangeSetNotFoundException, ClientError):
         cinfo = None
 
     if cinfo and cinfo["Status"] == "FAILED":
@@ -368,8 +376,9 @@ def validate(template):
 def cfn(module, template, resources, types, s3_path):
     """Export a cloudformation template and importable resources
 
-    s3 path only needs to be specified when handling resources with verbose definitions (step functions)
-    or a large cardinality of resources which would overflow cloudformation's api limits on templates (50k).
+    s3 path only needs to be specified when handling resources with verbose
+    definitions (step functions) or a large cardinality of resources which would
+    overflow cloudformation's api limits on templates (50k).
     """
     state = get_state_resources(module)
     type_map = get_type_mapping()
@@ -421,9 +430,9 @@ def cfn(module, template, resources, types, s3_path):
                 )
 
     # overflow to s3 for actual deployment on large templates
-    serialized_template = json.dumps(ctemplate).encode('utf8')
+    serialized_template = json.dumps(ctemplate).encode("utf8")
 
-    if s3_path: # and len(serialized_template) > 49000:
+    if s3_path:  # and len(serialized_template) > 49000:
         s3_url = format_template_url(
             s3_client,
             format_s3_path(
@@ -434,7 +443,9 @@ def cfn(module, template, resources, types, s3_path):
         )
         log.info("wrote s3 template url: %s", s3_url)
     elif len(serialized_template) > 49000:
-        log.warning("template too large for local deploy, pass --s3-path to deploy from s3")
+        log.warning(
+            "template too large for local deploy, pass --s3-path to deploy from s3"
+        )
 
     template.write(json.dumps(ctemplate, indent=2))
 
@@ -480,8 +491,9 @@ def write_s3_key(client, s3_path, key, content):
     result = client.put_object(
         Bucket=kinfo["Bucket"],
         Key=kinfo["Key"],
+        # this is the default but i've seen some orgs try to force this via request policy checks
         ACL="private",
-        ServerSideEncryption="AES256",
+        ServerSideEncryption=DEFAULT_S3_ENCRYPT,
         Body=content,
     )
     if result.get("VersionId"):
@@ -499,7 +511,7 @@ def format_s3_path(kinfo):
 def format_template_url(client, s3_path):
     parsed = parse.urlparse(s3_path)
     bucket = parsed.netloc
-    key = parsed.path.strip('/')
+    key = parsed.path.strip("/")
     version_id = None
     if parsed.query:
         query = parse.parse_qs(parsed.query)
@@ -514,6 +526,19 @@ def format_template_url(client, s3_path):
     return url.format(bucket=bucket, key=key, version_id=version_id, region=region)
 
 
+TF_CFN_MAP = {
+    "cloudwatch_event_rule": "AWS::Events::Rule",
+    "db_instance": "AWS::RDS::DBInstance",
+    "sns_topic": "AWS::SNS::Topic",
+    "sqs_queue": "AWS::SQS::Queue",
+    "lambda_function": "AWS::Lambda::Function",
+    "sfn_state_machine": "AWS::StepFunctions::StateMachine",
+    "cloudwatch_event_rule": "AWS::Events::Rule",
+    "ecs_service": "AWS::ECS::Service",
+    "dynamodb_table": "AWS::DynamoDB::Table",
+}
+
+
 class Translator:
 
     id = None
@@ -819,18 +844,4 @@ def get_properties(self, tf):
 
 
 if __name__ == "__main__":
-    try:
-        cli()
-    except WaiterError as e:
-        log.warning(
-            "failed waiting for async operation error\n reason: %s\n response: %s"
-            % (e, e.last_response)
-        )
-        raise
-    except SystemExit:
-        raise
-    except Exception:
-        import traceback, pdb, sys
-
-        traceback.print_exc()
-        pdb.post_mortem(sys.exc_info()[-1])
+    cli()