chore: merge develop to master (aws#488)

Author: brettstack

DESIGN.md

@@ -1,3 +1,11 @@
+# Overview
+
+SAM is called by the CloudFormation Service. CloudFormation recognises the `Transform: AWS::Serverless-2016-10-31` header and invokes the SAM translator. This will then take your SAM template and expand it
+into a full fledged CloudFormation Template. The CloudFormation Template that is produced from SAM is the template that is executed by CloudFormation to create/update/delete AWS resources.
+
+The entry point for SAM starts in the Translator class [here](https://github.com/awslabs/serverless-application-model/blob/develop/samtranslator/translator/translator.py#L29), where SAM iterates through the
+template and acts on `AWS::Serverless::*` Type Resources.
+
 # Design decisions
 
 Document design decisions here.

DEVELOPMENT_GUIDE.rst

@@ -71,4 +71,14 @@ Tests are also a documentation of the success and failure cases, which is crucia
 .. _excellent cheatsheet: http://python-future.org/compatible_idioms.html
 .. _pyenv: https://github.com/pyenv/pyenv
 .. _tox: http://tox.readthedocs.io/en/latest/
-.. _installation instructions: https://github.com/pyenv/pyenv#installation
+.. _installation instructions: https://github.com/pyenv/pyenv#installation
+
+Profiling
+---------
+
+Install snakeviz `pip install snakeviz`
+
+```
+python -m cProfile -o sam_profile_results bin/sam-translate.py translate --input-file=tests/translator/input/alexa_skill.yaml --output-file=cfn-template.json
+snakeviz sam_profile_results
+```

bin/sam-translate.py

@@ -0,0 +1,57 @@
+#!/usr/bin/env python2
+
+"""Convert SAM templates to CloudFormation templates.
+
+Known limitations: cannot transform CodeUri pointing at local directory.
+
+Usage:
+  sam-translate.py --input-file=sam-template.yaml [--output-file=<o>]
+
+Options:
+  --input-file=<i>     Location of SAM template to transform.
+  --output-file=<o>    Location to store resulting CloudFormation template [default: cfn-template.json].
+
+"""
+import json
+import os
+
+import boto3
+from docopt import docopt
+
+from samtranslator.public.translator import ManagedPolicyLoader
+from samtranslator.translator.transform import transform
+from samtranslator.yaml_helper import yaml_parse
+
+cli_options = docopt(__doc__)
+iam_client = boto3.client('iam')
+cwd = os.getcwd()
+
+
+def get_input_output_file_paths():
+    input_file_option = cli_options.get('--input-file')
+    output_file_option = cli_options.get('--output-file')
+    input_file_path = os.path.join(cwd, input_file_option)
+    output_file_path = os.path.join(cwd, output_file_option)
+
+    return input_file_path, output_file_path
+
+
+def main():
+    input_file_path, output_file_path = get_input_output_file_paths()
+
+    with open(input_file_path, 'r') as f:
+        sam_template = yaml_parse(f)
+
+    cloud_formation_template = transform(
+        sam_template, {}, ManagedPolicyLoader(iam_client))
+    cloud_formation_template_prettified = json.dumps(
+        cloud_formation_template, indent=2)
+
+    with open(output_file_path, 'w') as f:
+        f.write(cloud_formation_template_prettified)
+
+    print('Wrote transformed CloudFormation template to: ' + output_file_path)
+
+
+if __name__ == '__main__':
+    main()

docs/cloudformation_compatibility.rst

@@ -91,6 +91,7 @@ Kinesis
      Property Name        Intrinsic(s) Supported            Reasons
 ======================== ================================== ========================
 Stream                   All
+Queue                    All
 StartingPosition         All
 BatchSize                All
 ======================== ================================== ========================
@@ -103,6 +104,7 @@ DynamoDB
 Stream                   All
 StartingPosition         All
 BatchSize                All
+SSESpecification         All
 ======================== ================================== ========================
 
 Api

docs/globals.rst

@@ -80,6 +80,9 @@ presently.
       BinaryMediaTypes:
       Cors:
 
+    SimpleTable:
+      SSESpecification
+
 Implicit APIs
 ~~~~~~~~~~~~~
 

docs/internals/generated_resources.rst

@@ -220,6 +220,33 @@ AWS::Lambda::Permissions           MyFunction\ **MyTrigger**\ Permission
 AWS::Lambda::EventSourceMapping    MyFunction\ **MyTrigger** 
 ================================== ================================
 
+SQS
+^^^^^^^
+
+Example:
+
+.. code:: yaml
+
+  MyFunction:
+    Type: AWS::Serverless::Function
+    Properties:
+      ...
+      Events:
+        MyTrigger:
+          Type: SQS
+          Properties:
+            Queue: arn:aws:sqs:us-east-1:123456789012:my-queue
+      ...
+
+Additional generated resources:
+
+================================== ================================
+CloudFormation Resource Type       Logical ID 
+================================== ================================
+AWS::Lambda::Permissions           MyFunction\ **MyTrigger**\ Permission
+AWS::Lambda::EventSourceMapping    MyFunction\ **MyTrigger** 
+================================== ================================
+
 DynamoDb
 ^^^^^^^^
 

docs/policy_templates_data/policy_templates.json

@@ -1132,6 +1132,113 @@
           "Resource": "*"
         }]
       }
+    },
+    "MobileAnalyticsWriteOnlyAccessPolicy": {
+      "Description": "Gives write only permissions to put event data for all application resources",
+      "Parameters": {},
+      "Definition": {
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "mobileanalytics:PutEvents"
+            ],
+            "Resource": "*"
+          }
+        ]
+      }
+    },
+    "PinpointEndpointAccessPolicy": {
+      "Description": "Gives permissions to get and update endpoints for a Pinpoint application",
+      "Parameters": {
+        "PinpointApplicationId": {
+          "Description": "The id of your Pinpoint application"
+        }
+      },
+      "Definition": {
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "mobiletargeting:GetEndpoint",
+              "mobiletargeting:UpdateEndpoint",
+              "mobiletargeting:UpdateEndpointsBatch"
+            ],
+            "Resource": {
+              "Fn::Sub": [
+                "arn:${AWS::Partition}:mobiletargeting:${AWS::Region}:${AWS::AccountId}:apps/${pinpointApplicationId}/endpoints/*",
+                {
+                  "pinpointApplicationId": {
+                    "Ref": "PinpointApplicationId"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      }
+    },
+    "FirehoseWritePolicy": {
+      "Description": "Gives permission to write to a Kinesis Firehose Delivery Stream",
+      "Parameters": {
+        "DeliveryStreamName": {
+          "Description": "Name of Kinesis Firehose Delivery Stream"
+        }
+      },
+      "Definition": {
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "firehose:PutRecord",
+              "firehose:PutRecordBatch"
+            ],
+            "Resource": {
+              "Fn::Sub": [
+                "arn:${AWS::Partition}:firehose:${AWS::Region}:${AWS::AccountId}:deliverystream/${deliveryStreamName}",
+                {
+                  "deliveryStreamName": {
+                    "Ref": "DeliveryStreamName"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      }
+    },
+    "FirehoseCrudPolicy": {
+      "Description": "Gives permission to create, write to, update, and delete a Kinesis Firehose Delivery Stream",
+      "Parameters": {
+        "DeliveryStreamName": {
+          "Description": "Name of Kinesis Firehose Delivery Stream"
+        }
+      },
+      "Definition": {
+        "Statement": [
+          {
+            "Effect": "Allow",
+            "Action": [
+              "firehose:CreateDeliveryStream",
+              "firehose:DeleteDeliveryStream",
+              "firehose:DescribeDeliveryStream",
+              "firehose:PutRecord",
+              "firehose:PutRecordBatch",
+              "firehose:UpdateDestination"
+            ],
+            "Resource": {
+              "Fn::Sub": [
+                "arn:${AWS::Partition}:firehose:${AWS::Region}:${AWS::AccountId}:deliverystream/${deliveryStreamName}",
+                {
+                  "deliveryStreamName": {
+                    "Ref": "DeliveryStreamName"
+                  }
+                }
+              ]
+            }
+          }
+        ]
+      }
     }
   }
 }

docs/safe_lambda_deployments.rst

@@ -3,14 +3,14 @@ Safe Lambda deployments
 
 .. contents::
 
-Pushing to production can be nerve-racking even if you have 100% unit test coverage and state-of-art full CD system. 
+Pushing to production can be nerve-wracking even if you have 100% unit test coverage and a state-of-art full CD system. 
 It is a good practice to expose your new code to a small percentage of production traffic, run tests, watch for alarms 
 and dial up traffic as you gain more confidence. The goal is to minimize production impact as much as possible. 
 
 To enable traffic shifting deployments for Lambda functions, we will use Lambda Aliases, which can balance incoming 
 traffic between two different versions of your function, based on preassigned weights. Before deployment, 
 the alias sends 100% of invokes to the version used in production. During deployment, we will upload the code to Lambda,
-publish a new version, send a small percentage of traffic to new version, monitor, and validate before shifting 
+publish a new version, send a small percentage of traffic to the new version, monitor, and validate before shifting 
 100% of traffic to the new version. You can do this manually by calling Lambda APIs or let AWS CodeDeploy automate 
 it for you. CodeDeploy will shift traffic, monitor alarms, run validation logic and even trigger an automatic rollback 
 if something goes wrong.
@@ -32,24 +32,24 @@ Instant traffic shifting using Lambda Aliases
 ---------------------------------------------
 
 Every Lambda function can have any number of Versions and Aliases
-associated with them. Versions are immutable snapshot of function
+associated with them. Versions are immutable snapshots of a function
 including code & configuration. If you are familiar with git, they are
-similar to commits. It is a good practice in general to publish a new
+similar to commits. In general, it is a good practice to publish a new
 version every time you update your function code. When you invoke a
-specific version (using function name + version number combination) you
-are guaranteed to get the same code & configuration irrespective of
+specific version (using the function name + version number combination) you
+are guaranteed to get the same code & configuration irrespective of the
 state of the function. This protects you against accidentally updating
 production code.
 
 To effectively use the versions, you should create an Alias which is
 literally a pointer to a version. Aliases have a name and an ARN similar
-to the function and accepted by the Invoke APIs. If you invoke an Alias,
+to the function and are accepted by the Invoke APIs. If you invoke an Alias,
 Lambda will in turn invoke the version that the Alias is pointing to.
 
 In production, you will first update your function code, publish a new
-version, invoke the version directly to run tests against it, and after
-you are satisfied flip the Alias to point to the new version. Traffic
-will instantly shift from using your old version to the new version.
+version, invoke the version directly to run tests against it, and, after
+you are satisfied, flip the Alias to point to the new version. Traffic
+will instantly shift from using your old version to using the new version.
 
 SAM provides a simple primitive to do this for you. Add the following
 property to your ``AWS::Serverless::Function`` resource:
@@ -61,10 +61,10 @@ property to your ``AWS::Serverless::Function`` resource:
 This will:
 
 - Create an Alias with ``<alias-name>`` 
-- Creates & publishes a Lambda version with the latest code & configuration 
-  derived from ``CodeUri`` property 
+- Create & publish a Lambda version with the latest code & configuration 
+  derived from the ``CodeUri`` property 
 - Point the Alias to the latest published version 
-- Point all event sources to the Alias & not the function 
+- Point all event sources to the Alias & not to the function 
 - When the ``CodeUri`` property of ``AWS::Serverless::Function`` changes, 
   SAM will automatically publish a new version & point the alias to the 
   new version
@@ -79,8 +79,8 @@ In other words, your traffic will shift "instantly" to your new code.
 Traffic shifting using CodeDeploy
 ----------------------------------
 
-For production deployments, you want a more controlled traffic shifting
-from old version to new version while monitoring alarms and triggering a
+For production deployments, you may want more controlled traffic shifting
+from an old version to a new version which monitors alarms and triggers a
 rollback if necessary. CodeDeploy is an AWS service which can do this
 for you. It uses Lambda Alias' ability to route a percentage of traffic
 to two different Lambda Versions. To use this feature, set the
@@ -174,14 +174,14 @@ CloudFormation, the following happens:
 - Before traffic shifting starts, CodeDeploy will invoke the **PreTraffic Hook** Lambda function. This Lambda function must call back to CodeDeploy with an explicit status of Success or Failure, via the PutLifecycleEventHookExecutionStatus_ API. On Failure, CodeDeploy will abort and report a failure back to CloudFormation. On Success, CodeDeploy will proceed with the specified traffic shifting. Here_ is a sample Lambda Hook function.
 - ``Type: Linear10PercentEvery10Minutes`` instructs CodeDeploy to start with 10% traffic on new version and add 10% every 10 minutes. It will complete traffic shifting in 100 minutes.
 - During traffic shifting, if any of the CloudWatch Alarms go to *Alarm* state, CodeDeploy will immediately flip the Alias back to old version and report a failure to CloudFormation.
-- After traffic shifting completes, CodeDeploy will invoke the **PostTraffic Hook** Lambda function. This is similar to PreTraffic Hook where the function must callback to CodeDeploy to report a Success or Failure. PostTraffic hook is a great place to run integration tests or other validation actions.
+- After traffic shifting completes, CodeDeploy will invoke the **PostTraffic Hook** Lambda function. This is similar to PreTraffic Hook where the function must callback to CodeDeploy to report a Success or a Failure. PostTraffic hook is a great place to run integration tests or other validation actions.
 - If everything went well, the Alias will be pointing to the new Lambda Version.
 
 NOTE: Verify that your AWS SDK version supports PutLifecycleEventHookExecutionStatus. For example, Python requires SDK version 1.4.8 or newer.
 
 .. _PutLifecycleEventHookExecutionStatus: https://docs.aws.amazon.com/codedeploy/latest/APIReference/API_PutLifecycleEventHookExecutionStatus.html
 
-.. _Here: https://github.com/awslabs/serverless-application-model/blob/master/examples/2016-10-31/lambda_safe_deployments/preTrafficHook.js
+.. _Here: https://github.com/awslabs/serverless-application-model/blob/master/examples/2016-10-31/lambda_safe_deployments/src/preTrafficHook.js
 
 Traffic Shifting Configurations
 ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~
@@ -207,8 +207,8 @@ They work as follows:
 
   Ex: ``Linear10PercentEvery10Minutes`` will add 10 percentage of traffic every 10 minute to complete in 100 minutes.
 
-- **CanaryXPercentYMinutes**: X percent of traffic will be routed to new Version once, and wait for Y minutes in this
-  state before sending 100 percent of traffic to new version. Some people call this as Blue/Green deployment. 
+- **CanaryXPercentYMinutes**: X percent of traffic will be routed to new version for Y minutes. After Y minutes,
+  100 percent of traffic will be sent to new version. Some people call this as Blue/Green deployment. 
 
   Ex: ``Canary10Percent15Minutes`` will send 10 percent traffic to new version and 15 minutes later complete deployment
   by sending all traffic to new version.
@@ -254,8 +254,8 @@ Hooks are extremely powerful because:
   Function). So you can customize the hooks logic to the function that is being deployed.
 
     NOTE: If the Hook functions are created by the same SAM template that is deployed, then make sure to turn off
-    traffic shifting deployments for the hook functions. Also, the Role SAM generates for a Lambda Execution Role does not include all permissions needed for Per and Post hook functions, since it
-    will not contain the necessary permissions to call the CodeDepoloy APIs or Invoke your new Lambda function for testing.
+    traffic shifting deployments for the hook functions. Also, the Role SAM generates for a Lambda Execution Role does not include all permissions needed for Pre and Post hook functions, since it
+    will not contain the necessary permissions to call the CodeDeploy APIs or Invoke your new Lambda function for testing.
     Instead, use the Policies_ attribute to provide the CodeDeploy and Lambda permissions needed. The example also shows a Policy that provides access to the CodeDeploy resource that SAM automatically generates.
     Finally, use the ``FunctionName`` property to control the exact name of the Lambda function CloudFormation creates. Otherwise, CloudFormation will create your Lambda function with the Stack name and a unique ID added as part of the name.
 

examples/2016-10-31/api_backend/src/index.js

@@ -6,13 +6,7 @@ const dynamo = new AWS.DynamoDB.DocumentClient();
 
 const tableName = process.env.TABLE_NAME;
 
-const createResponse = (statusCode, body) => {
-    
-    return {
-        statusCode: statusCode,
-        body: body
-    }
-};
+const createResponse = (statusCode, body) => ({ statusCode, body });
 
 exports.get = (event, context, callback) => {
 

examples/2016-10-31/policy_templates/all_policy_templates.yaml

@@ -49,6 +49,8 @@ Resources:
         - RekognitionWriteOnlyAccessPolicy:
             CollectionId: id
 
+        - RekognitionLabelsPolicy: {}
+
         - SQSSendMessagePolicy:
             QueueName: name