fix: Strip app metadata in templates sent to SAR (#20)

* Strip app metadata in templates sent to SAR
* Accept passing template as dictionary to publish_application and update_application_metadata
* Use OrderedDict for json to preserve elements order

Author: paoptu023

README.md

@@ -28,6 +28,7 @@ For example:
 
 ```python
 import boto3
+import yaml
 from serverlessrepo import publish_application
 
 sar_client = boto3.client('serverlessrepo', region_name='us-east-1')
@@ -36,7 +37,10 @@ with open('template.yaml', 'r') as f:
     template = f.read()
     # if sar_client is not provided, we will initiate the client using region inferred from aws configurations
     output = publish_application(template, sar_client)
-    print (output)
+
+    # Alternatively, pass parsed template as a dictionary
+    template_dict = yaml.loads(template)
+    output = publish_application(template_dict, sar_client)
 ```
 
 The output of `publish_application` has the following structure:
@@ -75,6 +79,7 @@ For example:
 
 ```python
 import boto3
+import yaml
 from serverlessrepo import update_application_metadata
 
 sar_client = boto3.client('serverlessrepo', region_name='us-east-1')
@@ -84,6 +89,10 @@ with open('template.yaml', 'r') as f:
     application_id = 'arn:aws:serverlessrepo:us-east-1:123456789012:applications/test-app'
     # if sar_client is not provided, we will initiate the client using region inferred from aws configurations
     update_application_metadata(template, application_id, sar_client)
+
+    # Alternatively, pass parsed template as a dictionary
+    template_dict = yaml.loads(template)
+    update_application_metadata(template_dict, application_id, sar_client)
 ```
 
 ### Manage Application Permissions

serverlessrepo/__version__.py

@@ -1,7 +1,7 @@
 """Serverlessrepo version and package meta-data."""
 
 __title__ = 'serverlessrepo'
-__version__ = '0.1.4'
+__version__ = '0.1.5'
 __license__ = 'Apache 2.0'
 __description__ = (
     'A Python library with convenience helpers for working '

serverlessrepo/parser.py

@@ -1,8 +1,10 @@
 """Helper to parse JSON/YAML SAM template and dump YAML files."""
 
 import re
+import copy
 import json
 from collections import OrderedDict
+
 import six
 import yaml
 from yaml.resolver import ScalarNode, SequenceNode
@@ -86,7 +88,7 @@ def parse_template(template_str):
         # PyYAML doesn't support json as well as it should, so if the input
         # is actually just json it is better to parse it with the standard
         # json parser.
-        return json.loads(template_str)
+        return json.loads(template_str, object_pairs_hook=OrderedDict)
     except ValueError:
         yaml.SafeLoader.add_constructor(yaml.resolver.BaseResolver.DEFAULT_MAPPING_TAG, _dict_constructor)
         yaml.SafeLoader.add_multi_constructor('!', intrinsics_multi_constructor)
@@ -103,8 +105,8 @@ def get_app_metadata(template_dict):
     :rtype: ApplicationMetadata
     :raises ApplicationMetadataNotFoundError
     """
-    if METADATA in template_dict and SERVERLESS_REPO_APPLICATION in template_dict[METADATA]:
-        app_metadata_dict = template_dict[METADATA][SERVERLESS_REPO_APPLICATION]
+    if SERVERLESS_REPO_APPLICATION in template_dict.get(METADATA, {}):
+        app_metadata_dict = template_dict.get(METADATA).get(SERVERLESS_REPO_APPLICATION)
         return ApplicationMetadata(app_metadata_dict)
 
     raise ApplicationMetadataNotFoundError(
@@ -122,3 +124,26 @@ def parse_application_id(text):
     """
     result = re.search(APPLICATION_ID_PATTERN, text)
     return result.group(0) if result else None
+
+
+def strip_app_metadata(template_dict):
+    """
+    Strip the "AWS::ServerlessRepo::Application" metadata section from template.
+
+    :param template_dict: SAM template as a dictionary
+    :type template_dict: dict
+    :return: stripped template content
+    :rtype: str
+    """
+    if SERVERLESS_REPO_APPLICATION not in template_dict.get(METADATA, {}):
+        return template_dict
+
+    template_dict_copy = copy.deepcopy(template_dict)
+
+    # strip the whole metadata section if SERVERLESS_REPO_APPLICATION is the only key in it
+    if not [k for k in template_dict_copy.get(METADATA) if k != SERVERLESS_REPO_APPLICATION]:
+        template_dict_copy.pop(METADATA, None)
+    else:
+        template_dict_copy.get(METADATA).pop(SERVERLESS_REPO_APPLICATION, None)
+
+    return template_dict_copy

serverlessrepo/publish.py

@@ -1,11 +1,16 @@
 """Module containing functions to publish or update application."""
 
 import re
+import copy
+
 import boto3
 from botocore.exceptions import ClientError
 
 from .application_metadata import ApplicationMetadata
-from .parser import parse_template, get_app_metadata, parse_application_id
+from .parser import (
+    yaml_dump, parse_template, get_app_metadata,
+    parse_application_id, strip_app_metadata
+)
 from .exceptions import S3PermissionsRequired
 
 CREATE_APPLICATION = 'CREATE_APPLICATION'
@@ -17,8 +22,8 @@ def publish_application(template, sar_client=None):
     """
     Create a new application or new application version in SAR.
 
-    :param template: A packaged YAML or JSON SAM template
-    :type template: str
+    :param template: Content of a packaged YAML or JSON SAM template
+    :type template: str_or_dict
     :param sar_client: The boto3 client used to access SAR
     :type sar_client: boto3.client
     :return: Dictionary containing application id, actions taken, and updated details
@@ -28,13 +33,15 @@ def publish_application(template, sar_client=None):
     if not template:
         raise ValueError('Require SAM template to publish the application')
 
-    template_dict = parse_template(template)
-    app_metadata = get_app_metadata(template_dict)
     if not sar_client:
         sar_client = boto3.client('serverlessrepo')
 
+    template_dict = _get_template_dict(template)
+    app_metadata = get_app_metadata(template_dict)
+    stripped_template_dict = strip_app_metadata(template_dict)
+    stripped_template = yaml_dump(stripped_template_dict)
     try:
-        request = _create_application_request(app_metadata, template)
+        request = _create_application_request(app_metadata, stripped_template)
         response = sar_client.create_application(**request)
         application_id = response['ApplicationId']
         actions = [CREATE_APPLICATION]
@@ -55,7 +62,7 @@ def publish_application(template, sar_client=None):
         # Create application version if semantic version is specified
         if app_metadata.semantic_version:
             try:
-                request = _create_application_version_request(app_metadata, application_id, template)
+                request = _create_application_version_request(app_metadata, application_id, stripped_template)
                 sar_client.create_application_version(**request)
                 actions.append(CREATE_APPLICATION_VERSION)
             except ClientError as e:
@@ -73,8 +80,8 @@ def update_application_metadata(template, application_id, sar_client=None):
     """
     Update the application metadata.
 
-    :param template: A packaged YAML or JSON SAM template
-    :type template: str
+    :param template: Content of a packaged YAML or JSON SAM template
+    :type template: str_or_dict
     :param application_id: The Amazon Resource Name (ARN) of the application
     :type application_id: str
     :param sar_client: The boto3 client used to access SAR
@@ -87,12 +94,31 @@ def update_application_metadata(template, application_id, sar_client=None):
     if not sar_client:
         sar_client = boto3.client('serverlessrepo')
 
-    template_dict = parse_template(template)
+    template_dict = _get_template_dict(template)
     app_metadata = get_app_metadata(template_dict)
     request = _update_application_request(app_metadata, application_id)
     sar_client.update_application(**request)
 
 
+def _get_template_dict(template):
+    """
+    Parse string template and or copy dictionary template.
+
+    :param template: Content of a packaged YAML or JSON SAM template
+    :type template: str_or_dict
+    :return: Template as a dictionary
+    :rtype: dict
+    :raises ValueError
+    """
+    if isinstance(template, str):
+        return parse_template(template)
+
+    if isinstance(template, dict):
+        return copy.deepcopy(template)
+
+    raise ValueError('Input template should be a string or dictionary')
+
+
 def _create_application_request(app_metadata, template):
     """
     Construct the request body to create application.

tests/unit/test_parser.py

@@ -1,4 +1,5 @@
 import re
+from collections import OrderedDict
 from unittest import TestCase
 
 from serverlessrepo.exceptions import ApplicationMetadataNotFoundError
@@ -79,6 +80,34 @@ def test_parse_json_with_tabs(self):
         output = parser.parse_template(template)
         self.assertEqual(output, {'foo': 'bar'})
 
+    def test_parse_json_preserve_elements_order(self):
+        input_template = """
+        {
+            "B_Resource": {
+                "Key2": {
+                    "Name": "name2"
+                },
+                "Key1": {
+                    "Name": "name1"
+                }
+            },
+            "A_Resource": {
+                "Key2": {
+                    "Name": "name2"
+                },
+                "Key1": {
+                    "Name": "name1"
+                }
+            }
+        }
+        """
+        expected_dict = OrderedDict([
+            ('B_Resource', OrderedDict([('Key2', {'Name': 'name2'}), ('Key1', {'Name': 'name1'})])),
+            ('A_Resource', OrderedDict([('Key2', {'Name': 'name2'}), ('Key1', {'Name': 'name1'})]))
+        ])
+        output_dict = parser.parse_template(input_template)
+        self.assertEqual(expected_dict, output_dict)
+
     def test_parse_yaml_preserve_elements_order(self):
         input_template = """
         B_Resource:
@@ -93,17 +122,12 @@ def test_parse_yaml_preserve_elements_order(self):
                 Name: name1
         """
         output_dict = parser.parse_template(input_template)
-        expected_dict = {
-            'B_Resource': {
-                'Key2': {'Name': 'name2'},
-                'Key1': {'Name': 'name1'}
-            },
-            'A_Resource': {
-                'Key2': {'Name': 'name2'},
-                'Key1': {'Name': 'name1'}
-            }
-        }
+        expected_dict = OrderedDict([
+            ('B_Resource', OrderedDict([('Key2', {'Name': 'name2'}), ('Key1', {'Name': 'name1'})])),
+            ('A_Resource', OrderedDict([('Key2', {'Name': 'name2'}), ('Key1', {'Name': 'name1'})]))
+        ])
         self.assertEqual(expected_dict, output_dict)
+
         output_template = parser.yaml_dump(output_dict)
         # yaml dump changes indentation, remove spaces and new line characters to just compare the text
         self.assertEqual(re.sub(r'\n|\s', '', input_template),
@@ -174,3 +198,36 @@ def test_parse_application_id_return_none(self):
         text_without_application_id = 'text without application id'
         result = parser.parse_application_id(text_without_application_id)
         self.assertIsNone(result)
+
+    def test_strip_app_metadata_when_input_does_not_contain_metadata(self):
+        template_dict = {'Resources': {}}
+        actual_output = parser.strip_app_metadata(template_dict)
+        self.assertEqual(actual_output, template_dict)
+
+    def test_strip_app_metadata_when_metadata_only_contains_app_metadata(self):
+        template_dict = {
+            'Metadata': {
+                'AWS::ServerlessRepo::Application': {}
+            },
+            'Resources': {},
+        }
+        expected_output = {'Resources': {}}
+        actual_output = parser.strip_app_metadata(template_dict)
+        self.assertEqual(actual_output, expected_output)
+
+    def test_strip_app_metadata_when_metadata_contains_additional_keys(self):
+        template_dict = {
+            'Metadata': {
+                'AWS::ServerlessRepo::Application': {},
+                'AnotherKey': {}
+            },
+            'Resources': {}
+        }
+        expected_output = {
+            'Metadata': {
+                'AnotherKey': {}
+            },
+            'Resources': {}
+        }
+        actual_output = parser.strip_app_metadata(template_dict)
+        self.assertEqual(actual_output, expected_output)