From fa7157e64ccf18cb7442efcf1602e236644dd239 Mon Sep 17 00:00:00 2001 From: Michal Chomo Date: Fri, 26 Jul 2019 11:41:41 +0200 Subject: [PATCH 1/6] fix S3 bucket detection and refactor --- .gitignore | 2 + AWSscripts/SQS3script.py | 406 ++++++++++++--------------------------- 2 files changed, 122 insertions(+), 286 deletions(-) diff --git a/.gitignore b/.gitignore index 67c3b2b..16ca81e 100644 --- a/.gitignore +++ b/.gitignore @@ -37,3 +37,5 @@ nosetests.xml loggly_setup.log env_details.txt + +environment diff --git a/AWSscripts/SQS3script.py b/AWSscripts/SQS3script.py index 4f1fdff..d951557 100644 --- a/AWSscripts/SQS3script.py +++ b/AWSscripts/SQS3script.py @@ -7,8 +7,6 @@ # region examples: us-east-1, us-west-2 etc. -import boto -import boto.sqs import boto.sqs.connection import boto3 import json @@ -22,6 +20,89 @@ from optparse import OptionParser +SQS_QUEUE_POLICY_TEMPLATE = """ { + "Version": "2008-10-17", + "Id": "PolicyExample", + "Statement": [ + { + "Sid": "example-statement-ID", + "Effect": "Allow", + "Principal": { + "AWS": "*" + }, + "Action": "SQS:SendMessage", + "Resource": "arn:aws:sqs:%(region)s:%(acnumber)s:%(queue_name)s", + "Condition": { + "ArnLike": { + "aws:SourceArn": %(source_arn)s + } + } + }, + { + "Sid": "GiveAccessToLoggly", + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::%(acnumber)s:root" + }, + "Action": "SQS:*", + "Resource": "arn:aws:sqs:%(region)s:%(acnumber)s:%(queue_name)s" + } + ] + } + """ + + +def put_bucket_notification_config(client, region, s3bucket, acnumber, sqs_queue_name): + client.put_bucket_notification_configuration( + Bucket=s3bucket, + NotificationConfiguration={ + "QueueConfigurations": [{ + "Id": "Notification", + "Events": ["s3:ObjectCreated:*"], + "QueueArn": "arn:aws:sqs:" + region + ":" + acnumber + ":" + sqs_queue_name + }], + } + ) + + +def get_source_arn_with_appended_bucket(queue_attr, s3bucket): + start = '\"aws:SourceArn\":' + end = '}}}' + result = re.search('%s(.*)%s' % (start, end), queue_attr).group(1) + + leftbracketremoved = result.replace('[', '') + rightbracketremoved = leftbracketremoved.replace(']', '') + + return '[' + rightbracketremoved + ',' + '\"arn:aws:s3:*:*:' + s3bucket + '\"' + ']' + + +def set_sqs_queue_policy(conn, region, acnumber, queue_name, source_arn): + queue_policy = SQS_QUEUE_POLICY_TEMPLATE % ( + {"region": region, "acnumber": acnumber, "queue_name": queue_name, "source_arn": source_arn}) + + parsed = json.loads(queue_policy) + + conn.set_queue_attribute(queue_name, 'Policy', json.dumps(parsed)) + + +def add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3bucket): + queue_attr_raw = conn.get_queue_attributes(queue_name, attribute='All') + queue_attr = str(queue_attr_raw) + + if 'arn:aws:s3' in queue_attr: + if "arn:aws:s3:*:*:" + s3bucket in queue_attr: + print 'Given bucket already exists in queue\'s policy' + else: + # append the bucket to the existing policy + print "A bucket already exists in this queue's policy, appending this bucket to it" + + source_arn = get_source_arn_with_appended_bucket(queue_attr, s3bucket) + set_sqs_queue_policy(conn, region, acnumber, queue_name, source_arn) + put_bucket_notification_config(client, region, s3bucket, acnumber, queue_name) + else: + set_sqs_queue_policy(conn, region, acnumber, queue_name, "arn:aws:s3:*:*:" + s3bucket) + put_bucket_notification_config(client, region, s3bucket, acnumber, queue_name) + parser = OptionParser() parser.add_option("--acnumber", dest="acnumber", @@ -37,36 +118,34 @@ (opts, args) = parser.parse_args() - s3bucket = opts.s3bucket acnumber = opts.acnumber sqsname = opts.sqsname user = opts.user admin = opts.admin -if acnumber.isdigit()==False: +if acnumber.isdigit() == False: print "Please check your account number, it should only contain digits, no other characters." sys.exit() conn = boto.connect_s3() -access_key='' -secret_key='' -bucket='' +access_key = '' +secret_key = '' +bucket = '' credentials_name = admin if admin else "default" home = os.path.expanduser("~") -with open(home+ '/.aws/credentials') as f: +with open(home + '/.aws/credentials') as f: for line in f: if credentials_name in line: for line in f: if "aws_access_key_id" in line: - access_key = line.split("=",1)[1].strip() + access_key = line.split("=", 1)[1].strip() if "aws_secret_access_key" in line: - secret_key = line.split("=",1)[1].strip() - break - + secret_key = line.split("=", 1)[1].strip() + break if not access_key: print "Please check your ~/.aws/credentials file and make sure that access key and secret access key are set. Run aws configure to set them up" @@ -85,12 +164,10 @@ print e sys.exit() - - region = bucket.get_location() if region == '': - region = 'us-east-1' + region = 'us-east-1' if not s3bucket: parser.error("S3 bucket name not provided") @@ -98,134 +175,13 @@ if not acnumber: parser.error("Account number not provided") - -conn = boto.sqs.connect_to_region(region, aws_access_key_id=access_key, aws_secret_access_key=secret_key) +conn = boto.sqs.connect_to_region(region, aws_access_key_id=access_key, aws_secret_access_key=secret_key) client = boto3.client('s3', region) queue_name = conn.get_queue(sqsname) - if "None" not in str(queue_name): - #queue exists - - queue_attr_raw = conn.get_queue_attributes(queue_name, attribute='All') - queue_attr = str(queue_attr_raw) - - if s3bucket in queue_attr: - print 'Bucket already exists in queue\'s policy' - - elif 'arn:aws:s3' in queue_attr: - # append the bucket to the existing policy - print "A bucket already exists in this queue's policy, appending this bucket to it" - - start = '\"aws:SourceArn\":' - end = '}}}' - result = re.search('%s(.*)%s' % (start, end), queue_attr).group(1) - - - leftbracketremoved = result.replace('[','') - rightbracketremoved = leftbracketremoved.replace(']','') - - addon = '[' + rightbracketremoved + ',' + '\"arn:aws:s3:*:*:' + s3bucket +'\"' +']' - - - text = """ { - "Version": "2008-10-17", - "Id": "PolicyExample", - "Statement": [ - { - "Sid": "example-statement-ID", - "Effect": "Allow", - "Principal": { - "AWS": "*" - }, - "Action": "SQS:SendMessage", - "Resource": "arn:aws:sqs:%s:%s:%s", - "Condition": { - "ArnLike": { - "aws:SourceArn": %s - } - } - }, - { - "Sid": "GiveAccessToLoggly", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::%s:root" - }, - "Action": "SQS:*", - "Resource": "arn:aws:sqs:%s:%s:%s" - } - ] - } - """ % (region, acnumber, queue_name, addon, acnumber, region, acnumber, queue_name) - - - parsed = json.loads(text) - - conn.set_queue_attribute(queue_name, 'Policy', json.dumps(parsed)) - - # s3 bucket notification configuration - client = boto3.client('s3', region) - - - response = client.put_bucket_notification_configuration( - Bucket=s3bucket, - NotificationConfiguration={ - "QueueConfigurations": [{ - "Id": "Notification", - "Events": ["s3:ObjectCreated:*"], - "QueueArn": "arn:aws:sqs:" + region + ":" + acnumber + ":" + queue_name - }], - } - ) - - - else: - conn.set_queue_attribute(queue_name, 'Policy', json.dumps({ - "Version": "2008-10-17", - "Id": "PolicyExample", - "Statement": [ - { - "Sid": "example-statement-ID", - "Effect": "Allow", - "Principal": { - "AWS": "*" - }, - "Action": "SQS:SendMessage", - "Resource": "arn:aws:sqs:" + region + ":" + acnumber + ":" + sqsname, - "Condition": { - "ArnLike": { - "aws:SourceArn": "arn:aws:s3:*:*:" + s3bucket - } - } - }, - { - "Sid": "GiveAccessToLoggly", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::" + acnumber + ":root" - }, - "Action": "SQS:*", - "Resource": "arn:aws:sqs:" + region + ":" + acnumber + ":" + sqsname - } - ] - })) - - # s3 bucket notification configuration - client = boto3.client('s3', region) - - - response = client.put_bucket_notification_configuration( - Bucket=s3bucket, - NotificationConfiguration={ - "QueueConfigurations": [{ - "Id": "Notification", - "Events": ["s3:ObjectCreated:*"], - "QueueArn": "arn:aws:sqs:" + region + ":" + acnumber + ":" + sqsname - }], - } - ) - + # queue exists + add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3bucket) else: # queue does not exist and no sqs queue name is passed if sqsname == None: @@ -234,139 +190,24 @@ queue_name = conn.get_queue(sqsname) # Default queue already exists - if queue_name!= None: - queue_attr_raw = conn.get_queue_attributes(queue_name, attribute='All') - queue_attr = str(queue_attr_raw) - - if s3bucket in queue_attr: - print 'Bucket already exists in queue\'s policy' - - else: - # append the bucket to the existing policy - print "A bucket already exists in this queue's policy, appending this bucket to it" - - start = '\"aws:SourceArn\":' - end = '}}}' - result = re.search('%s(.*)%s' % (start, end), queue_attr).group(1) - - - leftbracketremoved = result.replace('[','') - rightbracketremoved = leftbracketremoved.replace(']','') - - addon = '[' + rightbracketremoved + ',' + '\"arn:aws:s3:*:*:' + s3bucket +'\"' +']' - - - text = """ { - "Version": "2008-10-17", - "Id": "PolicyExample", - "Statement": [ - { - "Sid": "example-statement-ID", - "Effect": "Allow", - "Principal": { - "AWS": "*" - }, - "Action": "SQS:SendMessage", - "Resource": "arn:aws:sqs:%s:%s:%s", - "Condition": { - "ArnLike": { - "aws:SourceArn": %s - } - } - }, - { - "Sid": "GiveAccessToLoggly", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::%s:root" - }, - "Action": "SQS:*", - "Resource": "arn:aws:sqs:%s:%s:%s" - } - ] - } - """ % (region, acnumber, sqsname, addon, acnumber, region, acnumber, sqsname) - - - parsed = json.loads(text) - - conn.set_queue_attribute(queue_name, 'Policy', json.dumps(parsed)) - - # s3 bucket notification configuration - client = boto3.client('s3', region) - - - response = client.put_bucket_notification_configuration( - Bucket=s3bucket, - NotificationConfiguration={ - "QueueConfigurations": [{ - "Id": "Notification", - "Events": ["s3:ObjectCreated:*"], - "QueueArn": "arn:aws:sqs:" + region + ":" + acnumber + ":" + sqsname - }], - } - ) - + if queue_name != None: + add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3bucket) else: # create the default queue or the queue passed as a parameter q = conn.create_queue(sqsname) queue_name = conn.get_queue(sqsname) - conn.set_queue_attribute(queue_name, 'Policy', json.dumps({ - "Version": "2008-10-17", - "Id": "PolicyExample", - "Statement": [ - { - "Sid": "example-statement-ID", - "Effect": "Allow", - "Principal": { - "AWS": "*" - }, - "Action": "SQS:SendMessage", - "Resource": "arn:aws:sqs:" + region + ":" + acnumber + ":" + sqsname, - "Condition": { - "ArnLike": { - "aws:SourceArn": "arn:aws:s3:*:*:" + s3bucket - } - } - }, - { - "Sid": "GiveAccessToLoggly", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::" + acnumber + ":root" - }, - "Action": "SQS:*", - "Resource": "arn:aws:sqs:" + region + ":" + acnumber + ":" + sqsname - } - ] - })) - - # s3 bucket notification configuration - client = boto3.client('s3', region) - - response = client.put_bucket_notification_configuration( - Bucket=s3bucket, - NotificationConfiguration={ - "QueueConfigurations": [{ - "Id": "Notification", - "Events": ["s3:ObjectCreated:*"], - "QueueArn": "arn:aws:sqs:" + region + ":" + acnumber + ":" + sqsname - }], - } - ) - + add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3bucket) print "Queue Name" print sqsname iam = boto.connect_iam(access_key, secret_key) - if user != None and user != '': try: - response = iam.get_user(user) + response = iam.get_user(user) if 'get_user_response' in response: print 'IAM user %s already exists, appending the sqs queue and s3 bucket to this IAM user\'s policy' % user @@ -380,24 +221,23 @@ s = StringIO.StringIO(existing_policy_decoded) for line in s: - if 'arn:aws:sqs' in line: - sqsQueues.append(line.strip().replace(",", "")) - if 'arn:aws:s3' in line: - s3Buckets.append(line.strip().replace(",", "")) + if 'arn:aws:sqs' in line: + sqsQueues.append(line.strip().replace(",", "")) + if 'arn:aws:s3' in line: + s3Buckets.append(line.strip().replace(",", "")) # append current s3bucket and sqs queue sqsQueues.append('\"arn:aws:sqs:%s:%s:%s\"' % (region, acnumber, sqsname,)) s3Buckets.append('\"arn:aws:s3:::%s/*\"' % (s3bucket)) s3Buckets.append('\"arn:aws:s3:::%s\"' % (s3bucket)) - sqsQueueAddOn="" + sqsQueueAddOn = "" for entry in sqsQueues: - sqsQueueAddOn = sqsQueueAddOn + entry + ",\n" + sqsQueueAddOn = sqsQueueAddOn + entry + ",\n" - - s3BucketAddOn="" + s3BucketAddOn = "" for entry in s3Buckets: - s3BucketAddOn = s3BucketAddOn + entry + ",\n" + s3BucketAddOn = s3BucketAddOn + entry + ",\n" policy_json = """{ "Version": "2012-10-17", @@ -426,14 +266,13 @@ ] }""" % (sqsQueueAddOn[:-2], s3BucketAddOn[:-2],) - response = iam.put_user_policy(user, 'LogglyUserPolicy', policy_json) print "" print 'Appended! Please provide the access key and secret key for the IAM user %s in the form fields' % user except BotoServerError, e: - if "The user with name" in e.message and "cannot be found" in e.message : + if "The user with name" in e.message and "cannot be found" in e.message: # create an IAM user response = iam.create_user(user) @@ -455,7 +294,6 @@ print "" print "Please save the above credentials" - policy_json = """{ "Version": "2012-10-17", "Statement": [ @@ -484,7 +322,6 @@ ] }""" % (region, acnumber, sqsname, s3bucket, s3bucket,) - response = iam.put_user_policy(user, 'LogglyUserPolicy', policy_json) @@ -497,7 +334,7 @@ user = 'loggly-s3-user' try: - response = iam.get_user(user) + response = iam.get_user(user) if 'get_user_response' in response: print 'The default IAM user \'loggly-s3-user\' which the script creates already exists, appending the sqs queue and s3 bucket to this IAM user\'s policy' @@ -511,24 +348,23 @@ s = StringIO.StringIO(existing_policy_decoded) for line in s: - if 'arn:aws:sqs' in line: - sqsQueues.append(line.strip().replace(",", "")) - if 'arn:aws:s3' in line: - s3Buckets.append(line.strip().replace(",", "")) + if 'arn:aws:sqs' in line: + sqsQueues.append(line.strip().replace(",", "")) + if 'arn:aws:s3' in line: + s3Buckets.append(line.strip().replace(",", "")) # append current s3bucket and sqs queue sqsQueues.append('\"arn:aws:sqs:%s:%s:%s\"' % (region, acnumber, sqsname,)) s3Buckets.append('\"arn:aws:s3:::%s/*\"' % (s3bucket)) s3Buckets.append('\"arn:aws:s3:::%s\"' % (s3bucket)) - sqsQueueAddOn="" + sqsQueueAddOn = "" for entry in sqsQueues: - sqsQueueAddOn = sqsQueueAddOn + entry + ",\n" - + sqsQueueAddOn = sqsQueueAddOn + entry + ",\n" - s3BucketAddOn="" + s3BucketAddOn = "" for entry in s3Buckets: - s3BucketAddOn = s3BucketAddOn + entry + ",\n" + s3BucketAddOn = s3BucketAddOn + entry + ",\n" policy_json = """{ "Version": "2012-10-17", @@ -557,18 +393,16 @@ ] }""" % (sqsQueueAddOn[:-2], s3BucketAddOn[:-2],) - try: - response = iam.put_user_policy(user, 'LogglyUserPolicy', policy_json) + response = iam.put_user_policy(user, 'LogglyUserPolicy', policy_json) except Exception, e: - print e + print e print "" print "Appended! Please provide the access key and secret key for the IAM user \'loggly-s3-user\' in the form fields" except BotoServerError, e: - if "The user with name" in e.message and "cannot be found" in e.message : - + if "The user with name" in e.message and "cannot be found" in e.message: response = iam.create_user(user) # create an access key From 265f743a7c5eca18245fcc7fe1dfc4591be0cb5e Mon Sep 17 00:00:00 2001 From: Michal Chomo Date: Fri, 26 Jul 2019 14:23:02 +0200 Subject: [PATCH 2/6] further refactoring --- AWSscripts/SQS3script.py | 398 +++++++++++++-------------------------- 1 file changed, 126 insertions(+), 272 deletions(-) diff --git a/AWSscripts/SQS3script.py b/AWSscripts/SQS3script.py index d951557..3db4c91 100644 --- a/AWSscripts/SQS3script.py +++ b/AWSscripts/SQS3script.py @@ -18,9 +18,9 @@ from boto.exception import BotoServerError, S3ResponseError -from optparse import OptionParser +import argparse -SQS_QUEUE_POLICY_TEMPLATE = """ { +SQS_QUEUE_POLICY_TEMPLATE = """{ "Version": "2008-10-17", "Id": "PolicyExample", "Statement": [ @@ -48,8 +48,34 @@ "Resource": "arn:aws:sqs:%(region)s:%(acnumber)s:%(queue_name)s" } ] - } - """ + }""" + +USER_POLICY_TEMPLATE = """{ + "Version": "2012-10-17", + "Statement": [ + { + "Sid": "Sidtest", + "Effect": "Allow", + "Action": [ + "sqs:*" + ], + "Resource": [ + "%(sqs_resource)s" + ] + }, + { + "Effect": "Allow", + "Action":[ + "s3:ListBucket", + "s3:GetObject", + "s3:GetBucketLocation" + ], + "Resource": [ + "%(s3_resource)s" + ] + } + ] + }""" def put_bucket_notification_config(client, region, s3bucket, acnumber, sqs_queue_name): @@ -104,19 +130,75 @@ def add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3buc put_bucket_notification_config(client, region, s3bucket, acnumber, queue_name) -parser = OptionParser() -parser.add_option("--acnumber", dest="acnumber", +def get_policy_resources(iam, user): + existing_policy = str(iam.get_user_policy(user, 'LogglyUserPolicy')) + existing_policy_decoded = urllib.unquote(existing_policy) + + s3Buckets = [] + sqsQueues = [] + + response = iam.get_all_access_keys(user, max_items=1) + + s = StringIO.StringIO(existing_policy_decoded) + for line in s: + if 'arn:aws:sqs' in line: + sqsQueues.append(line.strip().replace(",", "")) + if 'arn:aws:s3' in line: + s3Buckets.append(line.strip().replace(",", "")) + + # append current s3bucket and sqs queue + sqsQueues.append('\"arn:aws:sqs:%s:%s:%s\"' % (region, acnumber, sqsname,)) + s3Buckets.append('\"arn:aws:s3:::%s/*\"' % (s3bucket)) + s3Buckets.append('\"arn:aws:s3:::%s\"' % (s3bucket)) + + sqsQueuesString = "" + for entry in sqsQueues: + sqsQueuesString = sqsQueuesString + entry + ",\n" + + s3BucketsString = "" + for entry in s3Buckets: + s3BucketsString = s3BucketsString + entry + ",\n" + + return sqsQueuesString, s3BucketsString + + +def set_user_policy(iam, user, sqs_resource, s3_resource): + policy_json = USER_POLICY_TEMPLATE % ({"sqs_resource": sqs_resource, "s3_resource": s3_resource}) + iam.put_user_policy(user, 'LogglyUserPolicy', policy_json) + + +def create_user_and_access_keys(iam, user): + # create an IAM user + iam.create_user(user) + + # create an access key + response = iam.create_access_key(user) + loggly_access_key = response.access_key_id + loggly_secret_key = response.secret_access_key + + print "Access key for Loggly" + print loggly_access_key + + print "Secret key for Loggly" + print loggly_secret_key + + print "" + print "Please save the above credentials" + + +parser = argparse.ArgumentParser() +parser.add_argument("--acnumber", dest="acnumber", help="account number") -parser.add_option("--s3bucket", dest="s3bucket", +parser.add_argument("--s3bucket", dest="s3bucket", help="s3 bucket name") -parser.add_option("--user", dest="user", +parser.add_argument("--user", dest="user", help="user") -parser.add_option("--admin", dest="admin", +parser.add_argument("--admin", dest="admin", help="admin user name") -parser.add_option("--sqsname", dest="sqsname", +parser.add_argument("--sqsname", dest="sqsname", help="sqsname") -(opts, args) = parser.parse_args() +opts = parser.parse_args() s3bucket = opts.s3bucket acnumber = opts.acnumber @@ -124,9 +206,14 @@ def add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3buc user = opts.user admin = opts.admin -if acnumber.isdigit() == False: - print "Please check your account number, it should only contain digits, no other characters." - sys.exit() +if not s3bucket: + parser.error("S3 bucket name not provided") + +if not acnumber: + parser.error("Account number not provided") + +if not acnumber.isdigit(): + parser.error("Please check your account number, it should only contain digits, no other characters.") conn = boto.connect_s3() @@ -148,38 +235,33 @@ def add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3buc break if not access_key: - print "Please check your ~/.aws/credentials file and make sure that access key and secret access key are set. Run aws configure to set them up" + print "AWS access key is not set. Please make sure to execute 'aws configure' before this script" + sys.exit() if not secret_key: - print "Please check your ~/.aws/credentials file and make sure that access key and secret access key are set. Run aws configure to set them up" + print "AWS secret key is not set. Please make sure to execute 'aws configure' before this script" + sys.exit() try: bucket = conn.get_bucket(s3bucket) except S3ResponseError, e: + print e if "Not Found" in e: - print e print 'S3 bucket ' + s3bucket + ' does not exist, please create it and run the script again' - sys.exit() - else: - print e - sys.exit() + elif "Forbidden": + print "Access to AWS is forbidden, please make sure to execute 'aws configure' before this script" + sys.exit() region = bucket.get_location() if region == '': region = 'us-east-1' -if not s3bucket: - parser.error("S3 bucket name not provided") - -if not acnumber: - parser.error("Account number not provided") - conn = boto.sqs.connect_to_region(region, aws_access_key_id=access_key, aws_secret_access_key=secret_key) client = boto3.client('s3', region) queue_name = conn.get_queue(sqsname) -if "None" not in str(queue_name): +if queue_name is not None: # queue exists add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3bucket) else: @@ -204,252 +286,24 @@ def add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3buc iam = boto.connect_iam(access_key, secret_key) -if user != None and user != '': - - try: - response = iam.get_user(user) - if 'get_user_response' in response: - print 'IAM user %s already exists, appending the sqs queue and s3 bucket to this IAM user\'s policy' % user - - existing_policy = str(iam.get_user_policy(user, 'LogglyUserPolicy')) - existing_policy_decoded = urllib.unquote(existing_policy) - - s3Buckets = [] - sqsQueues = [] - - response = iam.get_all_access_keys(user, max_items=1) - - s = StringIO.StringIO(existing_policy_decoded) - for line in s: - if 'arn:aws:sqs' in line: - sqsQueues.append(line.strip().replace(",", "")) - if 'arn:aws:s3' in line: - s3Buckets.append(line.strip().replace(",", "")) - - # append current s3bucket and sqs queue - sqsQueues.append('\"arn:aws:sqs:%s:%s:%s\"' % (region, acnumber, sqsname,)) - s3Buckets.append('\"arn:aws:s3:::%s/*\"' % (s3bucket)) - s3Buckets.append('\"arn:aws:s3:::%s\"' % (s3bucket)) - - sqsQueueAddOn = "" - for entry in sqsQueues: - sqsQueueAddOn = sqsQueueAddOn + entry + ",\n" - - s3BucketAddOn = "" - for entry in s3Buckets: - s3BucketAddOn = s3BucketAddOn + entry + ",\n" - - policy_json = """{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "Sidtest", - "Effect": "Allow", - "Action": [ - "sqs:*" - ], - "Resource": [ - %s - ] - }, - { - "Effect": "Allow", - "Action":[ - "s3:ListBucket", - "s3:GetObject", - "s3:GetBucketLocation" - ], - "Resource": [ - %s - ] - } - ] - }""" % (sqsQueueAddOn[:-2], s3BucketAddOn[:-2],) - - response = iam.put_user_policy(user, 'LogglyUserPolicy', policy_json) - - print "" - print 'Appended! Please provide the access key and secret key for the IAM user %s in the form fields' % user - - except BotoServerError, e: - if "The user with name" in e.message and "cannot be found" in e.message: - - # create an IAM user - response = iam.create_user(user) - - # create an access key - iam.create_access_key(user) - response = iam.create_access_key(user) - loggly_access_key = response.access_key_id - loggly_secret_key = response.secret_access_key - - print "Access key for Loggly" - - print loggly_access_key - - print "Secret key for Loggly" - - print loggly_secret_key - - print "" - print "Please save the above credentials" - - policy_json = """{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "Sidtest", - "Effect": "Allow", - "Action": [ - "sqs:*" - ], - "Resource": [ - "arn:aws:sqs:%s:%s:%s" - ] - }, - { - "Effect": "Allow", - "Action":[ - "s3:ListBucket", - "s3:GetObject", - "s3:GetBucketLocation" - ], - "Resource": [ - "arn:aws:s3:::%s/*", - "arn:aws:s3:::%s" - ] - } - ] - }""" % (region, acnumber, sqsname, s3bucket, s3bucket,) - - response = iam.put_user_policy(user, - 'LogglyUserPolicy', - policy_json) - else: - - print(e.message) - -else: - # create an IAM user +if user is None or user == '': user = 'loggly-s3-user' +try: + response = iam.get_user(user) + if 'get_user_response' in response: + print 'IAM user %s already exists, appending the sqs queue and s3 bucket to this IAM user\'s policy' % user - try: - response = iam.get_user(user) - if 'get_user_response' in response: - print 'The default IAM user \'loggly-s3-user\' which the script creates already exists, appending the sqs queue and s3 bucket to this IAM user\'s policy' - - existing_policy = str(iam.get_user_policy(user, 'LogglyUserPolicy')) - existing_policy_decoded = urllib.unquote(existing_policy) - - s3Buckets = [] - sqsQueues = [] - - response = iam.get_all_access_keys(user, max_items=1) - - s = StringIO.StringIO(existing_policy_decoded) - for line in s: - if 'arn:aws:sqs' in line: - sqsQueues.append(line.strip().replace(",", "")) - if 'arn:aws:s3' in line: - s3Buckets.append(line.strip().replace(",", "")) - - # append current s3bucket and sqs queue - sqsQueues.append('\"arn:aws:sqs:%s:%s:%s\"' % (region, acnumber, sqsname,)) - s3Buckets.append('\"arn:aws:s3:::%s/*\"' % (s3bucket)) - s3Buckets.append('\"arn:aws:s3:::%s\"' % (s3bucket)) - - sqsQueueAddOn = "" - for entry in sqsQueues: - sqsQueueAddOn = sqsQueueAddOn + entry + ",\n" - - s3BucketAddOn = "" - for entry in s3Buckets: - s3BucketAddOn = s3BucketAddOn + entry + ",\n" - - policy_json = """{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "Sidtest", - "Effect": "Allow", - "Action": [ - "sqs:*" - ], - "Resource": [ - %s - ] - }, - { - "Effect": "Allow", - "Action":[ - "s3:ListBucket", - "s3:GetObject", - "s3:GetBucketLocation" - ], - "Resource": [ - %s - ] - } - ] - }""" % (sqsQueueAddOn[:-2], s3BucketAddOn[:-2],) - - try: - response = iam.put_user_policy(user, 'LogglyUserPolicy', policy_json) - except Exception, e: - print e - - print "" - print "Appended! Please provide the access key and secret key for the IAM user \'loggly-s3-user\' in the form fields" - - except BotoServerError, e: - if "The user with name" in e.message and "cannot be found" in e.message: - response = iam.create_user(user) - - # create an access key - iam.create_access_key(user) - response = iam.create_access_key(user) - loggly_access_key = response.access_key_id - loggly_secret_key = response.secret_access_key - - print "Access key for Loggly" - - print loggly_access_key - - print "Secret key for Loggly" + sqs_resource, s3_resource = get_policy_resources(iam, user) + set_user_policy(iam, user, sqs_resource, s3_resource) - print loggly_secret_key + print "" + print 'Appended! Please provide the access key and secret key for the IAM user %s in the form fields' % user - print "" - print "Please save the above credentials" +except BotoServerError, e: + if "The user with name" in e.message and "cannot be found" in e.message: - policy_json = """{ - "Version": "2012-10-17", - "Statement": [ - { - "Sid": "Sidtest", - "Effect": "Allow", - "Action": [ - "sqs:*" - ], - "Resource": [ - "arn:aws:sqs:%s:%s:%s" - ] - }, - { - "Effect": "Allow", - "Action":[ - "s3:ListBucket", - "s3:GetObject", - "s3:GetBucketLocation" - ], - "Resource": [ - "arn:aws:s3:::%s/*", - "arn:aws:s3:::%s" - ] - } - ] - }""" % (region, acnumber, sqsname, s3bucket, s3bucket,) - - response = iam.put_user_policy(user, - 'LogglyUserPolicy', - policy_json) + create_user_and_access_keys(iam, user) + set_user_policy(iam, user, "arn:aws:sqs:%s:%s:%s" % (region, acnumber, sqsname), + "arn:aws:s3:::%(bucket)s/*,\narn:aws:s3:::%(bucket)s" % ({"bucket": s3bucket})) + else: + print(e.message) From f361f2c97afd151186c2d9a96312e4d470c9c27c Mon Sep 17 00:00:00 2001 From: Michal Chomo Date: Fri, 16 Aug 2019 15:35:47 +0200 Subject: [PATCH 3/6] major refactor, use boto3 and OOP, improve work with policy --- AWSscripts/SQS3script.py | 627 ++++++++++++++++++++------------------- 1 file changed, 326 insertions(+), 301 deletions(-) diff --git a/AWSscripts/SQS3script.py b/AWSscripts/SQS3script.py index 3db4c91..8f2ff29 100644 --- a/AWSscripts/SQS3script.py +++ b/AWSscripts/SQS3script.py @@ -1,309 +1,334 @@ -# Instructions - -# python SQS3script.py --s3bucket --acnumber --sqsname --user -# s3bucket, acnumber parameters are mandatory, sqsname and user are optional - -# This script assumes that the aws credentials are created at ~/.aws/credentials by running aws configure on command line -# region examples: us-east-1, us-west-2 etc. - - -import boto.sqs.connection -import boto3 +import argparse import json -import os -import re import sys -import urllib -import StringIO -from boto.exception import BotoServerError, S3ResponseError - -import argparse +import boto3 +import botocore + + +class AWS: + """Encapsulates AWS session with resources (S3 bucket, SQS queue, IAM user).""" + QUEUE_BUCKET_POLICY_JSON = """ + { + "Effect": "Allow", + "Principal": { + "AWS": "*" + }, + "Action": "SQS:SendMessage", + "Resource": "%(queue_arn)s", + "Condition": { + "ArnLike": { + "aws:SourceArn": "arn:aws:s3:::%(bucket_name)s" + } + } + } + """ + + QUEUE_ACCOUNT_POLICY_JSON = """ + { + "Effect": "Allow", + "Principal": { + "AWS": "arn:aws:iam::%(account_number)s:root" + }, + "Action": "SQS:*", + "Resource": "%(queue_arn)s" + } + """ -SQS_QUEUE_POLICY_TEMPLATE = """{ - "Version": "2008-10-17", - "Id": "PolicyExample", - "Statement": [ - { - "Sid": "example-statement-ID", - "Effect": "Allow", - "Principal": { - "AWS": "*" - }, - "Action": "SQS:SendMessage", - "Resource": "arn:aws:sqs:%(region)s:%(acnumber)s:%(queue_name)s", - "Condition": { - "ArnLike": { - "aws:SourceArn": %(source_arn)s - } - } - }, - { - "Sid": "GiveAccessToLoggly", - "Effect": "Allow", - "Principal": { - "AWS": "arn:aws:iam::%(acnumber)s:root" - }, - "Action": "SQS:*", - "Resource": "arn:aws:sqs:%(region)s:%(acnumber)s:%(queue_name)s" - } - ] - }""" - -USER_POLICY_TEMPLATE = """{ + QUEUE_POLICY_WHOLE_JSON = """ + { "Version": "2012-10-17", - "Statement": [ - { - "Sid": "Sidtest", - "Effect": "Allow", - "Action": [ - "sqs:*" - ], - "Resource": [ - "%(sqs_resource)s" - ] - }, - { - "Effect": "Allow", - "Action":[ - "s3:ListBucket", - "s3:GetObject", - "s3:GetBucketLocation" - ], - "Resource": [ - "%(s3_resource)s" - ] - } - ] - }""" - - -def put_bucket_notification_config(client, region, s3bucket, acnumber, sqs_queue_name): - client.put_bucket_notification_configuration( - Bucket=s3bucket, - NotificationConfiguration={ - "QueueConfigurations": [{ - "Id": "Notification", - "Events": ["s3:ObjectCreated:*"], - "QueueArn": "arn:aws:sqs:" + region + ":" + acnumber + ":" + sqs_queue_name - }], + "Statement": [%s, %s] } - ) - + """ % (QUEUE_BUCKET_POLICY_JSON, QUEUE_ACCOUNT_POLICY_JSON) -def get_source_arn_with_appended_bucket(queue_attr, s3bucket): - start = '\"aws:SourceArn\":' - end = '}}}' - result = re.search('%s(.*)%s' % (start, end), queue_attr).group(1) - - leftbracketremoved = result.replace('[', '') - rightbracketremoved = leftbracketremoved.replace(']', '') - - return '[' + rightbracketremoved + ',' + '\"arn:aws:s3:*:*:' + s3bucket + '\"' + ']' - - -def set_sqs_queue_policy(conn, region, acnumber, queue_name, source_arn): - queue_policy = SQS_QUEUE_POLICY_TEMPLATE % ( - {"region": region, "acnumber": acnumber, "queue_name": queue_name, "source_arn": source_arn}) - - parsed = json.loads(queue_policy) - - conn.set_queue_attribute(queue_name, 'Policy', json.dumps(parsed)) - - -def add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3bucket): - queue_attr_raw = conn.get_queue_attributes(queue_name, attribute='All') - queue_attr = str(queue_attr_raw) - - if 'arn:aws:s3' in queue_attr: - if "arn:aws:s3:*:*:" + s3bucket in queue_attr: - print 'Given bucket already exists in queue\'s policy' + QUEUE_CONFIGURATIONS_JSON = """ + { + "QueueConfigurations": [ + { + "Id": "LogglyS3Notification", + "Events": ["s3:ObjectCreated:*"], + "QueueArn": "%(queue_arn)s" + } + ] + } + """ + + USER_POLICY_BUCKET_JSON = """ + { + "Effect": "Allow", + "Action":[ + "s3:ListBucket", + "s3:GetObject", + "s3:GetBucketLocation" + ], + "Resource": [ + "arn:aws:s3:::%(bucket_name)s/*", + "arn:aws:s3:::%(bucket_name)s" + ] + } + """ + + USER_POLICY_QUEUE_JSON = """ + { + "Effect": "Allow", + "Action": [ + "sqs:*" + ], + "Resource": [ + "%(queue_arn)s" + ] + } + """ + + USER_POLICY_WHOLE_JSON = """{ + "Version": "2012-10-17", + "Statement": [%s, %s] + } + """ % (USER_POLICY_BUCKET_JSON, USER_POLICY_QUEUE_JSON) + + class PolicyDocument: + """Represents the JSON AWS policy.""" + + def __init__(self, policy_document): + self._policy_document = policy_document + + def get_policy(self): + return self._policy_document + + def add_statement_or_resource(self, action, resource, statement_to_add): + statement, index = self.get_statement(action) + if not statement: + # No statement contains required action, create a new statement. + self.add_statement(statement_to_add) + else: + # Required action is contained in a statement, check resource. + statement, index = self.get_statement(action, resource) + if not statement: + # Required resource is missing, add it to the statement with the required action. + self.add_resource_to_statement(resource, index) + + def add_statement(self, statement, index=None): + if index: + self._policy_document['Statement'][index] = statement + else: + self._policy_document['Statement'].append(statement) + + def get_statement(self, action=None, resource=None, effect='Allow'): + """Returns a statement with matching action, resource and effect. For the resource and action it is + sufficient if it is contained in the list. Statement index is also returned.""" + + statements = self._policy_document['Statement'] + for s, i in zip(statements, range(0, len(statements))): + if s['Effect'] != effect: + continue + if action: + s['Action'] = self._make_lowercase(s['Action']) + action = self._make_lowercase(action) + if not self._compare_fields(s['Action'], action): + continue + if resource: + if not self._compare_fields(s['Resource'], resource): + continue + return s, i + return None, None + + def add_resource_to_statement(self, resource, statement_index): + statement = self._policy_document['Statement'][statement_index] + try: + if isinstance(statement['Resource'], list): + statement['Resource'].append(resource) + else: + statement['Resource'] = [statement['Resource'], resource] + except KeyError: + statement['Resource'] = resource + self.add_statement(statement, statement_index) + + def _compare_fields(self, statement_field, given_field): + if isinstance(statement_field, list): + if isinstance(given_field, list): + if set(statement_field) != set(given_field): + return False + elif given_field not in statement_field: + return False + elif statement_field != given_field: + return False + return True + + def _make_lowercase(self, field): + if isinstance(field, list): + for i, e in enumerate(field): + field[i] = e.lower() + else: + field = field.lower() + return field + + def __init__(self, session, bucket, queue, user, account_id): + self._session = session + self._bucket = bucket + self._queue = queue + self._user = user + self._account_id = account_id + + def set_queue_policy(self): + try: + queue_policy = json.loads(self._queue.attributes['Policy']) + except KeyError: + print('Queue policy not found, creating it') + params_dict = {'queue_arn': self._queue.attributes['QueueArn'], + 'bucket_name': self._bucket.name, + 'account_number': self._account_id} + self._queue.set_attributes(Attributes={'Policy': self.QUEUE_POLICY_WHOLE_JSON % params_dict}) + return + pd = self.PolicyDocument(queue_policy) + self._add_bucket_to_queue_policy(pd) + self._queue.add_permission(Label='AccountAccess', AWSAccountIds=[self._account_id], Actions=['*']) + + def set_bucket_notification(self): + self._bucket.Notification().put( + NotificationConfiguration=json.loads( + self.QUEUE_CONFIGURATIONS_JSON % {'queue_arn': self._queue.attributes['QueueArn']})) + + def set_user_policy(self): + policy_name = 'LogglyUserPolicy' + try: + policy = self._user.Policy(policy_name) + policy.policy_document # This raises exception on non existent policy. + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'NoSuchEntity': + print('Policy {} not found, creating it'.format(policy_name)) + params_dict = {'queue_arn': self._queue.attributes['QueueArn'], 'bucket_name': self._bucket.name} + self._user.create_policy( + PolicyName=policy_name, PolicyDocument=self.USER_POLICY_WHOLE_JSON % params_dict) + return + else: + raise e + pd = self.PolicyDocument(policy.policy_document) + self._add_bucket_to_user_policy(pd) + self._add_queue_to_user_policy(pd) + policy.put(PolicyDocument=json.dumps(pd.get_policy())) + + def _add_bucket_to_queue_policy(self, policy_document): + statement, index = policy_document.get_statement( + action='sqs:sendmessage', resource=self._queue.attributes['QueueArn']) + if not statement: + # Create a whole new statement. + params_dict = {'queue_arn': self._queue.attributes['QueueArn'], 'bucket_name': self._bucket.name} + policy_document.add_statement(json.loads(self.QUEUE_BUCKET_POLICY_JSON % params_dict)) else: - # append the bucket to the existing policy - print "A bucket already exists in this queue's policy, appending this bucket to it" - - source_arn = get_source_arn_with_appended_bucket(queue_attr, s3bucket) - set_sqs_queue_policy(conn, region, acnumber, queue_name, source_arn) - put_bucket_notification_config(client, region, s3bucket, acnumber, queue_name) - else: - set_sqs_queue_policy(conn, region, acnumber, queue_name, "arn:aws:s3:*:*:" + s3bucket) - put_bucket_notification_config(client, region, s3bucket, acnumber, queue_name) - - -def get_policy_resources(iam, user): - existing_policy = str(iam.get_user_policy(user, 'LogglyUserPolicy')) - existing_policy_decoded = urllib.unquote(existing_policy) - - s3Buckets = [] - sqsQueues = [] - - response = iam.get_all_access_keys(user, max_items=1) - - s = StringIO.StringIO(existing_policy_decoded) - for line in s: - if 'arn:aws:sqs' in line: - sqsQueues.append(line.strip().replace(",", "")) - if 'arn:aws:s3' in line: - s3Buckets.append(line.strip().replace(",", "")) - - # append current s3bucket and sqs queue - sqsQueues.append('\"arn:aws:sqs:%s:%s:%s\"' % (region, acnumber, sqsname,)) - s3Buckets.append('\"arn:aws:s3:::%s/*\"' % (s3bucket)) - s3Buckets.append('\"arn:aws:s3:::%s\"' % (s3bucket)) - - sqsQueuesString = "" - for entry in sqsQueues: - sqsQueuesString = sqsQueuesString + entry + ",\n" - - s3BucketsString = "" - for entry in s3Buckets: - s3BucketsString = s3BucketsString + entry + ",\n" - - return sqsQueuesString, s3BucketsString - - -def set_user_policy(iam, user, sqs_resource, s3_resource): - policy_json = USER_POLICY_TEMPLATE % ({"sqs_resource": sqs_resource, "s3_resource": s3_resource}) - iam.put_user_policy(user, 'LogglyUserPolicy', policy_json) - - -def create_user_and_access_keys(iam, user): - # create an IAM user - iam.create_user(user) - - # create an access key - response = iam.create_access_key(user) - loggly_access_key = response.access_key_id - loggly_secret_key = response.secret_access_key - - print "Access key for Loggly" - print loggly_access_key - - print "Secret key for Loggly" - print loggly_secret_key - - print "" - print "Please save the above credentials" - - -parser = argparse.ArgumentParser() -parser.add_argument("--acnumber", dest="acnumber", - help="account number") -parser.add_argument("--s3bucket", dest="s3bucket", - help="s3 bucket name") -parser.add_argument("--user", dest="user", - help="user") -parser.add_argument("--admin", dest="admin", - help="admin user name") -parser.add_argument("--sqsname", dest="sqsname", - help="sqsname") - -opts = parser.parse_args() - -s3bucket = opts.s3bucket -acnumber = opts.acnumber -sqsname = opts.sqsname -user = opts.user -admin = opts.admin - -if not s3bucket: - parser.error("S3 bucket name not provided") - -if not acnumber: - parser.error("Account number not provided") - -if not acnumber.isdigit(): - parser.error("Please check your account number, it should only contain digits, no other characters.") - -conn = boto.connect_s3() - -access_key = '' -secret_key = '' -bucket = '' - -credentials_name = admin if admin else "default" -home = os.path.expanduser("~") - -with open(home + '/.aws/credentials') as f: - for line in f: - if credentials_name in line: - for line in f: - if "aws_access_key_id" in line: - access_key = line.split("=", 1)[1].strip() - if "aws_secret_access_key" in line: - secret_key = line.split("=", 1)[1].strip() - break - -if not access_key: - print "AWS access key is not set. Please make sure to execute 'aws configure' before this script" - sys.exit() - -if not secret_key: - print "AWS secret key is not set. Please make sure to execute 'aws configure' before this script" - sys.exit() - -try: - bucket = conn.get_bucket(s3bucket) -except S3ResponseError, e: - print e - if "Not Found" in e: - print 'S3 bucket ' + s3bucket + ' does not exist, please create it and run the script again' - elif "Forbidden": - print "Access to AWS is forbidden, please make sure to execute 'aws configure' before this script" - sys.exit() - -region = bucket.get_location() - -if region == '': - region = 'us-east-1' - -conn = boto.sqs.connect_to_region(region, aws_access_key_id=access_key, aws_secret_access_key=secret_key) -client = boto3.client('s3', region) -queue_name = conn.get_queue(sqsname) - -if queue_name is not None: - # queue exists - add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3bucket) -else: - # queue does not exist and no sqs queue name is passed - if sqsname == None: - sqsname = 'loggly-s3-queue' - - queue_name = conn.get_queue(sqsname) - - # Default queue already exists - if queue_name != None: - add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3bucket) - else: - # create the default queue or the queue passed as a parameter - q = conn.create_queue(sqsname) - queue_name = conn.get_queue(sqsname) - - add_bucket_to_queue_policy(conn, client, region, acnumber, queue_name, s3bucket) - -print "Queue Name" -print sqsname - -iam = boto.connect_iam(access_key, secret_key) - -if user is None or user == '': - user = 'loggly-s3-user' -try: - response = iam.get_user(user) - if 'get_user_response' in response: - print 'IAM user %s already exists, appending the sqs queue and s3 bucket to this IAM user\'s policy' % user - - sqs_resource, s3_resource = get_policy_resources(iam, user) - set_user_policy(iam, user, sqs_resource, s3_resource) - - print "" - print 'Appended! Please provide the access key and secret key for the IAM user %s in the form fields' % user - -except BotoServerError, e: - if "The user with name" in e.message and "cannot be found" in e.message: - - create_user_and_access_keys(iam, user) - set_user_policy(iam, user, "arn:aws:sqs:%s:%s:%s" % (region, acnumber, sqsname), - "arn:aws:s3:::%(bucket)s/*,\narn:aws:s3:::%(bucket)s" % ({"bucket": s3bucket})) - else: - print(e.message) + try: + bucket_arn = statement['Condition']['ArnLike']['aws:SourceArn'] + except KeyError: + # A statement with 'sqs:sendmessage' action already exists without ARN condition, no need to add the + # bucket ARN. + return + if isinstance(bucket_arn, list): + bucket_arn = ",".join(bucket_arn) + if self._bucket.name in bucket_arn: + print("Given bucket already exists in queue's policy") + return + else: + # A statement with 'sqs:sendmessage' action and some ARN condition already exists, + # just append the bucket ARN to its condition. + bucket_arn += ',arn:aws:s3:::' + self._bucket.name + statement['Condition']['ArnLike']['aws:SourceArn'] = bucket_arn.split() + policy_document.add_statement(statement, index) + self._queue.set_attributes(Attributes={'Policy': json.dumps(policy_document.get_policy())}) + + def _add_bucket_to_user_policy(self, policy_document): + policy_document.add_statement_or_resource(["s3:ListBucket", "s3:GetObject", "s3:GetBucketLocation"], + 'arn:aws:s3:::' + self._bucket.name, + self.USER_POLICY_BUCKET_JSON % {'bucket_name': self._bucket.name}) + + def _add_queue_to_user_policy(self, policy_document): + queue_arn = self._queue.attributes['QueueArn'] + policy_document.add_statement_or_resource( + 'sqs:*', queue_arn, self.USER_POLICY_QUEUE_JSON % {'queue_arn': queue_arn}) + + +def get_args(): + parser = argparse.ArgumentParser() + parser.add_argument("--acnumber", dest="acnumber", help="account number") + parser.add_argument("--s3bucket", dest="s3bucket", help="s3 bucket name") + parser.add_argument("--user", dest="user", default='loggly-s3-user', help="user") + parser.add_argument("--admin", dest="admin", default="default", help="admin user name") + parser.add_argument("--sqsname", dest="sqsname", default='loggly-s3-queue', help="sqsname") + + args = parser.parse_args() + + if not args.s3bucket: + parser.error("S3 bucket name not provided") + + if not args.acnumber: + parser.error("Account number not provided") + + if not args.acnumber.isdigit(): + parser.error("Please check your account number, it should only contain digits, no other characters.") + + return args + + +def get_bucket(session, bucket_name): + bucket = session.resource('s3').Bucket(bucket_name) + if bucket.creation_date is None: + print('S3 bucket {} does not exist, please create it and run the script again'.format(bucket_name)) + sys.exit(1) + return bucket + + +def get_queue(session, queue_name): + sqs = session.resource('sqs') + try: + queue = sqs.get_queue_by_name(QueueName=queue_name) + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'AWS.SimpleQueueService.NonExistentQueue': + print('Queue {} not found, creating it.'.format(queue_name)) + queue = sqs.create_queue(QueueName=queue_name) + else: + raise e + print('Queue url: {}'.format(queue.url)) + return queue + + +def get_user(session, user_name): + iam = session.resource('iam') + try: + user = iam.User(user_name) + user.arn # This raises exception on non existent user. + except botocore.exceptions.ClientError as e: + if e.response['Error']['Code'] == 'NoSuchEntity': + print("User {} does not exist, creating it") + user = iam.create_user(UserName=user_name) + access_key_pair = user.create_access_key_pair() + print("Access key for Loggly") + print(access_key_pair.access_key_id) + print("Secret key for Loggly") + print(access_key_pair.secret_access_key) + print('Please provide the access key and secret key for the IAM user {} in the form fields'.format(user_name)) + else: + raise e + return user + + +def main(): + args = get_args() + try: + session = boto3.Session(profile_name=args.admin) + bucket = get_bucket(session, args.s3bucket) + queue = get_queue(session, args.sqsname) + user = get_user(session, args.user) + aws = AWS(session, bucket, queue, user, args.acnumber) + aws.set_queue_policy() + print('Queue policy has been set up') + aws.set_bucket_notification() + print('Bucket notification has been set up') + aws.set_user_policy() + print('User policy has been set up') + except Exception as e: + print(e) + return 1 + return 0 + + +if __name__ == '__main__': + sys.exit(main()) From a15b10d6a5531ce17c95cd4fbd1fab6974454859 Mon Sep 17 00:00:00 2001 From: Michal Chomo Date: Fri, 16 Aug 2019 15:46:54 +0200 Subject: [PATCH 4/6] update requirements --- AWSscripts/requirements.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/AWSscripts/requirements.txt b/AWSscripts/requirements.txt index c630d94..7adf135 100644 --- a/AWSscripts/requirements.txt +++ b/AWSscripts/requirements.txt @@ -1,2 +1,2 @@ -boto==2.39.0 -boto3==1.2.3 +boto==2.49.0 +boto3==1.9.209 From f5bdee17b846cd948ff17c9ace09e5f1e542ddcc Mon Sep 17 00:00:00 2001 From: Michal Chomo Date: Fri, 16 Aug 2019 16:20:22 +0200 Subject: [PATCH 5/6] fix adding bucket ARN to queue policy --- AWSscripts/SQS3script.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/AWSscripts/SQS3script.py b/AWSscripts/SQS3script.py index 8f2ff29..4534f45 100644 --- a/AWSscripts/SQS3script.py +++ b/AWSscripts/SQS3script.py @@ -103,13 +103,12 @@ def add_statement_or_resource(self, action, resource, statement_to_add): self.add_statement(statement_to_add) else: # Required action is contained in a statement, check resource. - statement, index = self.get_statement(action, resource) - if not statement: + if not self.get_statement(action, resource)[0]: # Required resource is missing, add it to the statement with the required action. self.add_resource_to_statement(resource, index) def add_statement(self, statement, index=None): - if index: + if index is not None: self._policy_document['Statement'][index] = statement else: self._policy_document['Statement'].append(statement) @@ -231,7 +230,7 @@ def _add_bucket_to_queue_policy(self, policy_document): # A statement with 'sqs:sendmessage' action and some ARN condition already exists, # just append the bucket ARN to its condition. bucket_arn += ',arn:aws:s3:::' + self._bucket.name - statement['Condition']['ArnLike']['aws:SourceArn'] = bucket_arn.split() + statement['Condition']['ArnLike']['aws:SourceArn'] = bucket_arn.split(',') policy_document.add_statement(statement, index) self._queue.set_attributes(Attributes={'Policy': json.dumps(policy_document.get_policy())}) From 5ecaace7c92891598735d817d327466e8eeeba68 Mon Sep 17 00:00:00 2001 From: Michal Chomo Date: Mon, 19 Aug 2019 09:25:45 +0200 Subject: [PATCH 6/6] modify output, prevent duplicate statements, add /* line for bucket resource --- AWSscripts/SQS3script.py | 26 ++++++++++++++++++-------- 1 file changed, 18 insertions(+), 8 deletions(-) diff --git a/AWSscripts/SQS3script.py b/AWSscripts/SQS3script.py index 4534f45..a2e8051 100644 --- a/AWSscripts/SQS3script.py +++ b/AWSscripts/SQS3script.py @@ -181,7 +181,7 @@ def set_queue_policy(self): return pd = self.PolicyDocument(queue_policy) self._add_bucket_to_queue_policy(pd) - self._queue.add_permission(Label='AccountAccess', AWSAccountIds=[self._account_id], Actions=['*']) + self._add_account_access_to_queue_policy(pd) def set_bucket_notification(self): self._bucket.Notification().put( @@ -234,10 +234,19 @@ def _add_bucket_to_queue_policy(self, policy_document): policy_document.add_statement(statement, index) self._queue.set_attributes(Attributes={'Policy': json.dumps(policy_document.get_policy())}) + def _add_account_access_to_queue_policy(self, policy_document): + statement, _ = policy_document.get_statement( + action='sqs:*', resource=self._queue.attributes['QueueArn']) + if not statement: + self._queue.add_permission(Label='AccountAccess', AWSAccountIds=[self._account_id], Actions=['*']) + def _add_bucket_to_user_policy(self, policy_document): policy_document.add_statement_or_resource(["s3:ListBucket", "s3:GetObject", "s3:GetBucketLocation"], 'arn:aws:s3:::' + self._bucket.name, self.USER_POLICY_BUCKET_JSON % {'bucket_name': self._bucket.name}) + policy_document.add_statement_or_resource(["s3:ListBucket", "s3:GetObject", "s3:GetBucketLocation"], + 'arn:aws:s3:::' + self._bucket.name + '/*', + self.USER_POLICY_BUCKET_JSON % {'bucket_name': self._bucket.name}) def _add_queue_to_user_policy(self, policy_document): queue_arn = self._queue.attributes['QueueArn'] @@ -279,13 +288,15 @@ def get_queue(session, queue_name): sqs = session.resource('sqs') try: queue = sqs.get_queue_by_name(QueueName=queue_name) + print('Queue {} already exists'.format(queue_name)) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'AWS.SimpleQueueService.NonExistentQueue': - print('Queue {} not found, creating it.'.format(queue_name)) + print('Queue {} does not exist, creating it.'.format(queue_name)) queue = sqs.create_queue(QueueName=queue_name) + print('Queue url: {}'.format(queue.url)) else: raise e - print('Queue url: {}'.format(queue.url)) + print('Queue name\n{}'.format(queue_name)) return queue @@ -294,16 +305,18 @@ def get_user(session, user_name): try: user = iam.User(user_name) user.arn # This raises exception on non existent user. + print('IAM user {} already exists'.format(user_name)) + print('Please provide the access key and secret key for the IAM user {} in the form fields'.format(user_name)) except botocore.exceptions.ClientError as e: if e.response['Error']['Code'] == 'NoSuchEntity': - print("User {} does not exist, creating it") + print("IAM user {} does not exist, creating it".format(user_name)) user = iam.create_user(UserName=user_name) access_key_pair = user.create_access_key_pair() print("Access key for Loggly") print(access_key_pair.access_key_id) print("Secret key for Loggly") print(access_key_pair.secret_access_key) - print('Please provide the access key and secret key for the IAM user {} in the form fields'.format(user_name)) + print('Please save the above credentials') else: raise e return user @@ -318,11 +331,8 @@ def main(): user = get_user(session, args.user) aws = AWS(session, bucket, queue, user, args.acnumber) aws.set_queue_policy() - print('Queue policy has been set up') aws.set_bucket_notification() - print('Bucket notification has been set up') aws.set_user_policy() - print('User policy has been set up') except Exception as e: print(e) return 1