Skip to content

Commit 6de6289

Browse files
committed
ENH: make it possible to specify "host" option for boto.connect_s3
While trying to crawl dandiarchive bucket with authentication, to fetch also files which are not publicly available, I have ran into <Error><Code>InvalidRequest</Code><Message>The authorization mechanism you have provided is not supported. Please use AWS4-HMAC-SHA256.< for which discussion was ongoing in 2017: jschneier/django-storages#28 . A workaround which worked for me was to specify host option to boto.connect_s3 to point to the specific region. So with this fix now it would be possible to use it in the provider configuration, e.g. [provider:dandi-s3] url_re = s3://dandiarchive($|/.*) credential = dandi-s3-backup authentication_type = aws-s3 aws-s3_host = s3.us-east-2.amazonaws.com There might be other options we might want to add later on, so I did not store host in the attribute, but right within the dictionary of optional kwargs for connect_s3.
1 parent 43c671d commit 6de6289

File tree

1 file changed

+14
-3
lines changed

1 file changed

+14
-3
lines changed

datalad/downloaders/s3.py

Lines changed: 14 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -46,10 +46,21 @@ class S3Authenticator(Authenticator):
4646
allows_anonymous = True
4747
DEFAULT_CREDENTIAL_TYPE = 'aws-s3'
4848

49-
def __init__(self, *args, **kwargs):
49+
def __init__(self, *args, host=None, **kwargs):
50+
"""
51+
52+
Parameters
53+
----------
54+
host: str, optional
55+
In some cases it is necessary to provide host to connect to. Passed
56+
to boto.connect_s3
57+
"""
5058
super(S3Authenticator, self).__init__(*args, **kwargs)
5159
self.connection = None
5260
self.bucket = None
61+
self._conn_kwargs = {}
62+
if host:
63+
self._conn_kwargs['host'] = host
5364

5465
def authenticate(self, bucket_name, credential, cache=True):
5566
"""Authenticates to the specified bucket using provided credentials
@@ -72,7 +83,7 @@ def authenticate(self, bucket_name, credential, cache=True):
7283
# credential might contain 'session' token as well
7384
# which could be provided as security_token=<token>.,
7485
# see http://stackoverflow.com/questions/7673840/is-there-a-way-to-create-a-s3-connection-with-a-sessions-token
75-
conn_kwargs = {}
86+
conn_kwargs = self._conn_kwargs.copy()
7687
if bucket_name.lower() != bucket_name:
7788
# per http://stackoverflow.com/a/19089045/1265472
7889
conn_kwargs['calling_format'] = OrdinaryCallingFormat()
@@ -87,7 +98,7 @@ def authenticate(self, bucket_name, credential, cache=True):
8798
conn_args = []
8899
conn_kwargs['anon'] = True
89100
if '.' in bucket_name:
90-
conn_kwargs['calling_format']=OrdinaryCallingFormat()
101+
conn_kwargs['calling_format'] = OrdinaryCallingFormat()
91102

92103
lgr.info(
93104
"S3 session: Connecting to the bucket %s %s", bucket_name, conn_kind

0 commit comments

Comments
 (0)