1# Copyright (c) 2010 Mitch Garnaat http://garnaat.org/
2# Copyright 2010 Google Inc.
3# Copyright (c) 2010, Eucalyptus Systems, Inc.
4# Copyright (c) 2011, Nexenta Systems Inc.
5# All rights reserved.
6#
7# Permission is hereby granted, free of charge, to any person obtaining a
8# copy of this software and associated documentation files (the
9# "Software"), to deal in the Software without restriction, including
10# without limitation the rights to use, copy, modify, merge, publish, dis-
11# tribute, sublicense, and/or sell copies of the Software, and to permit
12# persons to whom the Software is furnished to do so, subject to the fol-
13# lowing conditions:
14#
15# The above copyright notice and this permission notice shall be included
16# in all copies or substantial portions of the Software.
17#
18# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS
19# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABIL-
20# ITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT
21# SHALL THE AUTHOR BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY,
22# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
23# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS
24# IN THE SOFTWARE.
25"""
26This class encapsulates the provider-specific header differences.
27"""
28
29import os
30from boto.compat import six
31from datetime import datetime
32
33import boto
34from boto import config
35from boto.compat import expanduser
36from boto.pyami.config import Config
37from boto.gs.acl import ACL
38from boto.gs.acl import CannedACLStrings as CannedGSACLStrings
39from boto.s3.acl import CannedACLStrings as CannedS3ACLStrings
40from boto.s3.acl import Policy
41
42
43HEADER_PREFIX_KEY = 'header_prefix'
44METADATA_PREFIX_KEY = 'metadata_prefix'
45
46AWS_HEADER_PREFIX = 'x-amz-'
47GOOG_HEADER_PREFIX = 'x-goog-'
48
49ACL_HEADER_KEY = 'acl-header'
50AUTH_HEADER_KEY = 'auth-header'
51COPY_SOURCE_HEADER_KEY = 'copy-source-header'
52COPY_SOURCE_VERSION_ID_HEADER_KEY = 'copy-source-version-id-header'
53COPY_SOURCE_RANGE_HEADER_KEY = 'copy-source-range-header'
54DELETE_MARKER_HEADER_KEY = 'delete-marker-header'
55DATE_HEADER_KEY = 'date-header'
56METADATA_DIRECTIVE_HEADER_KEY = 'metadata-directive-header'
57RESUMABLE_UPLOAD_HEADER_KEY = 'resumable-upload-header'
58SECURITY_TOKEN_HEADER_KEY = 'security-token-header'
59STORAGE_CLASS_HEADER_KEY = 'storage-class'
60MFA_HEADER_KEY = 'mfa-header'
61SERVER_SIDE_ENCRYPTION_KEY = 'server-side-encryption-header'
62VERSION_ID_HEADER_KEY = 'version-id-header'
63RESTORE_HEADER_KEY = 'restore-header'
64
65STORAGE_COPY_ERROR = 'StorageCopyError'
66STORAGE_CREATE_ERROR = 'StorageCreateError'
67STORAGE_DATA_ERROR = 'StorageDataError'
68STORAGE_PERMISSIONS_ERROR = 'StoragePermissionsError'
69STORAGE_RESPONSE_ERROR = 'StorageResponseError'
70NO_CREDENTIALS_PROVIDED = object()
71
72
73class ProfileNotFoundError(ValueError):
74    pass
75
76
77class Provider(object):
78
79    CredentialMap = {
80        'aws':    ('aws_access_key_id', 'aws_secret_access_key',
81                   'aws_security_token', 'aws_profile'),
82        'google': ('gs_access_key_id',  'gs_secret_access_key',
83                   None, None),
84    }
85
86    AclClassMap = {
87        'aws':    Policy,
88        'google': ACL
89    }
90
91    CannedAclsMap = {
92        'aws':    CannedS3ACLStrings,
93        'google': CannedGSACLStrings
94    }
95
96    HostKeyMap = {
97        'aws':    's3',
98        'google': 'gs'
99    }
100
101    ChunkedTransferSupport = {
102        'aws':    False,
103        'google': True
104    }
105
106    MetadataServiceSupport = {
107        'aws': True,
108        'google': False
109    }
110
111    # If you update this map please make sure to put "None" for the
112    # right-hand-side for any headers that don't apply to a provider, rather
113    # than simply leaving that header out (which would cause KeyErrors).
114    HeaderInfoMap = {
115        'aws': {
116            HEADER_PREFIX_KEY: AWS_HEADER_PREFIX,
117            METADATA_PREFIX_KEY: AWS_HEADER_PREFIX + 'meta-',
118            ACL_HEADER_KEY: AWS_HEADER_PREFIX + 'acl',
119            AUTH_HEADER_KEY: 'AWS',
120            COPY_SOURCE_HEADER_KEY: AWS_HEADER_PREFIX + 'copy-source',
121            COPY_SOURCE_VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX +
122                                                'copy-source-version-id',
123            COPY_SOURCE_RANGE_HEADER_KEY: AWS_HEADER_PREFIX +
124                                           'copy-source-range',
125            DATE_HEADER_KEY: AWS_HEADER_PREFIX + 'date',
126            DELETE_MARKER_HEADER_KEY: AWS_HEADER_PREFIX + 'delete-marker',
127            METADATA_DIRECTIVE_HEADER_KEY: AWS_HEADER_PREFIX +
128                                            'metadata-directive',
129            RESUMABLE_UPLOAD_HEADER_KEY: None,
130            SECURITY_TOKEN_HEADER_KEY: AWS_HEADER_PREFIX + 'security-token',
131            SERVER_SIDE_ENCRYPTION_KEY: AWS_HEADER_PREFIX +
132                                         'server-side-encryption',
133            VERSION_ID_HEADER_KEY: AWS_HEADER_PREFIX + 'version-id',
134            STORAGE_CLASS_HEADER_KEY: AWS_HEADER_PREFIX + 'storage-class',
135            MFA_HEADER_KEY: AWS_HEADER_PREFIX + 'mfa',
136            RESTORE_HEADER_KEY: AWS_HEADER_PREFIX + 'restore',
137        },
138        'google': {
139            HEADER_PREFIX_KEY: GOOG_HEADER_PREFIX,
140            METADATA_PREFIX_KEY: GOOG_HEADER_PREFIX + 'meta-',
141            ACL_HEADER_KEY: GOOG_HEADER_PREFIX + 'acl',
142            AUTH_HEADER_KEY: 'GOOG1',
143            COPY_SOURCE_HEADER_KEY: GOOG_HEADER_PREFIX + 'copy-source',
144            COPY_SOURCE_VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX +
145                                                'copy-source-version-id',
146            COPY_SOURCE_RANGE_HEADER_KEY: None,
147            DATE_HEADER_KEY: GOOG_HEADER_PREFIX + 'date',
148            DELETE_MARKER_HEADER_KEY: GOOG_HEADER_PREFIX + 'delete-marker',
149            METADATA_DIRECTIVE_HEADER_KEY: GOOG_HEADER_PREFIX  +
150                                            'metadata-directive',
151            RESUMABLE_UPLOAD_HEADER_KEY: GOOG_HEADER_PREFIX + 'resumable',
152            SECURITY_TOKEN_HEADER_KEY: GOOG_HEADER_PREFIX + 'security-token',
153            SERVER_SIDE_ENCRYPTION_KEY: None,
154            # Note that this version header is not to be confused with
155            # the Google Cloud Storage 'x-goog-api-version' header.
156            VERSION_ID_HEADER_KEY: GOOG_HEADER_PREFIX + 'version-id',
157            STORAGE_CLASS_HEADER_KEY: None,
158            MFA_HEADER_KEY: None,
159            RESTORE_HEADER_KEY: None,
160        }
161    }
162
163    ErrorMap = {
164        'aws': {
165            STORAGE_COPY_ERROR: boto.exception.S3CopyError,
166            STORAGE_CREATE_ERROR: boto.exception.S3CreateError,
167            STORAGE_DATA_ERROR: boto.exception.S3DataError,
168            STORAGE_PERMISSIONS_ERROR: boto.exception.S3PermissionsError,
169            STORAGE_RESPONSE_ERROR: boto.exception.S3ResponseError,
170        },
171        'google': {
172            STORAGE_COPY_ERROR: boto.exception.GSCopyError,
173            STORAGE_CREATE_ERROR: boto.exception.GSCreateError,
174            STORAGE_DATA_ERROR: boto.exception.GSDataError,
175            STORAGE_PERMISSIONS_ERROR: boto.exception.GSPermissionsError,
176            STORAGE_RESPONSE_ERROR: boto.exception.GSResponseError,
177        }
178    }
179
180    def __init__(self, name, access_key=None, secret_key=None,
181                 security_token=None, profile_name=None):
182        self.host = None
183        self.port = None
184        self.host_header = None
185        self.access_key = access_key
186        self.secret_key = secret_key
187        self.security_token = security_token
188        self.profile_name = profile_name
189        self.name = name
190        self.acl_class = self.AclClassMap[self.name]
191        self.canned_acls = self.CannedAclsMap[self.name]
192        self._credential_expiry_time = None
193
194        # Load shared credentials file if it exists
195        shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials')
196        self.shared_credentials = Config(do_load=False)
197        if os.path.isfile(shared_path):
198            self.shared_credentials.load_from_path(shared_path)
199
200        self.get_credentials(access_key, secret_key, security_token, profile_name)
201        self.configure_headers()
202        self.configure_errors()
203
204        # Allow config file to override default host and port.
205        host_opt_name = '%s_host' % self.HostKeyMap[self.name]
206        if config.has_option('Credentials', host_opt_name):
207            self.host = config.get('Credentials', host_opt_name)
208        port_opt_name = '%s_port' % self.HostKeyMap[self.name]
209        if config.has_option('Credentials', port_opt_name):
210            self.port = config.getint('Credentials', port_opt_name)
211        host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
212        if config.has_option('Credentials', host_header_opt_name):
213            self.host_header = config.get('Credentials', host_header_opt_name)
214
215    def get_access_key(self):
216        if self._credentials_need_refresh():
217            self._populate_keys_from_metadata_server()
218        return self._access_key
219
220    def set_access_key(self, value):
221        self._access_key = value
222
223    access_key = property(get_access_key, set_access_key)
224
225    def get_secret_key(self):
226        if self._credentials_need_refresh():
227            self._populate_keys_from_metadata_server()
228        return self._secret_key
229
230    def set_secret_key(self, value):
231        self._secret_key = value
232
233    secret_key = property(get_secret_key, set_secret_key)
234
235    def get_security_token(self):
236        if self._credentials_need_refresh():
237            self._populate_keys_from_metadata_server()
238        return self._security_token
239
240    def set_security_token(self, value):
241        self._security_token = value
242
243    security_token = property(get_security_token, set_security_token)
244
245    def _credentials_need_refresh(self):
246        if self._credential_expiry_time is None:
247            return False
248        else:
249            # The credentials should be refreshed if they're going to expire
250            # in less than 5 minutes.
251            delta = self._credential_expiry_time - datetime.utcnow()
252            # python2.6 does not have timedelta.total_seconds() so we have
253            # to calculate this ourselves.  This is straight from the
254            # datetime docs.
255            seconds_left = (
256                (delta.microseconds + (delta.seconds + delta.days * 24 * 3600)
257                 * 10 ** 6) / 10 ** 6)
258            if seconds_left < (5 * 60):
259                boto.log.debug("Credentials need to be refreshed.")
260                return True
261            else:
262                return False
263
264    def get_credentials(self, access_key=None, secret_key=None,
265                        security_token=None, profile_name=None):
266        access_key_name, secret_key_name, security_token_name, \
267            profile_name_name = self.CredentialMap[self.name]
268
269        # Load profile from shared environment variable if it was not
270        # already passed in and the environment variable exists
271        if profile_name is None and profile_name_name is not None and \
272           profile_name_name.upper() in os.environ:
273            profile_name = os.environ[profile_name_name.upper()]
274
275        shared = self.shared_credentials
276
277        if access_key is not None:
278            self.access_key = access_key
279            boto.log.debug("Using access key provided by client.")
280        elif access_key_name.upper() in os.environ:
281            self.access_key = os.environ[access_key_name.upper()]
282            boto.log.debug("Using access key found in environment variable.")
283        elif profile_name is not None:
284            if shared.has_option(profile_name, access_key_name):
285                self.access_key = shared.get(profile_name, access_key_name)
286                boto.log.debug("Using access key found in shared credential "
287                               "file for profile %s." % profile_name)
288            elif config.has_option("profile %s" % profile_name,
289                                   access_key_name):
290                self.access_key = config.get("profile %s" % profile_name,
291                                             access_key_name)
292                boto.log.debug("Using access key found in config file: "
293                               "profile %s." % profile_name)
294            else:
295                raise ProfileNotFoundError('Profile "%s" not found!' %
296                                           profile_name)
297        elif shared.has_option('default', access_key_name):
298            self.access_key = shared.get('default', access_key_name)
299            boto.log.debug("Using access key found in shared credential file.")
300        elif config.has_option('Credentials', access_key_name):
301            self.access_key = config.get('Credentials', access_key_name)
302            boto.log.debug("Using access key found in config file.")
303
304        if secret_key is not None:
305            self.secret_key = secret_key
306            boto.log.debug("Using secret key provided by client.")
307        elif secret_key_name.upper() in os.environ:
308            self.secret_key = os.environ[secret_key_name.upper()]
309            boto.log.debug("Using secret key found in environment variable.")
310        elif profile_name is not None:
311            if shared.has_option(profile_name, secret_key_name):
312                self.secret_key = shared.get(profile_name, secret_key_name)
313                boto.log.debug("Using secret key found in shared credential "
314                               "file for profile %s." % profile_name)
315            elif config.has_option("profile %s" % profile_name, secret_key_name):
316                self.secret_key = config.get("profile %s" % profile_name,
317                                             secret_key_name)
318                boto.log.debug("Using secret key found in config file: "
319                               "profile %s." % profile_name)
320            else:
321                raise ProfileNotFoundError('Profile "%s" not found!' %
322                                           profile_name)
323        elif shared.has_option('default', secret_key_name):
324            self.secret_key = shared.get('default', secret_key_name)
325            boto.log.debug("Using secret key found in shared credential file.")
326        elif config.has_option('Credentials', secret_key_name):
327            self.secret_key = config.get('Credentials', secret_key_name)
328            boto.log.debug("Using secret key found in config file.")
329        elif config.has_option('Credentials', 'keyring'):
330            keyring_name = config.get('Credentials', 'keyring')
331            try:
332                import keyring
333            except ImportError:
334                boto.log.error("The keyring module could not be imported. "
335                               "For keyring support, install the keyring "
336                               "module.")
337                raise
338            self.secret_key = keyring.get_password(
339                keyring_name, self.access_key)
340            boto.log.debug("Using secret key found in keyring.")
341
342        if security_token is not None:
343            self.security_token = security_token
344            boto.log.debug("Using security token provided by client.")
345        elif ((security_token_name is not None) and
346              (access_key is None) and (secret_key is None)):
347            # Only provide a token from the environment/config if the
348            # caller did not specify a key and secret.  Otherwise an
349            # environment/config token could be paired with a
350            # different set of credentials provided by the caller
351            if security_token_name.upper() in os.environ:
352                self.security_token = os.environ[security_token_name.upper()]
353                boto.log.debug("Using security token found in environment"
354                               " variable.")
355            elif shared.has_option(profile_name or 'default',
356                                   security_token_name):
357                self.security_token = shared.get(profile_name or 'default',
358                                                 security_token_name)
359                boto.log.debug("Using security token found in shared "
360                               "credential file.")
361            elif profile_name is not None:
362                if config.has_option("profile %s" % profile_name,
363                                     security_token_name):
364                    boto.log.debug("config has option")
365                    self.security_token = config.get("profile %s" % profile_name,
366                                                     security_token_name)
367                    boto.log.debug("Using security token found in config file: "
368                                   "profile %s." % profile_name)
369            elif config.has_option('Credentials', security_token_name):
370                self.security_token = config.get('Credentials',
371                                                 security_token_name)
372                boto.log.debug("Using security token found in config file.")
373
374        if ((self._access_key is None or self._secret_key is None) and
375                self.MetadataServiceSupport[self.name]):
376            self._populate_keys_from_metadata_server()
377        self._secret_key = self._convert_key_to_str(self._secret_key)
378
379    def _populate_keys_from_metadata_server(self):
380        # get_instance_metadata is imported here because of a circular
381        # dependency.
382        boto.log.debug("Retrieving credentials from metadata server.")
383        from boto.utils import get_instance_metadata
384        timeout = config.getfloat('Boto', 'metadata_service_timeout', 1.0)
385        attempts = config.getint('Boto', 'metadata_service_num_attempts', 1)
386        # The num_retries arg is actually the total number of attempts made,
387        # so the config options is named *_num_attempts to make this more
388        # clear to users.
389        metadata = get_instance_metadata(
390            timeout=timeout, num_retries=attempts,
391            data='meta-data/iam/security-credentials/')
392        if metadata:
393            # I'm assuming there's only one role on the instance profile.
394            security = list(metadata.values())[0]
395            self._access_key = security['AccessKeyId']
396            self._secret_key = self._convert_key_to_str(security['SecretAccessKey'])
397            self._security_token = security['Token']
398            expires_at = security['Expiration']
399            self._credential_expiry_time = datetime.strptime(
400                expires_at, "%Y-%m-%dT%H:%M:%SZ")
401            boto.log.debug("Retrieved credentials will expire in %s at: %s",
402                           self._credential_expiry_time - datetime.now(), expires_at)
403
404    def _convert_key_to_str(self, key):
405        if isinstance(key, six.text_type):
406            # the secret key must be bytes and not unicode to work
407            #  properly with hmac.new (see http://bugs.python.org/issue5285)
408            return str(key)
409        return key
410
411    def configure_headers(self):
412        header_info_map = self.HeaderInfoMap[self.name]
413        self.metadata_prefix = header_info_map[METADATA_PREFIX_KEY]
414        self.header_prefix = header_info_map[HEADER_PREFIX_KEY]
415        self.acl_header = header_info_map[ACL_HEADER_KEY]
416        self.auth_header = header_info_map[AUTH_HEADER_KEY]
417        self.copy_source_header = header_info_map[COPY_SOURCE_HEADER_KEY]
418        self.copy_source_version_id = header_info_map[
419            COPY_SOURCE_VERSION_ID_HEADER_KEY]
420        self.copy_source_range_header = header_info_map[
421            COPY_SOURCE_RANGE_HEADER_KEY]
422        self.date_header = header_info_map[DATE_HEADER_KEY]
423        self.delete_marker = header_info_map[DELETE_MARKER_HEADER_KEY]
424        self.metadata_directive_header = (
425            header_info_map[METADATA_DIRECTIVE_HEADER_KEY])
426        self.security_token_header = header_info_map[SECURITY_TOKEN_HEADER_KEY]
427        self.resumable_upload_header = (
428            header_info_map[RESUMABLE_UPLOAD_HEADER_KEY])
429        self.server_side_encryption_header = header_info_map[SERVER_SIDE_ENCRYPTION_KEY]
430        self.storage_class_header = header_info_map[STORAGE_CLASS_HEADER_KEY]
431        self.version_id = header_info_map[VERSION_ID_HEADER_KEY]
432        self.mfa_header = header_info_map[MFA_HEADER_KEY]
433        self.restore_header = header_info_map[RESTORE_HEADER_KEY]
434
435    def configure_errors(self):
436        error_map = self.ErrorMap[self.name]
437        self.storage_copy_error = error_map[STORAGE_COPY_ERROR]
438        self.storage_create_error = error_map[STORAGE_CREATE_ERROR]
439        self.storage_data_error = error_map[STORAGE_DATA_ERROR]
440        self.storage_permissions_error = error_map[STORAGE_PERMISSIONS_ERROR]
441        self.storage_response_error = error_map[STORAGE_RESPONSE_ERROR]
442
443    def get_provider_name(self):
444        return self.HostKeyMap[self.name]
445
446    def supports_chunked_transfer(self):
447        return self.ChunkedTransferSupport[self.name]
448
449
450# Static utility method for getting default Provider.
451def get_default():
452    return Provider('aws')
453