1# -*- coding: utf-8 -*-
2# Copyright 2014 Google Inc. All Rights Reserved.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Utility module for translating XML API objects to/from JSON objects."""
16
17from __future__ import absolute_import
18
19import datetime
20import json
21import re
22import textwrap
23import xml.etree.ElementTree
24
25from apitools.base.py import encoding
26import boto
27from boto.gs.acl import ACL
28from boto.gs.acl import ALL_AUTHENTICATED_USERS
29from boto.gs.acl import ALL_USERS
30from boto.gs.acl import Entries
31from boto.gs.acl import Entry
32from boto.gs.acl import GROUP_BY_DOMAIN
33from boto.gs.acl import GROUP_BY_EMAIL
34from boto.gs.acl import GROUP_BY_ID
35from boto.gs.acl import USER_BY_EMAIL
36from boto.gs.acl import USER_BY_ID
37
38from gslib.cloud_api import ArgumentException
39from gslib.cloud_api import BucketNotFoundException
40from gslib.cloud_api import NotFoundException
41from gslib.cloud_api import Preconditions
42from gslib.exception import CommandException
43from gslib.third_party.storage_apitools import storage_v1_messages as apitools_messages
44
45# In Python 2.6, ElementTree raises ExpatError instead of ParseError.
46# pylint: disable=g-import-not-at-top
47try:
48  from xml.etree.ElementTree import ParseError as XmlParseError
49except ImportError:
50  from xml.parsers.expat import ExpatError as XmlParseError
51
52CACHE_CONTROL_REGEX = re.compile(r'^cache-control', re.I)
53CONTENT_DISPOSITION_REGEX = re.compile(r'^content-disposition', re.I)
54CONTENT_ENCODING_REGEX = re.compile(r'^content-encoding', re.I)
55CONTENT_LANGUAGE_REGEX = re.compile(r'^content-language', re.I)
56CONTENT_MD5_REGEX = re.compile(r'^content-md5', re.I)
57CONTENT_TYPE_REGEX = re.compile(r'^content-type', re.I)
58GOOG_API_VERSION_REGEX = re.compile(r'^x-goog-api-version', re.I)
59GOOG_GENERATION_MATCH_REGEX = re.compile(r'^x-goog-if-generation-match', re.I)
60GOOG_METAGENERATION_MATCH_REGEX = re.compile(
61    r'^x-goog-if-metageneration-match', re.I)
62CUSTOM_GOOG_METADATA_REGEX = re.compile(r'^x-goog-meta-(?P<header_key>.*)',
63                                        re.I)
64CUSTOM_AMZ_METADATA_REGEX = re.compile(r'^x-amz-meta-(?P<header_key>.*)', re.I)
65CUSTOM_AMZ_HEADER_REGEX = re.compile(r'^x-amz-(?P<header_key>.*)', re.I)
66
67# gsutil-specific GUIDs for marking special metadata for S3 compatibility.
68S3_ACL_MARKER_GUID = '3b89a6b5-b55a-4900-8c44-0b0a2f5eab43-s3-AclMarker'
69S3_DELETE_MARKER_GUID = 'eadeeee8-fa8c-49bb-8a7d-0362215932d8-s3-DeleteMarker'
70S3_MARKER_GUIDS = [S3_ACL_MARKER_GUID, S3_DELETE_MARKER_GUID]
71# This distinguishes S3 custom headers from S3 metadata on objects.
72S3_HEADER_PREFIX = 'custom-amz-header'
73
74DEFAULT_CONTENT_TYPE = 'application/octet-stream'
75
76# Because CORS is just a list in apitools, we need special handling or blank
77# CORS lists will get sent with other configuration commands such as lifecycle,
78# which would cause CORS configuration to be unintentionally removed.
79# Protorpc defaults list values to an empty list, and won't allow us to set the
80# value to None like other configuration fields, so there is no way to
81# distinguish the default value from when we actually want to remove the CORS
82# configuration.  To work around this, we create a dummy CORS entry that
83# signifies that we should nullify the CORS configuration.
84# A value of [] means don't modify the CORS configuration.
85# A value of REMOVE_CORS_CONFIG means remove the CORS configuration.
86REMOVE_CORS_CONFIG = [apitools_messages.Bucket.CorsValueListEntry(
87    maxAgeSeconds=-1, method=['REMOVE_CORS_CONFIG'])]
88
89# Similar to CORS above, we need a sentinel value allowing us to specify
90# when a default object ACL should be private (containing no entries).
91# A defaultObjectAcl value of [] means don't modify the default object ACL.
92# A value of [PRIVATE_DEFAULT_OBJ_ACL] means create an empty/private default
93# object ACL.
94PRIVATE_DEFAULT_OBJ_ACL = apitools_messages.ObjectAccessControl(
95    id='PRIVATE_DEFAULT_OBJ_ACL')
96
97
98def ObjectMetadataFromHeaders(headers):
99  """Creates object metadata according to the provided headers.
100
101  gsutil -h allows specifiying various headers (originally intended
102  to be passed to boto in gsutil v3).  For the JSON API to be compatible with
103  this option, we need to parse these headers into gsutil_api Object fields.
104
105  Args:
106    headers: Dict of headers passed via gsutil -h
107
108  Raises:
109    ArgumentException if an invalid header is encountered.
110
111  Returns:
112    apitools Object with relevant fields populated from headers.
113  """
114  obj_metadata = apitools_messages.Object()
115  for header, value in headers.items():
116    if CACHE_CONTROL_REGEX.match(header):
117      obj_metadata.cacheControl = value.strip()
118    elif CONTENT_DISPOSITION_REGEX.match(header):
119      obj_metadata.contentDisposition = value.strip()
120    elif CONTENT_ENCODING_REGEX.match(header):
121      obj_metadata.contentEncoding = value.strip()
122    elif CONTENT_MD5_REGEX.match(header):
123      obj_metadata.md5Hash = value.strip()
124    elif CONTENT_LANGUAGE_REGEX.match(header):
125      obj_metadata.contentLanguage = value.strip()
126    elif CONTENT_TYPE_REGEX.match(header):
127      if not value:
128        obj_metadata.contentType = DEFAULT_CONTENT_TYPE
129      else:
130        obj_metadata.contentType = value.strip()
131    elif GOOG_API_VERSION_REGEX.match(header):
132      # API version is only relevant for XML, ignore and rely on the XML API
133      # to add the appropriate version.
134      continue
135    elif GOOG_GENERATION_MATCH_REGEX.match(header):
136      # Preconditions are handled elsewhere, but allow these headers through.
137      continue
138    elif GOOG_METAGENERATION_MATCH_REGEX.match(header):
139      # Preconditions are handled elsewhere, but allow these headers through.
140      continue
141    else:
142      custom_goog_metadata_match = CUSTOM_GOOG_METADATA_REGEX.match(header)
143      custom_amz_metadata_match = CUSTOM_AMZ_METADATA_REGEX.match(header)
144      custom_amz_header_match = CUSTOM_AMZ_HEADER_REGEX.match(header)
145      header_key = None
146      if custom_goog_metadata_match:
147        header_key = custom_goog_metadata_match.group('header_key')
148      elif custom_amz_metadata_match:
149        header_key = custom_amz_metadata_match.group('header_key')
150      elif custom_amz_header_match:
151        # If we got here we are guaranteed by the prior statement that this is
152        # not an x-amz-meta- header.
153        header_key = (S3_HEADER_PREFIX +
154                      custom_amz_header_match.group('header_key'))
155      if header_key:
156        if header_key.lower() == 'x-goog-content-language':
157          # Work around content-language being inserted into custom metadata.
158          continue
159        if not obj_metadata.metadata:
160          obj_metadata.metadata = apitools_messages.Object.MetadataValue()
161        if not obj_metadata.metadata.additionalProperties:
162          obj_metadata.metadata.additionalProperties = []
163        obj_metadata.metadata.additionalProperties.append(
164            apitools_messages.Object.MetadataValue.AdditionalProperty(
165                key=header_key, value=value))
166      else:
167        raise ArgumentException(
168            'Invalid header specifed: %s:%s' % (header, value))
169  return obj_metadata
170
171
172def HeadersFromObjectMetadata(dst_obj_metadata, provider):
173  """Creates a header dictionary based on existing object metadata.
174
175  Args:
176    dst_obj_metadata: Object metadata to create the headers from.
177    provider: Provider string ('gs' or 's3')
178
179  Returns:
180    Headers dictionary.
181  """
182  headers = {}
183  if not dst_obj_metadata:
184    return
185  # Metadata values of '' mean suppress/remove this header.
186  if dst_obj_metadata.cacheControl is not None:
187    if not dst_obj_metadata.cacheControl:
188      headers['cache-control'] = None
189    else:
190      headers['cache-control'] = dst_obj_metadata.cacheControl.strip()
191  if dst_obj_metadata.contentDisposition:
192    if not dst_obj_metadata.contentDisposition:
193      headers['content-disposition'] = None
194    else:
195      headers['content-disposition'] = (
196          dst_obj_metadata.contentDisposition.strip())
197  if dst_obj_metadata.contentEncoding:
198    if not dst_obj_metadata.contentEncoding:
199      headers['content-encoding'] = None
200    else:
201      headers['content-encoding'] = dst_obj_metadata.contentEncoding.strip()
202  if dst_obj_metadata.contentLanguage:
203    if not dst_obj_metadata.contentLanguage:
204      headers['content-language'] = None
205    else:
206      headers['content-language'] = dst_obj_metadata.contentLanguage.strip()
207  if dst_obj_metadata.md5Hash:
208    if not dst_obj_metadata.md5Hash:
209      headers['Content-MD5'] = None
210    else:
211      headers['Content-MD5'] = dst_obj_metadata.md5Hash.strip()
212  if dst_obj_metadata.contentType is not None:
213    if not dst_obj_metadata.contentType:
214      headers['content-type'] = None
215    else:
216      headers['content-type'] = dst_obj_metadata.contentType.strip()
217  if (dst_obj_metadata.metadata and
218      dst_obj_metadata.metadata.additionalProperties):
219    for additional_property in dst_obj_metadata.metadata.additionalProperties:
220      # Work around content-language being inserted into custom metadata by
221      # the XML API.
222      if additional_property.key == 'content-language':
223        continue
224      # Don't translate special metadata markers.
225      if additional_property.key in S3_MARKER_GUIDS:
226        continue
227      if provider == 'gs':
228        header_name = 'x-goog-meta-' + additional_property.key
229      elif provider == 's3':
230        if additional_property.key.startswith(S3_HEADER_PREFIX):
231          header_name = ('x-amz-' +
232                         additional_property.key[len(S3_HEADER_PREFIX):])
233        else:
234          header_name = 'x-amz-meta-' + additional_property.key
235      else:
236        raise ArgumentException('Invalid provider specified: %s' % provider)
237      if (additional_property.value is not None and
238          not additional_property.value):
239        headers[header_name] = None
240      else:
241        headers[header_name] = additional_property.value
242  return headers
243
244
245def CopyObjectMetadata(src_obj_metadata, dst_obj_metadata, override=False):
246  """Copies metadata from src_obj_metadata to dst_obj_metadata.
247
248  Args:
249    src_obj_metadata: Metadata from source object
250    dst_obj_metadata: Initialized metadata for destination object
251    override: If true, will overwrite metadata in destination object.
252              If false, only writes metadata for values that don't already
253              exist.
254  """
255  if override or not dst_obj_metadata.cacheControl:
256    dst_obj_metadata.cacheControl = src_obj_metadata.cacheControl
257  if override or not dst_obj_metadata.contentDisposition:
258    dst_obj_metadata.contentDisposition = src_obj_metadata.contentDisposition
259  if override or not dst_obj_metadata.contentEncoding:
260    dst_obj_metadata.contentEncoding = src_obj_metadata.contentEncoding
261  if override or not dst_obj_metadata.contentLanguage:
262    dst_obj_metadata.contentLanguage = src_obj_metadata.contentLanguage
263  if override or not dst_obj_metadata.contentType:
264    dst_obj_metadata.contentType = src_obj_metadata.contentType
265  if override or not dst_obj_metadata.md5Hash:
266    dst_obj_metadata.md5Hash = src_obj_metadata.md5Hash
267
268  # TODO: Apitools should ideally treat metadata like a real dictionary instead
269  # of a list of key/value pairs (with an O(N^2) lookup).  In practice the
270  # number of values is typically small enough not to matter.
271  # Work around this by creating our own dictionary.
272  if (src_obj_metadata.metadata and
273      src_obj_metadata.metadata.additionalProperties):
274    if not dst_obj_metadata.metadata:
275      dst_obj_metadata.metadata = apitools_messages.Object.MetadataValue()
276    if not dst_obj_metadata.metadata.additionalProperties:
277      dst_obj_metadata.metadata.additionalProperties = []
278    dst_metadata_dict = {}
279    for dst_prop in dst_obj_metadata.metadata.additionalProperties:
280      dst_metadata_dict[dst_prop.key] = dst_prop.value
281    for src_prop in src_obj_metadata.metadata.additionalProperties:
282      if src_prop.key in dst_metadata_dict:
283        if override:
284          # Metadata values of '' mean suppress/remove this header.
285          if src_prop.value is not None and not src_prop.value:
286            dst_metadata_dict[src_prop.key] = None
287          else:
288            dst_metadata_dict[src_prop.key] = src_prop.value
289      else:
290        dst_metadata_dict[src_prop.key] = src_prop.value
291    # Rewrite the list with our updated dict.
292    dst_obj_metadata.metadata.additionalProperties = []
293    for k, v in dst_metadata_dict.iteritems():
294      dst_obj_metadata.metadata.additionalProperties.append(
295          apitools_messages.Object.MetadataValue.AdditionalProperty(key=k,
296                                                                    value=v))
297
298
299def PreconditionsFromHeaders(headers):
300  """Creates bucket or object preconditions acccording to the provided headers.
301
302  Args:
303    headers: Dict of headers passed via gsutil -h
304
305  Returns:
306    gsutil Cloud API Preconditions object fields populated from headers, or None
307    if no precondition headers are present.
308  """
309  return_preconditions = Preconditions()
310  try:
311    for header, value in headers.items():
312      if GOOG_GENERATION_MATCH_REGEX.match(header):
313        return_preconditions.gen_match = long(value)
314      if GOOG_METAGENERATION_MATCH_REGEX.match(header):
315        return_preconditions.meta_gen_match = long(value)
316  except ValueError, _:
317    raise ArgumentException('Invalid precondition header specified. '
318                            'x-goog-if-generation-match and '
319                            'x-goog-if-metageneration match must be specified '
320                            'with a positive integer value.')
321  return return_preconditions
322
323
324def CreateNotFoundExceptionForObjectWrite(
325    dst_provider, dst_bucket_name, src_provider=None,
326    src_bucket_name=None, src_object_name=None, src_generation=None):
327  """Creates a NotFoundException for an object upload or copy.
328
329  This is necessary because 404s don't necessarily specify which resource
330  does not exist.
331
332  Args:
333    dst_provider: String abbreviation of destination provider, e.g., 'gs'.
334    dst_bucket_name: Destination bucket name for the write operation.
335    src_provider: String abbreviation of source provider, i.e. 'gs', if any.
336    src_bucket_name: Source bucket name, if any (for the copy case).
337    src_object_name: Source object name, if any (for the copy case).
338    src_generation: Source object generation, if any (for the copy case).
339
340  Returns:
341    NotFoundException with appropriate message.
342  """
343  dst_url_string = '%s://%s' % (dst_provider, dst_bucket_name)
344  if src_bucket_name and src_object_name:
345    src_url_string = '%s://%s/%s' % (src_provider, src_bucket_name,
346                                     src_object_name)
347    if src_generation:
348      src_url_string += '#%s' % str(src_generation)
349    return NotFoundException(
350        'The source object %s or the destination bucket %s does not exist.' %
351        (src_url_string, dst_url_string))
352
353  return NotFoundException(
354      'The destination bucket %s does not exist or the write to the '
355      'destination must be restarted' % dst_url_string)
356
357
358def CreateBucketNotFoundException(code, provider, bucket_name):
359  return BucketNotFoundException('%s://%s bucket does not exist.' %
360                                 (provider, bucket_name), bucket_name,
361                                 status=code)
362
363
364def CreateObjectNotFoundException(code, provider, bucket_name, object_name,
365                                  generation=None):
366  uri_string = '%s://%s/%s' % (provider, bucket_name, object_name)
367  if generation:
368    uri_string += '#%s' % str(generation)
369  return NotFoundException('%s does not exist.' % uri_string, status=code)
370
371
372def EncodeStringAsLong(string_to_convert):
373  """Encodes an ASCII string as a python long.
374
375  This is used for modeling S3 version_id's as apitools generation.  Because
376  python longs can be arbitrarily large, this works.
377
378  Args:
379    string_to_convert: ASCII string to convert to a long.
380
381  Returns:
382    Long that represents the input string.
383  """
384  return long(string_to_convert.encode('hex'), 16)
385
386
387def _DecodeLongAsString(long_to_convert):
388  """Decodes an encoded python long into an ASCII string.
389
390  This is used for modeling S3 version_id's as apitools generation.
391
392  Args:
393    long_to_convert: long to convert to ASCII string. If this is already a
394                     string, it is simply returned.
395
396  Returns:
397    String decoded from the input long.
398  """
399  if isinstance(long_to_convert, basestring):
400    # Already converted.
401    return long_to_convert
402  return hex(long_to_convert)[2:-1].decode('hex')
403
404
405def GenerationFromUrlAndString(url, generation):
406  """Decodes a generation from a StorageURL and a generation string.
407
408  This is used to represent gs and s3 versioning.
409
410  Args:
411    url: StorageUrl representing the object.
412    generation: Long or string representing the object's generation or
413                version.
414
415  Returns:
416    Valid generation string for use in URLs.
417  """
418  if url.scheme == 's3' and generation:
419    return _DecodeLongAsString(generation)
420  return generation
421
422
423def CheckForXmlConfigurationAndRaise(config_type_string, json_txt):
424  """Checks a JSON parse exception for provided XML configuration."""
425  try:
426    xml.etree.ElementTree.fromstring(str(json_txt))
427    raise ArgumentException('\n'.join(textwrap.wrap(
428        'XML {0} data provided; Google Cloud Storage {0} configuration '
429        'now uses JSON format. To convert your {0}, set the desired XML '
430        'ACL using \'gsutil {1} set ...\' with gsutil version 3.x. Then '
431        'use \'gsutil {1} get ...\' with gsutil version 4 or greater to '
432        'get the corresponding JSON {0}.'.format(config_type_string,
433                                                 config_type_string.lower()))))
434  except XmlParseError:
435    pass
436  raise ArgumentException('JSON %s data could not be loaded '
437                          'from: %s' % (config_type_string, json_txt))
438
439
440class LifecycleTranslation(object):
441  """Functions for converting between various lifecycle formats.
442
443    This class handles conversation to and from Boto Cors objects, JSON text,
444    and apitools Message objects.
445  """
446
447  @classmethod
448  def BotoLifecycleFromMessage(cls, lifecycle_message):
449    """Translates an apitools message to a boto lifecycle object."""
450    boto_lifecycle = boto.gs.lifecycle.LifecycleConfig()
451    if lifecycle_message:
452      for rule_message in lifecycle_message.rule:
453        boto_rule = boto.gs.lifecycle.Rule()
454        if (rule_message.action and rule_message.action.type and
455            rule_message.action.type.lower() == 'delete'):
456          boto_rule.action = boto.gs.lifecycle.DELETE
457        if rule_message.condition:
458          if rule_message.condition.age:
459            boto_rule.conditions[boto.gs.lifecycle.AGE] = (
460                str(rule_message.condition.age))
461          if rule_message.condition.createdBefore:
462            boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE] = (
463                str(rule_message.condition.createdBefore))
464          if rule_message.condition.isLive:
465            boto_rule.conditions[boto.gs.lifecycle.IS_LIVE] = (
466                str(rule_message.condition.isLive))
467          if rule_message.condition.numNewerVersions:
468            boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS] = (
469                str(rule_message.condition.numNewerVersions))
470        boto_lifecycle.append(boto_rule)
471    return boto_lifecycle
472
473  @classmethod
474  def BotoLifecycleToMessage(cls, boto_lifecycle):
475    """Translates a boto lifecycle object to an apitools message."""
476    lifecycle_message = None
477    if boto_lifecycle:
478      lifecycle_message = apitools_messages.Bucket.LifecycleValue()
479      for boto_rule in boto_lifecycle:
480        lifecycle_rule = (
481            apitools_messages.Bucket.LifecycleValue.RuleValueListEntry())
482        lifecycle_rule.condition = (apitools_messages.Bucket.LifecycleValue.
483                                    RuleValueListEntry.ConditionValue())
484        if boto_rule.action and boto_rule.action == boto.gs.lifecycle.DELETE:
485          lifecycle_rule.action = (apitools_messages.Bucket.LifecycleValue.
486                                   RuleValueListEntry.ActionValue(
487                                       type='Delete'))
488        if boto.gs.lifecycle.AGE in boto_rule.conditions:
489          lifecycle_rule.condition.age = int(
490              boto_rule.conditions[boto.gs.lifecycle.AGE])
491        if boto.gs.lifecycle.CREATED_BEFORE in boto_rule.conditions:
492          lifecycle_rule.condition.createdBefore = (
493              LifecycleTranslation.TranslateBotoLifecycleTimestamp(
494                  boto_rule.conditions[boto.gs.lifecycle.CREATED_BEFORE]))
495        if boto.gs.lifecycle.IS_LIVE in boto_rule.conditions:
496          lifecycle_rule.condition.isLive = bool(
497              boto_rule.conditions[boto.gs.lifecycle.IS_LIVE])
498        if boto.gs.lifecycle.NUM_NEWER_VERSIONS in boto_rule.conditions:
499          lifecycle_rule.condition.numNewerVersions = int(
500              boto_rule.conditions[boto.gs.lifecycle.NUM_NEWER_VERSIONS])
501        lifecycle_message.rule.append(lifecycle_rule)
502    return lifecycle_message
503
504  @classmethod
505  def JsonLifecycleFromMessage(cls, lifecycle_message):
506    """Translates an apitools message to lifecycle JSON."""
507    return str(encoding.MessageToJson(lifecycle_message)) + '\n'
508
509  @classmethod
510  def JsonLifecycleToMessage(cls, json_txt):
511    """Translates lifecycle JSON to an apitools message."""
512    try:
513      deserialized_lifecycle = json.loads(json_txt)
514      # If lifecycle JSON is the in the following format
515      # {'lifecycle': {'rule': ... then strip out the 'lifecycle' key
516      # and reduce it to the following format
517      # {'rule': ...
518      if 'lifecycle' in deserialized_lifecycle:
519        deserialized_lifecycle = deserialized_lifecycle['lifecycle']
520      lifecycle = encoding.DictToMessage(
521          deserialized_lifecycle, apitools_messages.Bucket.LifecycleValue)
522      return lifecycle
523    except ValueError:
524      CheckForXmlConfigurationAndRaise('lifecycle', json_txt)
525
526  @classmethod
527  def TranslateBotoLifecycleTimestamp(cls, lifecycle_datetime):
528    """Parses the timestamp from the boto lifecycle into a datetime object."""
529    return datetime.datetime.strptime(lifecycle_datetime, '%Y-%m-%d').date()
530
531
532class CorsTranslation(object):
533  """Functions for converting between various CORS formats.
534
535    This class handles conversation to and from Boto Cors objects, JSON text,
536    and apitools Message objects.
537  """
538
539  @classmethod
540  def BotoCorsFromMessage(cls, cors_message):
541    """Translates an apitools message to a boto Cors object."""
542    cors = boto.gs.cors.Cors()
543    cors.cors = []
544    for collection_message in cors_message:
545      collection_elements = []
546      if collection_message.maxAgeSeconds:
547        collection_elements.append((boto.gs.cors.MAXAGESEC,
548                                    str(collection_message.maxAgeSeconds)))
549      if collection_message.method:
550        method_elements = []
551        for method in collection_message.method:
552          method_elements.append((boto.gs.cors.METHOD, method))
553        collection_elements.append((boto.gs.cors.METHODS, method_elements))
554      if collection_message.origin:
555        origin_elements = []
556        for origin in collection_message.origin:
557          origin_elements.append((boto.gs.cors.ORIGIN, origin))
558        collection_elements.append((boto.gs.cors.ORIGINS, origin_elements))
559      if collection_message.responseHeader:
560        header_elements = []
561        for header in collection_message.responseHeader:
562          header_elements.append((boto.gs.cors.HEADER, header))
563        collection_elements.append((boto.gs.cors.HEADERS, header_elements))
564      cors.cors.append(collection_elements)
565    return cors
566
567  @classmethod
568  def BotoCorsToMessage(cls, boto_cors):
569    """Translates a boto Cors object to an apitools message."""
570    message_cors = []
571    if boto_cors.cors:
572      for cors_collection in boto_cors.cors:
573        if cors_collection:
574          collection_message = apitools_messages.Bucket.CorsValueListEntry()
575          for element_tuple in cors_collection:
576            if element_tuple[0] == boto.gs.cors.MAXAGESEC:
577              collection_message.maxAgeSeconds = int(element_tuple[1])
578            if element_tuple[0] == boto.gs.cors.METHODS:
579              for method_tuple in element_tuple[1]:
580                collection_message.method.append(method_tuple[1])
581            if element_tuple[0] == boto.gs.cors.ORIGINS:
582              for origin_tuple in element_tuple[1]:
583                collection_message.origin.append(origin_tuple[1])
584            if element_tuple[0] == boto.gs.cors.HEADERS:
585              for header_tuple in element_tuple[1]:
586                collection_message.responseHeader.append(header_tuple[1])
587          message_cors.append(collection_message)
588    return message_cors
589
590  @classmethod
591  def JsonCorsToMessageEntries(cls, json_cors):
592    """Translates CORS JSON to an apitools message.
593
594    Args:
595      json_cors: JSON string representing CORS configuration.
596
597    Returns:
598      List of apitools Bucket.CorsValueListEntry. An empty list represents
599      no CORS configuration.
600    """
601    try:
602      deserialized_cors = json.loads(json_cors)
603      cors = []
604      for cors_entry in deserialized_cors:
605        cors.append(encoding.DictToMessage(
606            cors_entry, apitools_messages.Bucket.CorsValueListEntry))
607      return cors
608    except ValueError:
609      CheckForXmlConfigurationAndRaise('CORS', json_cors)
610
611  @classmethod
612  def MessageEntriesToJson(cls, cors_message):
613    """Translates an apitools message to CORS JSON."""
614    json_text = ''
615    # Because CORS is a MessageField, serialize/deserialize as JSON list.
616    json_text += '['
617    printed_one = False
618    for cors_entry in cors_message:
619      if printed_one:
620        json_text += ','
621      else:
622        printed_one = True
623      json_text += encoding.MessageToJson(cors_entry)
624    json_text += ']\n'
625    return json_text
626
627
628def S3MarkerAclFromObjectMetadata(object_metadata):
629  """Retrieves GUID-marked S3 ACL from object metadata, if present.
630
631  Args:
632    object_metadata: Object metadata to check.
633
634  Returns:
635    S3 ACL text, if present, None otherwise.
636  """
637  if (object_metadata and object_metadata.metadata and
638      object_metadata.metadata.additionalProperties):
639    for prop in object_metadata.metadata.additionalProperties:
640      if prop.key == S3_ACL_MARKER_GUID:
641        return prop.value
642
643
644def AddS3MarkerAclToObjectMetadata(object_metadata, acl_text):
645  """Adds a GUID-marked S3 ACL to the object metadata.
646
647  Args:
648    object_metadata: Object metadata to add the acl to.
649    acl_text: S3 ACL text to add.
650  """
651  if not object_metadata.metadata:
652    object_metadata.metadata = apitools_messages.Object.MetadataValue()
653  if not object_metadata.metadata.additionalProperties:
654    object_metadata.metadata.additionalProperties = []
655
656  object_metadata.metadata.additionalProperties.append(
657      apitools_messages.Object.MetadataValue.AdditionalProperty(
658          key=S3_ACL_MARKER_GUID, value=acl_text))
659
660
661class AclTranslation(object):
662  """Functions for converting between various ACL formats.
663
664    This class handles conversion to and from Boto ACL objects, JSON text,
665    and apitools Message objects.
666  """
667
668  JSON_TO_XML_ROLES = {'READER': 'READ', 'WRITER': 'WRITE',
669                       'OWNER': 'FULL_CONTROL'}
670  XML_TO_JSON_ROLES = {'READ': 'READER', 'WRITE': 'WRITER',
671                       'FULL_CONTROL': 'OWNER'}
672
673  @classmethod
674  def BotoAclFromJson(cls, acl_json):
675    acl = ACL()
676    acl.parent = None
677    acl.entries = cls.BotoEntriesFromJson(acl_json, acl)
678    return acl
679
680  @classmethod
681  # acl_message is a list of messages, either object or bucketaccesscontrol
682  def BotoAclFromMessage(cls, acl_message):
683    acl_dicts = []
684    for message in acl_message:
685      if message == PRIVATE_DEFAULT_OBJ_ACL:
686        # Sentinel value indicating acl_dicts should be an empty list to create
687        # a private (no entries) default object ACL.
688        break
689      acl_dicts.append(encoding.MessageToDict(message))
690    return cls.BotoAclFromJson(acl_dicts)
691
692  @classmethod
693  def BotoAclToJson(cls, acl):
694    if hasattr(acl, 'entries'):
695      return cls.BotoEntriesToJson(acl.entries)
696    return []
697
698  @classmethod
699  def BotoObjectAclToMessage(cls, acl):
700    for entry in cls.BotoAclToJson(acl):
701      message = encoding.DictToMessage(entry,
702                                       apitools_messages.ObjectAccessControl)
703      message.kind = u'storage#objectAccessControl'
704      yield message
705
706  @classmethod
707  def BotoBucketAclToMessage(cls, acl):
708    for entry in cls.BotoAclToJson(acl):
709      message = encoding.DictToMessage(entry,
710                                       apitools_messages.BucketAccessControl)
711      message.kind = u'storage#bucketAccessControl'
712      yield message
713
714  @classmethod
715  def BotoEntriesFromJson(cls, acl_json, parent):
716    entries = Entries(parent)
717    entries.parent = parent
718    entries.entry_list = [cls.BotoEntryFromJson(entry_json)
719                          for entry_json in acl_json]
720    return entries
721
722  @classmethod
723  def BotoEntriesToJson(cls, entries):
724    return [cls.BotoEntryToJson(entry) for entry in entries.entry_list]
725
726  @classmethod
727  def BotoEntryFromJson(cls, entry_json):
728    """Converts a JSON entry into a Boto ACL entry."""
729    entity = entry_json['entity']
730    permission = cls.JSON_TO_XML_ROLES[entry_json['role']]
731    if entity.lower() == ALL_USERS.lower():
732      return Entry(type=ALL_USERS, permission=permission)
733    elif entity.lower() == ALL_AUTHENTICATED_USERS.lower():
734      return Entry(type=ALL_AUTHENTICATED_USERS, permission=permission)
735    elif entity.startswith('project'):
736      raise CommandException('XML API does not support project scopes, '
737                             'cannot translate ACL.')
738    elif 'email' in entry_json:
739      if entity.startswith('user'):
740        scope_type = USER_BY_EMAIL
741      elif entity.startswith('group'):
742        scope_type = GROUP_BY_EMAIL
743      return Entry(type=scope_type, email_address=entry_json['email'],
744                   permission=permission)
745    elif 'entityId' in entry_json:
746      if entity.startswith('user'):
747        scope_type = USER_BY_ID
748      elif entity.startswith('group'):
749        scope_type = GROUP_BY_ID
750      return Entry(type=scope_type, id=entry_json['entityId'],
751                   permission=permission)
752    elif 'domain' in entry_json:
753      if entity.startswith('domain'):
754        scope_type = GROUP_BY_DOMAIN
755      return Entry(type=scope_type, domain=entry_json['domain'],
756                   permission=permission)
757    raise CommandException('Failed to translate JSON ACL to XML.')
758
759  @classmethod
760  def BotoEntryToJson(cls, entry):
761    """Converts a Boto ACL entry to a valid JSON dictionary."""
762    acl_entry_json = {}
763    # JSON API documentation uses camel case.
764    scope_type_lower = entry.scope.type.lower()
765    if scope_type_lower == ALL_USERS.lower():
766      acl_entry_json['entity'] = 'allUsers'
767    elif scope_type_lower == ALL_AUTHENTICATED_USERS.lower():
768      acl_entry_json['entity'] = 'allAuthenticatedUsers'
769    elif scope_type_lower == USER_BY_EMAIL.lower():
770      acl_entry_json['entity'] = 'user-%s' % entry.scope.email_address
771      acl_entry_json['email'] = entry.scope.email_address
772    elif scope_type_lower == USER_BY_ID.lower():
773      acl_entry_json['entity'] = 'user-%s' % entry.scope.id
774      acl_entry_json['entityId'] = entry.scope.id
775    elif scope_type_lower == GROUP_BY_EMAIL.lower():
776      acl_entry_json['entity'] = 'group-%s' % entry.scope.email_address
777      acl_entry_json['email'] = entry.scope.email_address
778    elif scope_type_lower == GROUP_BY_ID.lower():
779      acl_entry_json['entity'] = 'group-%s' % entry.scope.id
780      acl_entry_json['entityId'] = entry.scope.id
781    elif scope_type_lower == GROUP_BY_DOMAIN.lower():
782      acl_entry_json['entity'] = 'domain-%s' % entry.scope.domain
783      acl_entry_json['domain'] = entry.scope.domain
784    else:
785      raise ArgumentException('ACL contains invalid scope type: %s' %
786                              scope_type_lower)
787
788    acl_entry_json['role'] = cls.XML_TO_JSON_ROLES[entry.permission]
789    return acl_entry_json
790
791  @classmethod
792  def JsonToMessage(cls, json_data, message_type):
793    """Converts the input JSON data into list of Object/BucketAccessControls.
794
795    Args:
796      json_data: String of JSON to convert.
797      message_type: Which type of access control entries to return,
798                    either ObjectAccessControl or BucketAccessControl.
799
800    Raises:
801      ArgumentException on invalid JSON data.
802
803    Returns:
804      List of ObjectAccessControl or BucketAccessControl elements.
805    """
806    try:
807      deserialized_acl = json.loads(json_data)
808
809      acl = []
810      for acl_entry in deserialized_acl:
811        acl.append(encoding.DictToMessage(acl_entry, message_type))
812      return acl
813    except ValueError:
814      CheckForXmlConfigurationAndRaise('ACL', json_data)
815
816  @classmethod
817  def JsonFromMessage(cls, acl):
818    """Strips unnecessary fields from an ACL message and returns valid JSON.
819
820    Args:
821      acl: iterable ObjectAccessControl or BucketAccessControl
822
823    Returns:
824      ACL JSON string.
825    """
826    serializable_acl = []
827    if acl is not None:
828      for acl_entry in acl:
829        if acl_entry.kind == u'storage#objectAccessControl':
830          acl_entry.object = None
831          acl_entry.generation = None
832        acl_entry.kind = None
833        acl_entry.bucket = None
834        acl_entry.id = None
835        acl_entry.selfLink = None
836        acl_entry.etag = None
837        serializable_acl.append(encoding.MessageToDict(acl_entry))
838    return json.dumps(serializable_acl, sort_keys=True,
839                      indent=2, separators=(',', ': '))
840