1# Protocol Buffers - Google's data interchange format
2# Copyright 2008 Google Inc.  All rights reserved.
3# https://developers.google.com/protocol-buffers/
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions are
7# met:
8#
9#     * Redistributions of source code must retain the above copyright
10# notice, this list of conditions and the following disclaimer.
11#     * Redistributions in binary form must reproduce the above
12# copyright notice, this list of conditions and the following disclaimer
13# in the documentation and/or other materials provided with the
14# distribution.
15#     * Neither the name of Google Inc. nor the names of its
16# contributors may be used to endorse or promote products derived from
17# this software without specific prior written permission.
18#
19# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31"""Contains container classes to represent different protocol buffer types.
32
33This file defines container classes which represent categories of protocol
34buffer field types which need extra maintenance. Currently these categories
35are:
36  - Repeated scalar fields - These are all repeated fields which aren't
37    composite (e.g. they are of simple types like int32, string, etc).
38  - Repeated composite fields - Repeated fields which are composite. This
39    includes groups and nested messages.
40"""
41
42__author__ = 'petar@google.com (Petar Petrov)'
43
44import sys
45try:
46  # This fallback applies for all versions of Python before 3.3
47  import collections.abc as collections_abc
48except ImportError:
49  import collections as collections_abc
50
51if sys.version_info[0] < 3:
52  # We would use collections_abc.MutableMapping all the time, but in Python 2
53  # it doesn't define __slots__.  This causes two significant problems:
54  #
55  # 1. we can't disallow arbitrary attribute assignment, even if our derived
56  #    classes *do* define __slots__.
57  #
58  # 2. we can't safely derive a C type from it without __slots__ defined (the
59  #    interpreter expects to find a dict at tp_dictoffset, which we can't
60  #    robustly provide.  And we don't want an instance dict anyway.
61  #
62  # So this is the Python 2.7 definition of Mapping/MutableMapping functions
63  # verbatim, except that:
64  # 1. We declare __slots__.
65  # 2. We don't declare this as a virtual base class.  The classes defined
66  #    in collections_abc are the interesting base classes, not us.
67  #
68  # Note: deriving from object is critical.  It is the only thing that makes
69  # this a true type, allowing us to derive from it in C++ cleanly and making
70  # __slots__ properly disallow arbitrary element assignment.
71
72  class Mapping(object):
73    __slots__ = ()
74
75    def get(self, key, default=None):
76      try:
77        return self[key]
78      except KeyError:
79        return default
80
81    def __contains__(self, key):
82      try:
83        self[key]
84      except KeyError:
85        return False
86      else:
87        return True
88
89    def iterkeys(self):
90      return iter(self)
91
92    def itervalues(self):
93      for key in self:
94        yield self[key]
95
96    def iteritems(self):
97      for key in self:
98        yield (key, self[key])
99
100    def keys(self):
101      return list(self)
102
103    def items(self):
104      return [(key, self[key]) for key in self]
105
106    def values(self):
107      return [self[key] for key in self]
108
109    # Mappings are not hashable by default, but subclasses can change this
110    __hash__ = None
111
112    def __eq__(self, other):
113      if not isinstance(other, collections_abc.Mapping):
114        return NotImplemented
115      return dict(self.items()) == dict(other.items())
116
117    def __ne__(self, other):
118      return not (self == other)
119
120  class MutableMapping(Mapping):
121    __slots__ = ()
122
123    __marker = object()
124
125    def pop(self, key, default=__marker):
126      try:
127        value = self[key]
128      except KeyError:
129        if default is self.__marker:
130          raise
131        return default
132      else:
133        del self[key]
134        return value
135
136    def popitem(self):
137      try:
138        key = next(iter(self))
139      except StopIteration:
140        raise KeyError
141      value = self[key]
142      del self[key]
143      return key, value
144
145    def clear(self):
146      try:
147        while True:
148          self.popitem()
149      except KeyError:
150        pass
151
152    def update(*args, **kwds):
153      if len(args) > 2:
154        raise TypeError("update() takes at most 2 positional "
155                        "arguments ({} given)".format(len(args)))
156      elif not args:
157        raise TypeError("update() takes at least 1 argument (0 given)")
158      self = args[0]
159      other = args[1] if len(args) >= 2 else ()
160
161      if isinstance(other, Mapping):
162        for key in other:
163          self[key] = other[key]
164      elif hasattr(other, "keys"):
165        for key in other.keys():
166          self[key] = other[key]
167      else:
168        for key, value in other:
169          self[key] = value
170      for key, value in kwds.items():
171        self[key] = value
172
173    def setdefault(self, key, default=None):
174      try:
175        return self[key]
176      except KeyError:
177        self[key] = default
178      return default
179
180  collections_abc.Mapping.register(Mapping)
181  collections_abc.MutableMapping.register(MutableMapping)
182
183else:
184  # In Python 3 we can just use MutableMapping directly, because it defines
185  # __slots__.
186  MutableMapping = collections_abc.MutableMapping
187
188
189class BaseContainer(object):
190
191  """Base container class."""
192
193  # Minimizes memory usage and disallows assignment to other attributes.
194  __slots__ = ['_message_listener', '_values']
195
196  def __init__(self, message_listener):
197    """
198    Args:
199      message_listener: A MessageListener implementation.
200        The RepeatedScalarFieldContainer will call this object's
201        Modified() method when it is modified.
202    """
203    self._message_listener = message_listener
204    self._values = []
205
206  def __getitem__(self, key):
207    """Retrieves item by the specified key."""
208    return self._values[key]
209
210  def __len__(self):
211    """Returns the number of elements in the container."""
212    return len(self._values)
213
214  def __ne__(self, other):
215    """Checks if another instance isn't equal to this one."""
216    # The concrete classes should define __eq__.
217    return not self == other
218
219  def __hash__(self):
220    raise TypeError('unhashable object')
221
222  def __repr__(self):
223    return repr(self._values)
224
225  def sort(self, *args, **kwargs):
226    # Continue to support the old sort_function keyword argument.
227    # This is expected to be a rare occurrence, so use LBYL to avoid
228    # the overhead of actually catching KeyError.
229    if 'sort_function' in kwargs:
230      kwargs['cmp'] = kwargs.pop('sort_function')
231    self._values.sort(*args, **kwargs)
232
233collections_abc.MutableSequence.register(BaseContainer)
234
235
236class RepeatedScalarFieldContainer(BaseContainer):
237
238  """Simple, type-checked, list-like container for holding repeated scalars."""
239
240  # Disallows assignment to other attributes.
241  __slots__ = ['_type_checker']
242
243  def __init__(self, message_listener, type_checker):
244    """
245    Args:
246      message_listener: A MessageListener implementation.
247        The RepeatedScalarFieldContainer will call this object's
248        Modified() method when it is modified.
249      type_checker: A type_checkers.ValueChecker instance to run on elements
250        inserted into this container.
251    """
252    super(RepeatedScalarFieldContainer, self).__init__(message_listener)
253    self._type_checker = type_checker
254
255  def append(self, value):
256    """Appends an item to the list. Similar to list.append()."""
257    self._values.append(self._type_checker.CheckValue(value))
258    if not self._message_listener.dirty:
259      self._message_listener.Modified()
260
261  def insert(self, key, value):
262    """Inserts the item at the specified position. Similar to list.insert()."""
263    self._values.insert(key, self._type_checker.CheckValue(value))
264    if not self._message_listener.dirty:
265      self._message_listener.Modified()
266
267  def extend(self, elem_seq):
268    """Extends by appending the given iterable. Similar to list.extend()."""
269
270    if elem_seq is None:
271      return
272    try:
273      elem_seq_iter = iter(elem_seq)
274    except TypeError:
275      if not elem_seq:
276        # silently ignore falsy inputs :-/.
277        # TODO(ptucker): Deprecate this behavior. b/18413862
278        return
279      raise
280
281    new_values = [self._type_checker.CheckValue(elem) for elem in elem_seq_iter]
282    if new_values:
283      self._values.extend(new_values)
284    self._message_listener.Modified()
285
286  def MergeFrom(self, other):
287    """Appends the contents of another repeated field of the same type to this
288    one. We do not check the types of the individual fields.
289    """
290    self._values.extend(other._values)
291    self._message_listener.Modified()
292
293  def remove(self, elem):
294    """Removes an item from the list. Similar to list.remove()."""
295    self._values.remove(elem)
296    self._message_listener.Modified()
297
298  def pop(self, key=-1):
299    """Removes and returns an item at a given index. Similar to list.pop()."""
300    value = self._values[key]
301    self.__delitem__(key)
302    return value
303
304  def __setitem__(self, key, value):
305    """Sets the item on the specified position."""
306    if isinstance(key, slice):  # PY3
307      if key.step is not None:
308        raise ValueError('Extended slices not supported')
309      self.__setslice__(key.start, key.stop, value)
310    else:
311      self._values[key] = self._type_checker.CheckValue(value)
312      self._message_listener.Modified()
313
314  def __getslice__(self, start, stop):
315    """Retrieves the subset of items from between the specified indices."""
316    return self._values[start:stop]
317
318  def __setslice__(self, start, stop, values):
319    """Sets the subset of items from between the specified indices."""
320    new_values = []
321    for value in values:
322      new_values.append(self._type_checker.CheckValue(value))
323    self._values[start:stop] = new_values
324    self._message_listener.Modified()
325
326  def __delitem__(self, key):
327    """Deletes the item at the specified position."""
328    del self._values[key]
329    self._message_listener.Modified()
330
331  def __delslice__(self, start, stop):
332    """Deletes the subset of items from between the specified indices."""
333    del self._values[start:stop]
334    self._message_listener.Modified()
335
336  def __eq__(self, other):
337    """Compares the current instance with another one."""
338    if self is other:
339      return True
340    # Special case for the same type which should be common and fast.
341    if isinstance(other, self.__class__):
342      return other._values == self._values
343    # We are presumably comparing against some other sequence type.
344    return other == self._values
345
346
347class RepeatedCompositeFieldContainer(BaseContainer):
348
349  """Simple, list-like container for holding repeated composite fields."""
350
351  # Disallows assignment to other attributes.
352  __slots__ = ['_message_descriptor']
353
354  def __init__(self, message_listener, message_descriptor):
355    """
356    Note that we pass in a descriptor instead of the generated directly,
357    since at the time we construct a _RepeatedCompositeFieldContainer we
358    haven't yet necessarily initialized the type that will be contained in the
359    container.
360
361    Args:
362      message_listener: A MessageListener implementation.
363        The RepeatedCompositeFieldContainer will call this object's
364        Modified() method when it is modified.
365      message_descriptor: A Descriptor instance describing the protocol type
366        that should be present in this container.  We'll use the
367        _concrete_class field of this descriptor when the client calls add().
368    """
369    super(RepeatedCompositeFieldContainer, self).__init__(message_listener)
370    self._message_descriptor = message_descriptor
371
372  def add(self, **kwargs):
373    """Adds a new element at the end of the list and returns it. Keyword
374    arguments may be used to initialize the element.
375    """
376    new_element = self._message_descriptor._concrete_class(**kwargs)
377    new_element._SetListener(self._message_listener)
378    self._values.append(new_element)
379    if not self._message_listener.dirty:
380      self._message_listener.Modified()
381    return new_element
382
383  def append(self, value):
384    """Appends one element by copying the message."""
385    new_element = self._message_descriptor._concrete_class()
386    new_element._SetListener(self._message_listener)
387    new_element.CopyFrom(value)
388    self._values.append(new_element)
389    if not self._message_listener.dirty:
390      self._message_listener.Modified()
391
392  def insert(self, key, value):
393    """Inserts the item at the specified position by copying."""
394    new_element = self._message_descriptor._concrete_class()
395    new_element._SetListener(self._message_listener)
396    new_element.CopyFrom(value)
397    self._values.insert(key, new_element)
398    if not self._message_listener.dirty:
399      self._message_listener.Modified()
400
401  def extend(self, elem_seq):
402    """Extends by appending the given sequence of elements of the same type
403    as this one, copying each individual message.
404    """
405    message_class = self._message_descriptor._concrete_class
406    listener = self._message_listener
407    values = self._values
408    for message in elem_seq:
409      new_element = message_class()
410      new_element._SetListener(listener)
411      new_element.MergeFrom(message)
412      values.append(new_element)
413    listener.Modified()
414
415  def MergeFrom(self, other):
416    """Appends the contents of another repeated field of the same type to this
417    one, copying each individual message.
418    """
419    self.extend(other._values)
420
421  def remove(self, elem):
422    """Removes an item from the list. Similar to list.remove()."""
423    self._values.remove(elem)
424    self._message_listener.Modified()
425
426  def pop(self, key=-1):
427    """Removes and returns an item at a given index. Similar to list.pop()."""
428    value = self._values[key]
429    self.__delitem__(key)
430    return value
431
432  def __getslice__(self, start, stop):
433    """Retrieves the subset of items from between the specified indices."""
434    return self._values[start:stop]
435
436  def __delitem__(self, key):
437    """Deletes the item at the specified position."""
438    del self._values[key]
439    self._message_listener.Modified()
440
441  def __delslice__(self, start, stop):
442    """Deletes the subset of items from between the specified indices."""
443    del self._values[start:stop]
444    self._message_listener.Modified()
445
446  def __eq__(self, other):
447    """Compares the current instance with another one."""
448    if self is other:
449      return True
450    if not isinstance(other, self.__class__):
451      raise TypeError('Can only compare repeated composite fields against '
452                      'other repeated composite fields.')
453    return self._values == other._values
454
455
456class ScalarMap(MutableMapping):
457
458  """Simple, type-checked, dict-like container for holding repeated scalars."""
459
460  # Disallows assignment to other attributes.
461  __slots__ = ['_key_checker', '_value_checker', '_values', '_message_listener',
462               '_entry_descriptor']
463
464  def __init__(self, message_listener, key_checker, value_checker,
465               entry_descriptor):
466    """
467    Args:
468      message_listener: A MessageListener implementation.
469        The ScalarMap will call this object's Modified() method when it
470        is modified.
471      key_checker: A type_checkers.ValueChecker instance to run on keys
472        inserted into this container.
473      value_checker: A type_checkers.ValueChecker instance to run on values
474        inserted into this container.
475      entry_descriptor: The MessageDescriptor of a map entry: key and value.
476    """
477    self._message_listener = message_listener
478    self._key_checker = key_checker
479    self._value_checker = value_checker
480    self._entry_descriptor = entry_descriptor
481    self._values = {}
482
483  def __getitem__(self, key):
484    try:
485      return self._values[key]
486    except KeyError:
487      key = self._key_checker.CheckValue(key)
488      val = self._value_checker.DefaultValue()
489      self._values[key] = val
490      return val
491
492  def __contains__(self, item):
493    # We check the key's type to match the strong-typing flavor of the API.
494    # Also this makes it easier to match the behavior of the C++ implementation.
495    self._key_checker.CheckValue(item)
496    return item in self._values
497
498  # We need to override this explicitly, because our defaultdict-like behavior
499  # will make the default implementation (from our base class) always insert
500  # the key.
501  def get(self, key, default=None):
502    if key in self:
503      return self[key]
504    else:
505      return default
506
507  def __setitem__(self, key, value):
508    checked_key = self._key_checker.CheckValue(key)
509    checked_value = self._value_checker.CheckValue(value)
510    self._values[checked_key] = checked_value
511    self._message_listener.Modified()
512
513  def __delitem__(self, key):
514    del self._values[key]
515    self._message_listener.Modified()
516
517  def __len__(self):
518    return len(self._values)
519
520  def __iter__(self):
521    return iter(self._values)
522
523  def __repr__(self):
524    return repr(self._values)
525
526  def MergeFrom(self, other):
527    self._values.update(other._values)
528    self._message_listener.Modified()
529
530  def InvalidateIterators(self):
531    # It appears that the only way to reliably invalidate iterators to
532    # self._values is to ensure that its size changes.
533    original = self._values
534    self._values = original.copy()
535    original[None] = None
536
537  # This is defined in the abstract base, but we can do it much more cheaply.
538  def clear(self):
539    self._values.clear()
540    self._message_listener.Modified()
541
542  def GetEntryClass(self):
543    return self._entry_descriptor._concrete_class
544
545
546class MessageMap(MutableMapping):
547
548  """Simple, type-checked, dict-like container for with submessage values."""
549
550  # Disallows assignment to other attributes.
551  __slots__ = ['_key_checker', '_values', '_message_listener',
552               '_message_descriptor', '_entry_descriptor']
553
554  def __init__(self, message_listener, message_descriptor, key_checker,
555               entry_descriptor):
556    """
557    Args:
558      message_listener: A MessageListener implementation.
559        The ScalarMap will call this object's Modified() method when it
560        is modified.
561      key_checker: A type_checkers.ValueChecker instance to run on keys
562        inserted into this container.
563      value_checker: A type_checkers.ValueChecker instance to run on values
564        inserted into this container.
565      entry_descriptor: The MessageDescriptor of a map entry: key and value.
566    """
567    self._message_listener = message_listener
568    self._message_descriptor = message_descriptor
569    self._key_checker = key_checker
570    self._entry_descriptor = entry_descriptor
571    self._values = {}
572
573  def __getitem__(self, key):
574    key = self._key_checker.CheckValue(key)
575    try:
576      return self._values[key]
577    except KeyError:
578      new_element = self._message_descriptor._concrete_class()
579      new_element._SetListener(self._message_listener)
580      self._values[key] = new_element
581      self._message_listener.Modified()
582
583      return new_element
584
585  def get_or_create(self, key):
586    """get_or_create() is an alias for getitem (ie. map[key]).
587
588    Args:
589      key: The key to get or create in the map.
590
591    This is useful in cases where you want to be explicit that the call is
592    mutating the map.  This can avoid lint errors for statements like this
593    that otherwise would appear to be pointless statements:
594
595      msg.my_map[key]
596    """
597    return self[key]
598
599  # We need to override this explicitly, because our defaultdict-like behavior
600  # will make the default implementation (from our base class) always insert
601  # the key.
602  def get(self, key, default=None):
603    if key in self:
604      return self[key]
605    else:
606      return default
607
608  def __contains__(self, item):
609    item = self._key_checker.CheckValue(item)
610    return item in self._values
611
612  def __setitem__(self, key, value):
613    raise ValueError('May not set values directly, call my_map[key].foo = 5')
614
615  def __delitem__(self, key):
616    key = self._key_checker.CheckValue(key)
617    del self._values[key]
618    self._message_listener.Modified()
619
620  def __len__(self):
621    return len(self._values)
622
623  def __iter__(self):
624    return iter(self._values)
625
626  def __repr__(self):
627    return repr(self._values)
628
629  def MergeFrom(self, other):
630    for key in other:
631      # According to documentation: "When parsing from the wire or when merging,
632      # if there are duplicate map keys the last key seen is used".
633      if key in self:
634        del self[key]
635      self[key].CopyFrom(other[key])
636    # self._message_listener.Modified() not required here, because
637    # mutations to submessages already propagate.
638
639  def InvalidateIterators(self):
640    # It appears that the only way to reliably invalidate iterators to
641    # self._values is to ensure that its size changes.
642    original = self._values
643    self._values = original.copy()
644    original[None] = None
645
646  # This is defined in the abstract base, but we can do it much more cheaply.
647  def clear(self):
648    self._values.clear()
649    self._message_listener.Modified()
650
651  def GetEntryClass(self):
652    return self._entry_descriptor._concrete_class
653
654
655class _UnknownField(object):
656
657  """A parsed unknown field."""
658
659  # Disallows assignment to other attributes.
660  __slots__ = ['_field_number', '_wire_type', '_data']
661
662  def __init__(self, field_number, wire_type, data):
663    self._field_number = field_number
664    self._wire_type = wire_type
665    self._data = data
666    return
667
668  def __lt__(self, other):
669    # pylint: disable=protected-access
670    return self._field_number < other._field_number
671
672  def __eq__(self, other):
673    if self is other:
674      return True
675    # pylint: disable=protected-access
676    return (self._field_number == other._field_number and
677            self._wire_type == other._wire_type and
678            self._data == other._data)
679
680
681class UnknownFieldRef(object):
682
683  def __init__(self, parent, index):
684    self._parent = parent
685    self._index = index
686    return
687
688  def _check_valid(self):
689    if not self._parent:
690      raise ValueError('UnknownField does not exist. '
691                       'The parent message might be cleared.')
692    if self._index >= len(self._parent):
693      raise ValueError('UnknownField does not exist. '
694                       'The parent message might be cleared.')
695
696  @property
697  def field_number(self):
698    self._check_valid()
699    # pylint: disable=protected-access
700    return self._parent._internal_get(self._index)._field_number
701
702  @property
703  def wire_type(self):
704    self._check_valid()
705    # pylint: disable=protected-access
706    return self._parent._internal_get(self._index)._wire_type
707
708  @property
709  def data(self):
710    self._check_valid()
711    # pylint: disable=protected-access
712    return self._parent._internal_get(self._index)._data
713
714
715class UnknownFieldSet(object):
716
717  """UnknownField container"""
718
719  # Disallows assignment to other attributes.
720  __slots__ = ['_values']
721
722  def __init__(self):
723    self._values = []
724
725  def __getitem__(self, index):
726    if self._values is None:
727      raise ValueError('UnknownFields does not exist. '
728                       'The parent message might be cleared.')
729    size = len(self._values)
730    if index < 0:
731      index += size
732    if index < 0 or index >= size:
733      raise IndexError('index %d out of range'.index)
734
735    return UnknownFieldRef(self, index)
736
737  def _internal_get(self, index):
738    return self._values[index]
739
740  def __len__(self):
741    if self._values is None:
742      raise ValueError('UnknownFields does not exist. '
743                       'The parent message might be cleared.')
744    return len(self._values)
745
746  def _add(self, field_number, wire_type, data):
747    unknown_field = _UnknownField(field_number, wire_type, data)
748    self._values.append(unknown_field)
749    return unknown_field
750
751  def __iter__(self):
752    for i in range(len(self)):
753      yield UnknownFieldRef(self, i)
754
755  def _extend(self, other):
756    if other is None:
757      return
758    # pylint: disable=protected-access
759    self._values.extend(other._values)
760
761  def __eq__(self, other):
762    if self is other:
763      return True
764    # Sort unknown fields because their order shouldn't
765    # affect equality test.
766    values = list(self._values)
767    if other is None:
768      return not values
769    values.sort()
770    # pylint: disable=protected-access
771    other_values = sorted(other._values)
772    return values == other_values
773
774  def _clear(self):
775    for value in self._values:
776      # pylint: disable=protected-access
777      if isinstance(value._data, UnknownFieldSet):
778        value._data._clear()  # pylint: disable=protected-access
779    self._values = None
780