1# Copyright 2015 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import posixpath
6import re
7
8from telemetry.timeline import event as timeline_event
9
10
11class MmapCategory(object):
12  _DEFAULT_CATEGORY = None
13
14  def __init__(self, name, file_pattern, children=None):
15    """A (sub)category for classifying memory maps.
16
17    Args:
18      name: A string to identify the category.
19      file_pattern: A regex pattern, the category will aggregate memory usage
20          for all mapped files matching this pattern.
21      children: A list of MmapCategory objects, used to sub-categorize memory
22          usage.
23    """
24    self.name = name
25    self._file_pattern = re.compile(file_pattern) if file_pattern else None
26    self._children = list(children) if children else None
27
28  @classmethod
29  def DefaultCategory(cls):
30    """An implicit 'Others' match-all category with no children."""
31    if cls._DEFAULT_CATEGORY is None:
32      cls._DEFAULT_CATEGORY = cls('Others', None)
33    return cls._DEFAULT_CATEGORY
34
35  def Match(self, mapped_file):
36    """Test whether a mapped file matches this category."""
37    return (self._file_pattern is None
38            or bool(self._file_pattern.search(mapped_file)))
39
40  def GetMatchingChild(self, mapped_file):
41    """Get the first matching sub-category for a given mapped file.
42
43    Returns None if the category has no children, or the DefaultCategory if
44    it does have children but none of them match.
45    """
46    if not self._children:
47      return None
48    for child in self._children:
49      if child.Match(mapped_file):
50        return child
51    return type(self).DefaultCategory()
52
53
54ROOT_CATEGORY = MmapCategory('/', None, [
55  MmapCategory('Android', r'^\/dev\/ashmem(?!\/libc malloc)', [
56    MmapCategory('Java runtime', r'^\/dev\/ashmem\/dalvik-', [
57      MmapCategory('Spaces', r'\/dalvik-(alloc|main|large'
58                             r' object|non moving|zygote) space', [
59        MmapCategory('Normal', r'\/dalvik-(alloc|main)'),
60        MmapCategory('Large', r'\/dalvik-large object'),
61        MmapCategory('Zygote', r'\/dalvik-zygote'),
62        MmapCategory('Non-moving', r'\/dalvik-non moving')
63      ]),
64      MmapCategory('Linear Alloc', r'\/dalvik-LinearAlloc'),
65      MmapCategory('Indirect Reference Table', r'\/dalvik-indirect.ref'),
66      MmapCategory('Cache', r'\/dalvik-jit-code-cache'),
67      MmapCategory('Accounting', None)
68    ]),
69    MmapCategory('Cursor', r'\/CursorWindow'),
70    MmapCategory('Ashmem', None)
71  ]),
72  MmapCategory('Native heap',
73               r'^((\[heap\])|(\[anon:)|(\/dev\/ashmem\/libc malloc)|$)'),
74  MmapCategory('Stack', r'^\[stack'),
75  MmapCategory('Files',
76               r'\.((((so)|(jar)|(apk)|(ttf)|(odex)|(oat)|(art))$)|(dex))', [
77    MmapCategory('so', r'\.so$'),
78    MmapCategory('jar', r'\.jar$'),
79    MmapCategory('apk', r'\.apk$'),
80    MmapCategory('ttf', r'\.ttf$'),
81    MmapCategory('dex', r'\.((dex)|(odex$))'),
82    MmapCategory('oat', r'\.oat$'),
83    MmapCategory('art', r'\.art$'),
84  ]),
85  MmapCategory('Devices', r'(^\/dev\/)|(anon_inode:dmabuf)', [
86    MmapCategory('GPU', r'\/((nv)|(mali)|(kgsl))'),
87    MmapCategory('DMA', r'anon_inode:dmabuf'),
88  ]),
89  MmapCategory('Discounted tracing overhead',
90               r'\[discounted tracing overhead\]')
91])
92
93
94# Map long descriptive attribute names, as understood by MemoryBucket.GetValue,
95# to the short keys used by events in raw json traces.
96BUCKET_ATTRS = {
97  'proportional_resident': 'pss',
98  'private_dirty_resident': 'pd',
99  'private_clean_resident': 'pc',
100  'shared_dirty_resident': 'sd',
101  'shared_clean_resident': 'sc',
102  'swapped': 'sw'}
103
104
105# Map of {memory_key: (category_path, discount_tracing), ...}.
106# When discount_tracing is True, we have to discount the resident_size of the
107# tracing allocator to get the correct value for that key.
108MMAPS_METRICS = {
109  'mmaps_overall_pss': ('/.proportional_resident', True),
110  'mmaps_private_dirty' : ('/.private_dirty_resident', True),
111  'mmaps_java_heap': ('/Android/Java runtime/Spaces.proportional_resident',
112                      False),
113  'mmaps_ashmem': ('/Android/Ashmem.proportional_resident', False),
114  'mmaps_native_heap': ('/Native heap.proportional_resident', True)}
115
116
117class MemoryBucket(object):
118  """Simple object to hold and aggregate memory values."""
119  def __init__(self):
120    self._bucket = dict.fromkeys(BUCKET_ATTRS.iterkeys(), 0)
121
122  def __repr__(self):
123    values = ', '.join('%s=%d' % (src_key, self._bucket[dst_key])
124                       for dst_key, src_key
125                       in sorted(BUCKET_ATTRS.iteritems()))
126    return '%s[%s]' % (type(self).__name__, values)
127
128  def AddRegion(self, byte_stats):
129    for dst_key, src_key in BUCKET_ATTRS.iteritems():
130      self._bucket[dst_key] += int(byte_stats.get(src_key, '0'), 16)
131
132  def GetValue(self, name):
133    return self._bucket[name]
134
135
136class ProcessMemoryDumpEvent(timeline_event.TimelineEvent):
137  """A memory dump event belonging to a single timeline.Process object.
138
139  It's a subclass of telemetry's TimelineEvent so it can be included in
140  the stream of events contained in timeline.model objects, and have its
141  timing correlated with that of other events in the model.
142
143  Args:
144    process: The Process object associated with the memory dump.
145    dump_events: A list of dump events of the process with the same dump id.
146
147  Properties:
148    dump_id: A string to identify events belonging to the same global dump.
149    process: The timeline.Process object that owns this memory dump event.
150    has_mmaps: True if the memory dump has mmaps information. If False then
151        GetMemoryUsage will report all zeros.
152  """
153  def __init__(self, process, dump_events):
154    assert dump_events
155
156    start_time = min(event['ts'] for event in dump_events) / 1000.0
157    duration = max(event['ts'] for event in dump_events) / 1000.0 - start_time
158    super(ProcessMemoryDumpEvent, self).__init__('memory', 'memory_dump',
159                                                 start_time, duration)
160
161    self.process = process
162    self.dump_id = dump_events[0]['id']
163
164    allocator_dumps = {}
165    vm_regions = []
166    for event in dump_events:
167      assert (event['ph'] == 'v' and self.process.pid == event['pid'] and
168              self.dump_id == event['id'])
169      try:
170        allocator_dumps.update(event['args']['dumps']['allocators'])
171      except KeyError:
172        pass  # It's ok if any of those keys are not present.
173      try:
174        value = event['args']['dumps']['process_mmaps']['vm_regions']
175        assert not vm_regions
176        vm_regions = value
177      except KeyError:
178        pass  # It's ok if any of those keys are not present.
179
180    self._allocators = {}
181    parent_path = ''
182    parent_has_size = False
183    for allocator_name, size_values in sorted(allocator_dumps.iteritems()):
184      if ((allocator_name.startswith(parent_path) and parent_has_size) or
185          allocator_name.startswith('global/')):
186        continue
187      parent_path = allocator_name + '/'
188      parent_has_size = 'size' in size_values['attrs']
189      name_parts = allocator_name.split('/')
190      allocator_name = name_parts[0]
191      # For 'gpu/android_memtrack/*' we want to keep track of individual
192      # components. E.g. 'gpu/android_memtrack/gl' will be stored as
193      # 'android_memtrack_gl' in the allocators dict.
194      if (len(name_parts) == 3 and allocator_name == 'gpu' and
195          name_parts[1] == 'android_memtrack'):
196        allocator_name = '_'.join(name_parts[1:3])
197      allocator = self._allocators.setdefault(allocator_name, {})
198      for size_key, size_value in size_values['attrs'].iteritems():
199        if size_value['units'] == 'bytes':
200          allocator[size_key] = (allocator.get(size_key, 0)
201                                 + int(size_value['value'], 16))
202    # we need to discount tracing from malloc size.
203    try:
204      self._allocators['malloc']['size'] -= self._allocators['tracing']['size']
205    except KeyError:
206      pass  # It's ok if any of those keys are not present.
207
208    self.has_mmaps = bool(vm_regions)
209    self._buckets = {}
210    for vm_region in vm_regions:
211      self._AddRegion(vm_region)
212
213  @property
214  def process_name(self):
215    return self.process.name
216
217  def _AddRegion(self, vm_region):
218    path = ''
219    category = ROOT_CATEGORY
220    while category:
221      path = posixpath.join(path, category.name)
222      self.GetMemoryBucket(path).AddRegion(vm_region['bs'])
223      mapped_file = vm_region['mf']
224      category = category.GetMatchingChild(mapped_file)
225
226  def __repr__(self):
227    values = ['pid=%d' % self.process.pid]
228    for key, value in sorted(self.GetMemoryUsage().iteritems()):
229      values.append('%s=%d' % (key, value))
230    values = ', '.join(values)
231    return '%s[%s]' % (type(self).__name__, values)
232
233  def GetMemoryBucket(self, path):
234    """Return the MemoryBucket associated with a category path.
235
236    An empty bucket will be created if the path does not already exist.
237
238    path: A string with path in the classification tree, e.g.
239        '/Android/Java runtime/Cache'. Note: no trailing slash, except for
240        the root path '/'.
241    """
242    if not path in self._buckets:
243      self._buckets[path] = MemoryBucket()
244    return self._buckets[path]
245
246  def GetMemoryValue(self, category_path, discount_tracing=False):
247    """Return a specific value from within a MemoryBucket.
248
249    category_path: A string composed of a path in the classification tree,
250        followed by a '.', followed by a specific bucket value, e.g.
251        '/Android/Java runtime/Cache.private_dirty_resident'.
252    discount_tracing: A boolean indicating whether the returned value should
253        be discounted by the resident size of the tracing allocator.
254    """
255    path, name = category_path.rsplit('.', 1)
256    value = self.GetMemoryBucket(path).GetValue(name)
257    if discount_tracing and 'tracing' in self._allocators:
258      value -= self._allocators['tracing'].get('resident_size', 0)
259    return value
260
261  def GetMemoryUsage(self):
262    """Get a dictionary with the memory usage of this process."""
263    usage = {}
264    for name, values in self._allocators.iteritems():
265      # If you wish to track more attributes here, make sure they are correctly
266      # calculated by the ProcessMemoryDumpEvent method. All dumps whose parent
267      # has "size" attribute are ignored to avoid double counting. So, the
268      # other attributes are totals of only top level dumps.
269      if 'size' in values:
270        usage['allocator_%s' % name] = values['size']
271      if 'allocated_objects_size' in values:
272        usage['allocated_objects_%s' % name] = values['allocated_objects_size']
273      if 'memtrack_pss' in values:
274        usage[name] = values['memtrack_pss']
275    if self.has_mmaps:
276      usage.update((key, self.GetMemoryValue(*value))
277                   for key, value in MMAPS_METRICS.iteritems())
278    return usage
279
280
281class GlobalMemoryDump(object):
282  """Object to aggregate individual process dumps with the same dump id.
283
284  Args:
285    process_dumps: A sequence of ProcessMemoryDumpEvent objects, all sharing
286        the same global dump id.
287
288  Attributes:
289    dump_id: A string identifying this dump.
290    has_mmaps: True if the memory dump has mmaps information. If False then
291        GetMemoryUsage will report all zeros.
292  """
293  def __init__(self, process_dumps):
294    assert process_dumps
295    # Keep dumps sorted in chronological order.
296    self._process_dumps = sorted(process_dumps, key=lambda dump: dump.start)
297
298    # All process dump events should have the same dump id.
299    dump_ids = set(dump.dump_id for dump in self._process_dumps)
300    assert len(dump_ids) == 1
301    self.dump_id = dump_ids.pop()
302
303    # Either all processes have mmaps or none of them do.
304    have_mmaps = set(dump.has_mmaps for dump in self._process_dumps)
305    assert len(have_mmaps) == 1
306    self.has_mmaps = have_mmaps.pop()
307
308  @property
309  def start(self):
310    return self._process_dumps[0].start
311
312  @property
313  def end(self):
314    return max(dump.end for dump in self._process_dumps)
315
316  @property
317  def duration(self):
318    return self.end - self.start
319
320  @property
321  def pids(self):
322    return set(d.process.pid for d in self._process_dumps)
323
324  def IterProcessMemoryDumps(self):
325    return iter(self._process_dumps)
326
327  def CountProcessMemoryDumps(self):
328    return len(self._process_dumps)
329
330  def __repr__(self):
331    values = ['id=%s' % self.dump_id]
332    for key, value in sorted(self.GetMemoryUsage().iteritems()):
333      values.append('%s=%d' % (key, value))
334    values = ', '.join(values)
335    return '%s[%s]' % (type(self).__name__, values)
336
337  def GetMemoryUsage(self):
338    """Get the aggregated memory usage over all processes in this dump."""
339    result = {}
340    for dump in self._process_dumps:
341      for key, value in dump.GetMemoryUsage().iteritems():
342        result[key] = result.get(key, 0) + value
343    return result
344