1#!/usr/bin/env python3
2
3#
4# Copyright (C) 2019 The Android Open Source Project
5#
6# Licensed under the Apache License, Version 2.0 (the "License");
7# you may not use this file except in compliance with the License.
8# You may obtain a copy of the License at
9#
10#      http://www.apache.org/licenses/LICENSE-2.0
11#
12# Unless required by applicable law or agreed to in writing, software
13# distributed under the License is distributed on an "AS IS" BASIS,
14# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15# See the License for the specific language governing permissions and
16# limitations under the License.
17#
18
19#
20# Dependencies:
21#
22# $> sudo apt-get install python3-pip
23# $> pip3 install --user protobuf sqlalchemy sqlite3
24#
25
26import optparse
27import os
28import re
29import sys
30import tempfile
31from pathlib import Path
32from datetime import timedelta
33from typing import Iterable, Optional, List
34
35DIR = os.path.abspath(os.path.dirname(__file__))
36sys.path.append(os.path.dirname(DIR))
37from iorap.generated.TraceFile_pb2 import *
38from iorap.lib.inode2filename import Inode2Filename
39
40parent_dir_name = os.path.dirname(os.path.dirname(os.path.realpath(__file__)))
41sys.path.append(parent_dir_name)
42from trace_analyzer.lib.trace2db import Trace2Db, MmFilemapAddToPageCache, \
43    RawFtraceEntry
44import lib.cmd_utils as cmd_utils
45
46_PAGE_SIZE = 4096 # adb shell getconf PAGESIZE ## size of a memory page in bytes.
47ANDROID_BUILD_TOP = Path(parent_dir_name).parents[3]
48TRACECONV_BIN = ANDROID_BUILD_TOP.joinpath(
49    'external/perfetto/tools/traceconv')
50
51class PageRun:
52  """
53  Intermediate representation for a run of one or more pages.
54  """
55  def __init__(self, device_number: int, inode: int, offset: int, length: int):
56    self.device_number = device_number
57    self.inode = inode
58    self.offset = offset
59    self.length = length
60
61  def __str__(self):
62    return "PageRun(device_number=%d, inode=%d, offset=%d, length=%d)" \
63        %(self.device_number, self.inode, self.offset, self.length)
64
65def debug_print(msg):
66  #print(msg)
67  pass
68
69UNDER_LAUNCH = False
70
71def page_cache_entries_to_runs(page_cache_entries: Iterable[MmFilemapAddToPageCache]):
72  global _PAGE_SIZE
73
74  runs = [
75      PageRun(device_number=pg_entry.dev, inode=pg_entry.ino, offset=pg_entry.ofs,
76              length=_PAGE_SIZE)
77        for pg_entry in page_cache_entries
78  ]
79
80  for r in runs:
81    debug_print(r)
82
83  print("Stats: Page runs totaling byte length: %d" %(len(runs) * _PAGE_SIZE))
84
85  return runs
86
87def optimize_page_runs(page_runs):
88  new_entries = []
89  last_entry = None
90  for pg_entry in page_runs:
91    if last_entry:
92      if pg_entry.device_number == last_entry.device_number and pg_entry.inode == last_entry.inode:
93        # we are dealing with a run for the same exact file as a previous run.
94        if pg_entry.offset == last_entry.offset + last_entry.length:
95          # trivially contiguous entries. merge them together.
96          last_entry.length += pg_entry.length
97          continue
98    # Default: Add the run without merging it to a previous run.
99    last_entry = pg_entry
100    new_entries.append(pg_entry)
101  return new_entries
102
103def is_filename_matching_filter(file_name, filters=[]):
104  """
105  Blacklist-style regular expression filters.
106
107  :return: True iff file_name has an RE match in one of the filters.
108  """
109  for filt in filters:
110    res = re.search(filt, file_name)
111    if res:
112      return True
113
114  return False
115
116def build_protobuf(page_runs, inode2filename, filters=[]):
117  trace_file = TraceFile()
118  trace_file_index = trace_file.index
119
120  file_id_counter = 0
121  file_id_map = {} # filename -> id
122
123  stats_length_total = 0
124  filename_stats = {} # filename -> total size
125
126  skipped_inode_map = {}
127  filtered_entry_map = {} # filename -> count
128
129  for pg_entry in page_runs:
130    fn = inode2filename.resolve(pg_entry.device_number, pg_entry.inode)
131    if not fn:
132      skipped_inode_map[pg_entry.inode] = skipped_inode_map.get(pg_entry.inode, 0) + 1
133      continue
134
135    filename = fn
136
137    if filters and not is_filename_matching_filter(filename, filters):
138      filtered_entry_map[filename] = filtered_entry_map.get(filename, 0) + 1
139      continue
140
141    file_id = file_id_map.get(filename)
142    # file_id could 0, which satisfies "if file_id" and causes duplicate
143    # filename for file id 0.
144    if file_id is None:
145      file_id = file_id_counter
146      file_id_map[filename] = file_id_counter
147      file_id_counter = file_id_counter + 1
148
149      file_index_entry = trace_file_index.entries.add()
150      file_index_entry.id = file_id
151      file_index_entry.file_name = filename
152
153    # already in the file index, add the file entry.
154    file_entry = trace_file.list.entries.add()
155    file_entry.index_id = file_id
156    file_entry.file_length = pg_entry.length
157    stats_length_total += file_entry.file_length
158    file_entry.file_offset = pg_entry.offset
159
160    filename_stats[filename] = filename_stats.get(filename, 0) + file_entry.file_length
161
162  for inode, count in skipped_inode_map.items():
163    print("WARNING: Skip inode %s because it's not in inode map (%d entries)" %(inode, count))
164
165  print("Stats: Sum of lengths %d" %(stats_length_total))
166
167  if filters:
168    print("Filter: %d total files removed." %(len(filtered_entry_map)))
169
170    for fn, count in filtered_entry_map.items():
171      print("Filter: File '%s' removed '%d' entries." %(fn, count))
172
173  for filename, file_size in filename_stats.items():
174    print("%s,%s" %(filename, file_size))
175
176  return trace_file
177
178def calc_trace_end_time(trace2db: Trace2Db,
179                        trace_duration: Optional[timedelta]) -> float:
180  """
181  Calculates the end time based on the trace duration.
182  The start time is the first receiving mm file map event.
183  The end time is the start time plus the trace duration.
184  All of them are in milliseconds.
185  """
186  # If the duration is not set, assume all time is acceptable.
187  if trace_duration is None:
188    # float('inf')
189    return RawFtraceEntry.__table__.c.timestamp.type.python_type('inf')
190
191  first_event = trace2db.session.query(MmFilemapAddToPageCache).join(
192      MmFilemapAddToPageCache.raw_ftrace_entry).order_by(
193      RawFtraceEntry.timestamp).first()
194
195  # total_seconds() will return a float number.
196  return first_event.raw_ftrace_entry.timestamp + trace_duration.total_seconds()
197
198def query_add_to_page_cache(trace2db: Trace2Db, trace_duration: Optional[timedelta]):
199  end_time = calc_trace_end_time(trace2db, trace_duration)
200  # SELECT * FROM tbl ORDER BY id;
201  return trace2db.session.query(MmFilemapAddToPageCache).join(
202      MmFilemapAddToPageCache.raw_ftrace_entry).filter(
203      RawFtraceEntry.timestamp <= end_time).order_by(
204      MmFilemapAddToPageCache.id).all()
205
206def transform_perfetto_trace_to_systrace(path_to_perfetto_trace: str,
207                                         path_to_tmp_systrace: str) -> None:
208  """ Transforms the systrace file from perfetto trace. """
209  cmd_utils.run_command_nofail([str(TRACECONV_BIN),
210                                'systrace',
211                                path_to_perfetto_trace,
212                                path_to_tmp_systrace])
213
214
215def run(sql_db_path:str,
216        trace_file:str,
217        trace_duration:Optional[timedelta],
218        output_file:str,
219        inode_table:str,
220        filter:List[str]) -> int:
221  trace2db = Trace2Db(sql_db_path)
222  # Speed optimization: Skip any entries that aren't mm_filemap_add_to_pagecache.
223  trace2db.set_raw_ftrace_entry_filter(\
224      lambda entry: entry['function'] == 'mm_filemap_add_to_page_cache')
225  # TODO: parse multiple trace files here.
226  parse_count = trace2db.parse_file_into_db(trace_file)
227
228  mm_filemap_add_to_page_cache_rows = query_add_to_page_cache(trace2db,
229                                                              trace_duration)
230  print("DONE. Parsed %d entries into sql db." %(len(mm_filemap_add_to_page_cache_rows)))
231
232  page_runs = page_cache_entries_to_runs(mm_filemap_add_to_page_cache_rows)
233  print("DONE. Converted %d entries" %(len(page_runs)))
234
235  # TODO: flags to select optimizations.
236  optimized_page_runs = optimize_page_runs(page_runs)
237  print("DONE. Optimized down to %d entries" %(len(optimized_page_runs)))
238
239  print("Build protobuf...")
240  trace_file = build_protobuf(optimized_page_runs, inode_table, filter)
241
242  print("Write protobuf to file...")
243  output_file = open(output_file, 'wb')
244  output_file.write(trace_file.SerializeToString())
245  output_file.close()
246
247  print("DONE")
248
249  # TODO: Silent running mode [no output except on error] for build runs.
250
251  return 0
252
253def main(argv):
254  parser = optparse.OptionParser(usage="Usage: %prog [options]", description="Compile systrace file into TraceFile.pb")
255  parser.add_option('-i', dest='inode_data_file', metavar='FILE',
256                    help='Read cached inode data from a file saved earlier with pagecache.py -d')
257  parser.add_option('-t', dest='trace_file', metavar='FILE',
258                    help='Path to systrace file (trace.html) that will be parsed')
259  parser.add_option('--perfetto-trace', dest='perfetto_trace_file',
260                    metavar='FILE',
261                    help='Path to perfetto trace that will be parsed')
262
263  parser.add_option('--db', dest='sql_db', metavar='FILE',
264                    help='Path to intermediate sqlite3 database [default: in-memory].')
265
266  parser.add_option('-f', dest='filter', action="append", default=[],
267                    help="Add file filter. All file entries not matching one of the filters are discarded.")
268
269  parser.add_option('-l', dest='launch_lock', action="store_true", default=False,
270                    help="Exclude all events not inside launch_lock")
271
272  parser.add_option('-o', dest='output_file', metavar='FILE',
273                    help='Output protobuf file')
274
275  parser.add_option('--duration', dest='trace_duration', action="store",
276                    type=int, help='The duration of trace in milliseconds.')
277
278  options, categories = parser.parse_args(argv[1:])
279
280  # TODO: OptionParser should have some flags to make these mandatory.
281  if not options.inode_data_file:
282    parser.error("-i is required")
283  if not options.trace_file and not options.perfetto_trace_file:
284    parser.error("one of -t or --perfetto-trace is required")
285  if options.trace_file and options.perfetto_trace_file:
286    parser.error("please enter either -t or --perfetto-trace, not both")
287  if not options.output_file:
288    parser.error("-o is required")
289
290  if options.launch_lock:
291    print("INFO: Launch lock flag (-l) enabled; filtering all events not inside launch_lock.")
292
293  inode_table = Inode2Filename.new_from_filename(options.inode_data_file)
294
295  sql_db_path = ":memory:"
296  if options.sql_db:
297    sql_db_path = options.sql_db
298
299  trace_duration = timedelta(milliseconds=options.trace_duration) if \
300    options.trace_duration is not None else None
301
302  # if the input is systrace
303  if options.trace_file:
304    return run(sql_db_path,
305               options.trace_file,
306               trace_duration,
307               options.output_file,
308               inode_table,
309               options.filter)
310
311  # if the input is perfetto trace
312  # TODO python 3.7 switch to using nullcontext
313  with tempfile.NamedTemporaryFile() as trace_file:
314    transform_perfetto_trace_to_systrace(options.perfetto_trace_file,
315                                         trace_file.name)
316    return run(sql_db_path,
317               trace_file.name,
318               trace_duration,
319               options.output_file,
320               inode_table,
321               options.filter)
322
323if __name__ == '__main__':
324  print(sys.argv)
325  sys.exit(main(sys.argv))
326