1#!/usr/bin/env python3
2#
3# Copyright 2018, The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#     http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17#
18#
19# Measure application start-up time by launching applications under various combinations.
20# See --help for more details.
21#
22#
23# Sample usage:
24# $> ./app_startup_runner.py -p com.google.android.calculator -r warm -r cold -lc 10  -o out.csv
25# $> ./analyze_metrics.py out.csv
26#
27#
28
29import argparse
30import csv
31import itertools
32import os
33import sys
34import tempfile
35from datetime import timedelta
36from typing import Any, Callable, Iterable, List, NamedTuple, TextIO, Tuple, \
37    TypeVar, Union, Optional
38
39# local import
40DIR = os.path.abspath(os.path.dirname(__file__))
41sys.path.append(os.path.dirname(DIR))
42import lib.cmd_utils as cmd_utils
43import lib.print_utils as print_utils
44from app_startup.run_app_with_prefetch import PrefetchAppRunner
45import app_startup.lib.args_utils as args_utils
46from app_startup.lib.data_frame import DataFrame
47from app_startup.lib.perfetto_trace_collector import PerfettoTraceCollector
48from iorap.compiler import CompilerType
49import iorap.compiler as compiler
50
51# The following command line options participate in the combinatorial generation.
52# All other arguments have a global effect.
53_COMBINATORIAL_OPTIONS = ['package', 'readahead', 'compiler_filter',
54                          'activity', 'trace_duration']
55_TRACING_READAHEADS = ['mlock', 'fadvise']
56_FORWARD_OPTIONS = {'loop_count': '--count'}
57_RUN_SCRIPT = os.path.join(os.path.dirname(os.path.realpath(__file__)),
58                           'run_app_with_prefetch.py')
59
60CollectorPackageInfo = NamedTuple('CollectorPackageInfo',
61                                  [('package', str), ('compiler_filter', str)])
62# by 2; systrace starts up slowly.
63
64_UNLOCK_SCREEN_SCRIPT = os.path.join(
65    os.path.dirname(os.path.realpath(__file__)), 'unlock_screen')
66
67RunCommandArgs = NamedTuple('RunCommandArgs',
68                            [('package', str),
69                             ('readahead', str),
70                             ('activity', Optional[str]),
71                             ('compiler_filter', Optional[str]),
72                             ('timeout', Optional[int]),
73                             ('debug', bool),
74                             ('simulate', bool),
75                             ('input', Optional[str]),
76                             ('trace_duration', Optional[timedelta])])
77
78# This must be the only mutable global variable. All other global variables are constants to avoid magic literals.
79_debug = False  # See -d/--debug flag.
80_DEBUG_FORCE = None  # Ignore -d/--debug if this is not none.
81_PERFETTO_TRACE_DURATION_MS = 5000 # milliseconds
82_PERFETTO_TRACE_DURATION = timedelta(milliseconds=_PERFETTO_TRACE_DURATION_MS)
83
84# Type hinting names.
85T = TypeVar('T')
86NamedTupleMeta = Callable[
87    ..., T]  # approximation of a (S : NamedTuple<T> where S() == T) metatype.
88
89def parse_options(argv: List[str] = None):
90  """Parse command line arguments and return an argparse Namespace object."""
91  parser = argparse.ArgumentParser(description="Run one or more Android "
92                                               "applications under various "
93                                               "settings in order to measure "
94                                               "startup time.")
95  # argparse considers args starting with - and -- optional in --help, even though required=True.
96  # by using a named argument group --help will clearly say that it's required instead of optional.
97  required_named = parser.add_argument_group('required named arguments')
98  required_named.add_argument('-p', '--package', action='append',
99                              dest='packages',
100                              help='package of the application', required=True)
101  required_named.add_argument('-r', '--readahead', action='append',
102                              dest='readaheads',
103                              help='which readahead mode to use',
104                              choices=('warm', 'cold', 'mlock', 'fadvise'),
105                              required=True)
106
107  # optional arguments
108  # use a group here to get the required arguments to appear 'above' the optional arguments in help.
109  optional_named = parser.add_argument_group('optional named arguments')
110  optional_named.add_argument('-c', '--compiler-filter', action='append',
111                              dest='compiler_filters',
112                              help='which compiler filter to use. if omitted it does not enforce the app\'s compiler filter',
113                              choices=('speed', 'speed-profile', 'quicken'))
114  optional_named.add_argument('-s', '--simulate', dest='simulate',
115                              action='store_true',
116                              help='Print which commands will run, but don\'t run the apps')
117  optional_named.add_argument('-d', '--debug', dest='debug',
118                              action='store_true',
119                              help='Add extra debugging output')
120  optional_named.add_argument('-o', '--output', dest='output', action='store',
121                              help='Write CSV output to file.')
122  optional_named.add_argument('-t', '--timeout', dest='timeout', action='store',
123                              type=int, default=10,
124                              help='Timeout after this many seconds when executing a single run.')
125  optional_named.add_argument('-lc', '--loop-count', dest='loop_count',
126                              default=1, type=int, action='store',
127                              help='How many times to loop a single run.')
128  optional_named.add_argument('-in', '--inodes', dest='inodes', type=str,
129                              action='store',
130                              help='Path to inodes file (system/extras/pagecache/pagecache.py -d inodes)')
131  optional_named.add_argument('--compiler-trace-duration-ms',
132                              dest='trace_duration',
133                              type=lambda ms_str: timedelta(milliseconds=int(ms_str)),
134                              action='append',
135                              help='The trace duration (milliseconds) in '
136                                   'compilation')
137  optional_named.add_argument('--compiler-type', dest='compiler_type',
138                              type=CompilerType, choices=list(CompilerType),
139                              default=CompilerType.DEVICE,
140                              help='The type of compiler.')
141
142  return parser.parse_args(argv)
143
144def key_to_cmdline_flag(key: str) -> str:
145  """Convert key into a command line flag, e.g. 'foo-bars' -> '--foo-bar' """
146  if key.endswith("s"):
147    key = key[:-1]
148  return "--" + key.replace("_", "-")
149
150def as_run_command(tpl: NamedTuple) -> List[Union[str, Any]]:
151  """
152  Convert a named tuple into a command-line compatible arguments list.
153
154  Example: ABC(1, 2, 3) -> ['--a', 1, '--b', 2, '--c', 3]
155  """
156  args = []
157  for key, value in tpl._asdict().items():
158    if value is None:
159      continue
160    args.append(key_to_cmdline_flag(key))
161    args.append(value)
162  return args
163
164def run_perfetto_collector(collector_info: CollectorPackageInfo,
165                           timeout: int,
166                           simulate: bool) -> Tuple[bool, TextIO]:
167  """Run collector to collect prefetching trace.
168
169  Returns:
170    A tuple of whether the collection succeeds and the generated trace file.
171  """
172  tmp_output_file = tempfile.NamedTemporaryFile()
173
174  collector = PerfettoTraceCollector(package=collector_info.package,
175                                     activity=None,
176                                     compiler_filter=collector_info.compiler_filter,
177                                     timeout=timeout,
178                                     simulate=simulate,
179                                     trace_duration=_PERFETTO_TRACE_DURATION,
180                                     save_destination_file_path=tmp_output_file.name)
181  result = collector.run()
182
183  return result is not None, tmp_output_file
184
185def parse_run_script_csv_file(csv_file: TextIO) -> DataFrame:
186  """Parse a CSV file full of integers into a DataFrame."""
187  csv_reader = csv.reader(csv_file)
188
189  try:
190    header_list = next(csv_reader)
191  except StopIteration:
192    header_list = []
193
194  if not header_list:
195    return None
196
197  headers = [i for i in header_list]
198
199  d = {}
200  for row in csv_reader:
201    header_idx = 0
202
203    for i in row:
204      v = i
205      if i:
206        v = int(i)
207
208      header_key = headers[header_idx]
209      l = d.get(header_key, [])
210      l.append(v)
211      d[header_key] = l
212
213      header_idx = header_idx + 1
214
215  return DataFrame(d)
216
217def build_ri_compiler_argv(inodes_path: str,
218                           perfetto_trace_file: str,
219                           trace_duration: Optional[timedelta]
220                           ) -> str:
221  argv = ['-i', inodes_path, '--perfetto-trace',
222          perfetto_trace_file]
223
224  if trace_duration is not None:
225    argv += ['--duration', str(int(trace_duration.total_seconds()
226                                   * PerfettoTraceCollector.MS_PER_SEC))]
227
228  print_utils.debug_print(argv)
229  return argv
230
231def execute_run_using_perfetto_trace(collector_info,
232                                     run_combos: Iterable[RunCommandArgs],
233                                     simulate: bool,
234                                     inodes_path: str,
235                                     timeout: int,
236                                     compiler_type: CompilerType,
237                                     requires_trace_collection: bool) -> DataFrame:
238  """ Executes run based on perfetto trace. """
239  if requires_trace_collection:
240    passed, perfetto_trace_file = run_perfetto_collector(collector_info,
241                                                         timeout,
242                                                         simulate)
243    if not passed:
244      raise RuntimeError('Cannot run perfetto collector!')
245  else:
246    perfetto_trace_file = tempfile.NamedTemporaryFile()
247
248  with perfetto_trace_file:
249    for combos in run_combos:
250      if combos.readahead in _TRACING_READAHEADS:
251        if simulate:
252          compiler_trace_file = tempfile.NamedTemporaryFile()
253        else:
254          ri_compiler_argv = build_ri_compiler_argv(inodes_path,
255                                                    perfetto_trace_file.name,
256                                                    combos.trace_duration)
257          compiler_trace_file = compiler.compile(compiler_type,
258                                                 inodes_path,
259                                                 ri_compiler_argv,
260                                                 combos.package,
261                                                 combos.activity)
262
263        with compiler_trace_file:
264          combos = combos._replace(input=compiler_trace_file.name)
265          print_utils.debug_print(combos)
266          output = PrefetchAppRunner(**combos._asdict()).run()
267      else:
268        print_utils.debug_print(combos)
269        output = PrefetchAppRunner(**combos._asdict()).run()
270
271      yield DataFrame(dict((x, [y]) for x, y in output)) if output else None
272
273def execute_run_combos(
274    grouped_run_combos: Iterable[Tuple[CollectorPackageInfo, Iterable[RunCommandArgs]]],
275    simulate: bool,
276    inodes_path: str,
277    timeout: int,
278    compiler_type: CompilerType,
279    requires_trace_collection: bool):
280  # nothing will work if the screen isn't unlocked first.
281  cmd_utils.execute_arbitrary_command([_UNLOCK_SCREEN_SCRIPT],
282                                      timeout,
283                                      simulate=simulate,
284                                      shell=False)
285
286  for collector_info, run_combos in grouped_run_combos:
287    yield from execute_run_using_perfetto_trace(collector_info,
288                                                run_combos,
289                                                simulate,
290                                                inodes_path,
291                                                timeout,
292                                                compiler_type,
293                                                requires_trace_collection)
294
295def gather_results(commands: Iterable[Tuple[DataFrame]],
296                   key_list: List[str], value_list: List[Tuple[str, ...]]):
297  print_utils.debug_print("gather_results: key_list = ", key_list)
298  stringify_none = lambda s: s is None and "<none>" or s
299  #  yield key_list + ["time(ms)"]
300  for (run_result_list, values) in itertools.zip_longest(commands, value_list):
301    print_utils.debug_print("run_result_list = ", run_result_list)
302    print_utils.debug_print("values = ", values)
303
304    if not run_result_list:
305      continue
306
307    # RunCommandArgs(package='com.whatever', readahead='warm', compiler_filter=None)
308    # -> {'package':['com.whatever'], 'readahead':['warm'], 'compiler_filter':[None]}
309    values_dict = {}
310    for k, v in values._asdict().items():
311      if not k in key_list:
312        continue
313      values_dict[k] = [stringify_none(v)]
314
315    values_df = DataFrame(values_dict)
316    # project 'values_df' to be same number of rows as run_result_list.
317    values_df = values_df.repeat(run_result_list.data_row_len)
318
319    # the results are added as right-hand-side columns onto the existing labels for the table.
320    values_df.merge_data_columns(run_result_list)
321
322    yield values_df
323
324def eval_and_save_to_csv(output, annotated_result_values):
325  printed_header = False
326
327  csv_writer = csv.writer(output)
328  for row in annotated_result_values:
329    if not printed_header:
330      headers = row.headers
331      csv_writer.writerow(headers)
332      printed_header = True
333      # TODO: what about when headers change?
334
335    for data_row in row.data_table:
336      data_row = [d for d in data_row]
337      csv_writer.writerow(data_row)
338
339    output.flush()  # see the output live.
340
341def coerce_to_list(opts: dict):
342  """Tranform values of the dictionary to list.
343  For example:
344  1 -> [1], None -> [None], [1,2,3] -> [1,2,3]
345  [[1],[2]] -> [[1],[2]], {1:1, 2:2} -> [{1:1, 2:2}]
346  """
347  result = {}
348  for key in opts:
349    val = opts[key]
350    result[key] = val if issubclass(type(val), list) else [val]
351  return result
352
353def main():
354  global _debug
355
356  opts = parse_options()
357  _debug = opts.debug
358  if _DEBUG_FORCE is not None:
359    _debug = _DEBUG_FORCE
360
361  print_utils.DEBUG = _debug
362  cmd_utils.SIMULATE = opts.simulate
363
364  print_utils.debug_print("parsed options: ", opts)
365
366  output_file = opts.output and open(opts.output, 'w') or sys.stdout
367
368  combos = lambda: args_utils.generate_run_combinations(
369      RunCommandArgs,
370      coerce_to_list(vars(opts)),
371      opts.loop_count)
372  print_utils.debug_print_gen("run combinations: ", combos())
373
374  grouped_combos = lambda: args_utils.generate_group_run_combinations(combos(),
375                                                                      CollectorPackageInfo)
376
377  print_utils.debug_print_gen("grouped run combinations: ", grouped_combos())
378  requires_trace_collection = any(i in _TRACING_READAHEADS for i in opts.readaheads)
379  exec = execute_run_combos(grouped_combos(),
380                            opts.simulate,
381                            opts.inodes,
382                            opts.timeout,
383                            opts.compiler_type,
384                            requires_trace_collection)
385
386  results = gather_results(exec, _COMBINATORIAL_OPTIONS, combos())
387
388  eval_and_save_to_csv(output_file, results)
389
390  return 1
391
392if __name__ == '__main__':
393  sys.exit(main())
394