1#!/usr/bin/env python
2#
3# Copyright (C) 2015 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17"""Simpleperf runtest runner: run simpleperf runtests on host or on device.
18
19For a simpleperf runtest like one_function test, it contains following steps:
201. Run simpleperf record command to record simpleperf_runtest_one_function's
21   running samples, which is generated in perf.data.
222. Run simpleperf report command to parse perf.data, generate perf.report.
234. Parse perf.report and see if it matches expectation.
24
25The information of all runtests is stored in runtest.conf.
26"""
27
28import re
29import subprocess
30import sys
31import xml.etree.ElementTree as ET
32
33
34class CallTreeNode(object):
35
36  def __init__(self, name):
37    self.name = name
38    self.children = []
39
40  def add_child(self, child):
41    self.children.append(child)
42
43  def __str__(self):
44    return 'CallTreeNode:\n' + '\n'.join(self._dump(1))
45
46  def _dump(self, indent):
47    indent_str = '  ' * indent
48    strs = [indent_str + self.name]
49    for child in self.children:
50      strs.extend(child._dump(indent + 1))
51    return strs
52
53
54class Symbol(object):
55
56  def __init__(self, name, comm, overhead, children_overhead):
57    self.name = name
58    self.comm = comm
59    self.overhead = overhead
60    # children_overhead is the overhead sum of this symbol and functions
61    # called by this symbol.
62    self.children_overhead = children_overhead
63    self.call_tree = None
64
65  def set_call_tree(self, call_tree):
66    self.call_tree = call_tree
67
68  def __str__(self):
69    strs = []
70    strs.append('Symbol name=%s comm=%s overhead=%f children_overhead=%f' % (
71        self.name, self.comm, self.overhead, self.children_overhead))
72    if self.call_tree:
73      strs.append('\t%s' % self.call_tree)
74    return '\n'.join(strs)
75
76
77class SymbolOverheadRequirement(object):
78
79  def __init__(self, symbol_name=None, comm=None, min_overhead=None,
80               max_overhead=None):
81    self.symbol_name = symbol_name
82    self.comm = comm
83    self.min_overhead = min_overhead
84    self.max_overhead = max_overhead
85
86  def __str__(self):
87    strs = []
88    strs.append('SymbolOverheadRequirement')
89    if self.symbol_name is not None:
90      strs.append('symbol_name=%s' % self.symbol_name)
91    if self.comm is not None:
92      strs.append('comm=%s' % self.comm)
93    if self.min_overhead is not None:
94      strs.append('min_overhead=%f' % self.min_overhead)
95    if self.max_overhead is not None:
96      strs.append('max_overhead=%f' % self.max_overhead)
97    return ' '.join(strs)
98
99  def is_match(self, symbol):
100    if self.symbol_name is not None:
101      if self.symbol_name != symbol.name:
102        return False
103    if self.comm is not None:
104      if self.comm != symbol.comm:
105        return False
106    return True
107
108  def check_overhead(self, overhead):
109    if self.min_overhead is not None:
110      if self.min_overhead > overhead:
111        return False
112    if self.max_overhead is not None:
113      if self.max_overhead < overhead:
114        return False
115    return True
116
117
118class SymbolRelationRequirement(object):
119
120  def __init__(self, symbol_name, comm=None):
121    self.symbol_name = symbol_name
122    self.comm = comm
123    self.children = []
124
125  def add_child(self, child):
126    self.children.append(child)
127
128  def __str__(self):
129    return 'SymbolRelationRequirement:\n' + '\n'.join(self._dump(1))
130
131  def _dump(self, indent):
132    indent_str = '  ' * indent
133    strs = [indent_str + self.symbol_name +
134            (' ' + self.comm if self.comm else '')]
135    for child in self.children:
136      strs.extend(child._dump(indent + 1))
137    return strs
138
139  def is_match(self, symbol):
140    if symbol.name != self.symbol_name:
141      return False
142    if self.comm is not None:
143      if symbol.comm != self.comm:
144        return False
145    return True
146
147  def check_relation(self, call_tree):
148    if not call_tree:
149      return False
150    if self.symbol_name != call_tree.name:
151      return False
152    for child in self.children:
153      child_matched = False
154      for node in call_tree.children:
155        if child.check_relation(node):
156          child_matched = True
157          break
158      if not child_matched:
159        return False
160    return True
161
162
163class Test(object):
164
165  def __init__(
166          self,
167          test_name,
168          executable_name,
169          report_options,
170          symbol_overhead_requirements,
171          symbol_children_overhead_requirements,
172          symbol_relation_requirements):
173    self.test_name = test_name
174    self.executable_name = executable_name
175    self.report_options = report_options
176    self.symbol_overhead_requirements = symbol_overhead_requirements
177    self.symbol_children_overhead_requirements = (
178        symbol_children_overhead_requirements)
179    self.symbol_relation_requirements = symbol_relation_requirements
180
181  def __str__(self):
182    strs = []
183    strs.append('Test test_name=%s' % self.test_name)
184    strs.append('\texecutable_name=%s' % self.executable_name)
185    strs.append('\treport_options=%s' % (' '.join(self.report_options)))
186    strs.append('\tsymbol_overhead_requirements:')
187    for req in self.symbol_overhead_requirements:
188      strs.append('\t\t%s' % req)
189    strs.append('\tsymbol_children_overhead_requirements:')
190    for req in self.symbol_children_overhead_requirements:
191      strs.append('\t\t%s' % req)
192    strs.append('\tsymbol_relation_requirements:')
193    for req in self.symbol_relation_requirements:
194      strs.append('\t\t%s' % req)
195    return '\n'.join(strs)
196
197
198def load_config_file(config_file):
199  tests = []
200  tree = ET.parse(config_file)
201  root = tree.getroot()
202  assert root.tag == 'runtests'
203  for test in root:
204    assert test.tag == 'test'
205    test_name = test.attrib['name']
206    executable_name = None
207    report_options = []
208    symbol_overhead_requirements = []
209    symbol_children_overhead_requirements = []
210    symbol_relation_requirements = []
211    for test_item in test:
212      if test_item.tag == 'executable':
213        executable_name = test_item.attrib['name']
214      elif test_item.tag == 'report':
215        report_options = test_item.attrib['option'].split()
216      elif (test_item.tag == 'symbol_overhead' or
217              test_item.tag == 'symbol_children_overhead'):
218        for symbol_item in test_item:
219          assert symbol_item.tag == 'symbol'
220          symbol_name = None
221          if 'name' in symbol_item.attrib:
222            symbol_name = symbol_item.attrib['name']
223          comm = None
224          if 'comm' in symbol_item.attrib:
225            comm = symbol_item.attrib['comm']
226          overhead_min = None
227          if 'min' in symbol_item.attrib:
228            overhead_min = float(symbol_item.attrib['min'])
229          overhead_max = None
230          if 'max' in symbol_item.attrib:
231            overhead_max = float(symbol_item.attrib['max'])
232
233          if test_item.tag == 'symbol_overhead':
234            symbol_overhead_requirements.append(
235                SymbolOverheadRequirement(
236                    symbol_name,
237                    comm,
238                    overhead_min,
239                    overhead_max)
240            )
241          else:
242            symbol_children_overhead_requirements.append(
243                SymbolOverheadRequirement(
244                    symbol_name,
245                    comm,
246                    overhead_min,
247                    overhead_max))
248      elif test_item.tag == 'symbol_callgraph_relation':
249        for symbol_item in test_item:
250          req = load_symbol_relation_requirement(symbol_item)
251          symbol_relation_requirements.append(req)
252
253    tests.append(
254        Test(
255            test_name,
256            executable_name,
257            report_options,
258            symbol_overhead_requirements,
259            symbol_children_overhead_requirements,
260            symbol_relation_requirements))
261  return tests
262
263
264def load_symbol_relation_requirement(symbol_item):
265  symbol_name = symbol_item.attrib['name']
266  comm = None
267  if 'comm' in symbol_item.attrib:
268    comm = symbol_item.attrib['comm']
269  req = SymbolRelationRequirement(symbol_name, comm)
270  for item in symbol_item:
271    child_req = load_symbol_relation_requirement(item)
272    req.add_child(child_req)
273  return req
274
275
276class Runner(object):
277
278  def __init__(self, perf_path):
279    self.perf_path = perf_path
280
281  def record(self, test_executable_name, record_file, additional_options=[]):
282    call_args = [self.perf_path,
283                 'record'] + additional_options + ['-e',
284                                                   'cpu-cycles:u',
285                                                   '-o',
286                                                   record_file,
287                                                   test_executable_name]
288    self._call(call_args)
289
290  def report(self, record_file, report_file, additional_options=[]):
291    call_args = [self.perf_path,
292                 'report'] + additional_options + ['-i',
293                                                   record_file]
294    self._call(call_args, report_file)
295
296  def _call(self, args, output_file=None):
297    pass
298
299
300class HostRunner(Runner):
301
302  """Run perf test on host."""
303
304  def _call(self, args, output_file=None):
305    output_fh = None
306    if output_file is not None:
307      output_fh = open(output_file, 'w')
308    subprocess.check_call(args, stdout=output_fh)
309    if output_fh is not None:
310      output_fh.close()
311
312
313class DeviceRunner(Runner):
314
315  """Run perf test on device."""
316
317  def _call(self, args, output_file=None):
318    output_fh = None
319    if output_file is not None:
320      output_fh = open(output_file, 'w')
321    args_with_adb = ['adb', 'shell']
322    args_with_adb.extend(args)
323    subprocess.check_call(args_with_adb, stdout=output_fh)
324    if output_fh is not None:
325      output_fh.close()
326
327
328class ReportAnalyzer(object):
329
330  """Check if perf.report matches expectation in Configuration."""
331
332  def _read_report_file(self, report_file, has_callgraph):
333    fh = open(report_file, 'r')
334    lines = fh.readlines()
335    fh.close()
336
337    lines = [x.rstrip() for x in lines]
338    blank_line_index = -1
339    for i in range(len(lines)):
340      if not lines[i]:
341        blank_line_index = i
342    assert blank_line_index != -1
343    assert blank_line_index + 1 < len(lines)
344    title_line = lines[blank_line_index + 1]
345    report_item_lines = lines[blank_line_index + 2:]
346
347    if has_callgraph:
348      assert re.search(r'^Children\s+Self\s+Command.+Symbol$', title_line)
349    else:
350      assert re.search(r'^Overhead\s+Command.+Symbol$', title_line)
351
352    return self._parse_report_items(report_item_lines, has_callgraph)
353
354  def _parse_report_items(self, lines, has_callgraph):
355    symbols = []
356    cur_symbol = None
357    call_tree_stack = {}
358    vertical_columns = []
359    last_node = None
360    last_depth = -1
361
362    for line in lines:
363      if not line:
364        continue
365      if not line[0].isspace():
366        if has_callgraph:
367          m = re.search(r'^([\d\.]+)%\s+([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
368          children_overhead = float(m.group(1))
369          overhead = float(m.group(2))
370          comm = m.group(3)
371          symbol_name = m.group(4)
372          cur_symbol = Symbol(symbol_name, comm, overhead, children_overhead)
373          symbols.append(cur_symbol)
374        else:
375          m = re.search(r'^([\d\.]+)%\s+(\S+).*\s+(\S+)$', line)
376          overhead = float(m.group(1))
377          comm = m.group(2)
378          symbol_name = m.group(3)
379          cur_symbol = Symbol(symbol_name, comm, overhead, 0)
380          symbols.append(cur_symbol)
381        # Each report item can have different column depths.
382        vertical_columns = []
383      else:
384        for i in range(len(line)):
385          if line[i] == '|':
386            if not vertical_columns or vertical_columns[-1] < i:
387              vertical_columns.append(i)
388
389        if not line.strip('| \t'):
390          continue
391        if line.find('-') == -1:
392          function_name = line.strip('| \t')
393          node = CallTreeNode(function_name)
394          last_node.add_child(node)
395          last_node = node
396          call_tree_stack[last_depth] = node
397        else:
398          pos = line.find('-')
399          depth = -1
400          for i in range(len(vertical_columns)):
401            if pos >= vertical_columns[i]:
402              depth = i
403          assert depth != -1
404
405          line = line.strip('|- \t')
406          m = re.search(r'^[\d\.]+%[-\s]+(.+)$', line)
407          if m:
408            function_name = m.group(1)
409          else:
410            function_name = line
411
412          node = CallTreeNode(function_name)
413          if depth == 0:
414            cur_symbol.set_call_tree(node)
415
416          else:
417            call_tree_stack[depth - 1].add_child(node)
418          call_tree_stack[depth] = node
419          last_node = node
420          last_depth = depth
421
422    return symbols
423
424  def check_report_file(self, test, report_file, has_callgraph):
425    symbols = self._read_report_file(report_file, has_callgraph)
426    if not self._check_symbol_overhead_requirements(test, symbols):
427      return False
428    if has_callgraph:
429      if not self._check_symbol_children_overhead_requirements(test, symbols):
430        return False
431      if not self._check_symbol_relation_requirements(test, symbols):
432        return False
433    return True
434
435  def _check_symbol_overhead_requirements(self, test, symbols):
436    result = True
437    matched = [False] * len(test.symbol_overhead_requirements)
438    matched_overhead = [0] * len(test.symbol_overhead_requirements)
439    for symbol in symbols:
440      for i in range(len(test.symbol_overhead_requirements)):
441        req = test.symbol_overhead_requirements[i]
442        if req.is_match(symbol):
443          matched[i] = True
444          matched_overhead[i] += symbol.overhead
445    for i in range(len(matched)):
446      if not matched[i]:
447        print 'requirement (%s) has no matched symbol in test %s' % (
448            test.symbol_overhead_requirements[i], test)
449        result = False
450      else:
451        fulfilled = req.check_overhead(matched_overhead[i])
452        if not fulfilled:
453          print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
454              symbol, req, test)
455          result = False
456    return result
457
458  def _check_symbol_children_overhead_requirements(self, test, symbols):
459    result = True
460    matched = [False] * len(test.symbol_children_overhead_requirements)
461    for symbol in symbols:
462      for i in range(len(test.symbol_children_overhead_requirements)):
463        req = test.symbol_children_overhead_requirements[i]
464        if req.is_match(symbol):
465          matched[i] = True
466          fulfilled = req.check_overhead(symbol.children_overhead)
467          if not fulfilled:
468            print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
469                symbol, req, test)
470            result = False
471    for i in range(len(matched)):
472      if not matched[i]:
473        print 'requirement (%s) has no matched symbol in test %s' % (
474            test.symbol_children_overhead_requirements[i], test)
475        result = False
476    return result
477
478  def _check_symbol_relation_requirements(self, test, symbols):
479    result = True
480    matched = [False] * len(test.symbol_relation_requirements)
481    for symbol in symbols:
482      for i in range(len(test.symbol_relation_requirements)):
483        req = test.symbol_relation_requirements[i]
484        if req.is_match(symbol):
485          matched[i] = True
486          fulfilled = req.check_relation(symbol.call_tree)
487          if not fulfilled:
488            print "Symbol (%s) doesn't match requirement (%s) in test %s" % (
489                symbol, req, test)
490            result = False
491    for i in range(len(matched)):
492      if not matched[i]:
493        print 'requirement (%s) has no matched symbol in test %s' % (
494            test.symbol_relation_requirements[i], test)
495        result = False
496    return result
497
498
499def runtest(host, device, normal, callgraph, selected_tests):
500  tests = load_config_file('runtest.conf')
501  host_runner = HostRunner('simpleperf')
502  device_runner = DeviceRunner('simpleperf')
503  report_analyzer = ReportAnalyzer()
504  for test in tests:
505    if selected_tests is not None:
506      if test.test_name not in selected_tests:
507        continue
508    if host and normal:
509      host_runner.record(test.executable_name, 'perf.data')
510      host_runner.report('perf.data', 'perf.report',
511                         additional_options = test.report_options)
512      result = report_analyzer.check_report_file(
513          test, 'perf.report', False)
514      print 'test %s on host %s' % (
515          test.test_name, 'Succeeded' if result else 'Failed')
516      if not result:
517        exit(1)
518
519    if device and normal:
520      device_runner.record(test.executable_name, '/data/perf.data')
521      device_runner.report('/data/perf.data', 'perf.report',
522                           additional_options = test.report_options)
523      result = report_analyzer.check_report_file(test, 'perf.report', False)
524      print 'test %s on device %s' % (
525          test.test_name, 'Succeeded' if result else 'Failed')
526      if not result:
527        exit(1)
528
529    if host and callgraph:
530      host_runner.record(
531          test.executable_name,
532          'perf_g.data',
533          additional_options=['-g'])
534      host_runner.report(
535          'perf_g.data',
536          'perf_g.report',
537          additional_options=['-g'] + test.report_options)
538      result = report_analyzer.check_report_file(test, 'perf_g.report', True)
539      print 'call-graph test %s on host %s' % (
540          test.test_name, 'Succeeded' if result else 'Failed')
541      if not result:
542        exit(1)
543
544    if device and callgraph:
545      device_runner.record(
546          test.executable_name,
547          '/data/perf_g.data',
548          additional_options=['-g'])
549      device_runner.report(
550          '/data/perf_g.data',
551          'perf_g.report',
552          additional_options=['-g'] + test.report_options)
553      result = report_analyzer.check_report_file(test, 'perf_g.report', True)
554      print 'call-graph test %s on device %s' % (
555          test.test_name, 'Succeeded' if result else 'Failed')
556      if not result:
557        exit(1)
558
559def main():
560  host = True
561  device = True
562  normal = True
563  callgraph = True
564  selected_tests = None
565  i = 1
566  while i < len(sys.argv):
567    if sys.argv[i] == '--host':
568      host = True
569      device = False
570    elif sys.argv[i] == '--device':
571      host = False
572      device = True
573    elif sys.argv[i] == '--normal':
574      normal = True
575      callgraph = False
576    elif sys.argv[i] == '--callgraph':
577      normal = False
578      callgraph = True
579    elif sys.argv[i] == '--test':
580      if i < len(sys.argv):
581        i += 1
582        for test in sys.argv[i].split(','):
583          if selected_tests is None:
584            selected_tests = {}
585          selected_tests[test] = True
586    i += 1
587  runtest(host, device, normal, callgraph, selected_tests)
588
589if __name__ == '__main__':
590  main()
591