1#!/usr/bin/env python3.4
2#
3# Copyright (C) 2016 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#   http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import abc
18import argparse
19import filecmp
20import os
21import shlex
22import shutil
23import subprocess
24import sys
25
26from glob import glob
27from subprocess import DEVNULL
28from tempfile import mkdtemp
29
30sys.path.append(os.path.dirname(os.path.dirname(
31    os.path.realpath(__file__))))
32
33from common.common import RetCode
34from common.common import CommandListToCommandString
35from common.common import FatalError
36from common.common import GetJackClassPath
37from common.common import GetEnvVariableOrError
38from common.common import RunCommand
39from common.common import RunCommandForOutput
40from common.common import DeviceTestEnv
41
42# Return codes supported by bisection bug search.
43BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT)
44
45
46def GetExecutionModeRunner(dexer, debug_info, device, mode):
47  """Returns a runner for the given execution mode.
48
49  Args:
50    dexer: string, defines dexer
51    debug_info: boolean, if True include debugging info
52    device: string, target device serial number (or None)
53    mode: string, execution mode
54  Returns:
55    TestRunner with given execution mode
56  Raises:
57    FatalError: error for unknown execution mode
58  """
59  if mode == 'ri':
60    return TestRunnerRIOnHost(debug_info)
61  if mode == 'hint':
62    return TestRunnerArtIntOnHost(dexer, debug_info)
63  if mode == 'hopt':
64    return TestRunnerArtOptOnHost(dexer, debug_info)
65  if mode == 'tint':
66    return TestRunnerArtIntOnTarget(dexer, debug_info, device)
67  if mode == 'topt':
68    return TestRunnerArtOptOnTarget(dexer, debug_info, device)
69  raise FatalError('Unknown execution mode')
70
71
72#
73# Execution mode classes.
74#
75
76
77class TestRunner(object):
78  """Abstraction for running a test in a particular execution mode."""
79  __meta_class__ = abc.ABCMeta
80
81  @abc.abstractproperty
82  def description(self):
83    """Returns a description string of the execution mode."""
84
85  @abc.abstractproperty
86  def id(self):
87    """Returns a short string that uniquely identifies the execution mode."""
88
89  @property
90  def output_file(self):
91    return self.id + '_out.txt'
92
93  @abc.abstractmethod
94  def GetBisectionSearchArgs(self):
95    """Get arguments to pass to bisection search tool.
96
97    Returns:
98      list of strings - arguments for bisection search tool, or None if
99      runner is not bisectable
100    """
101
102  @abc.abstractmethod
103  def CompileAndRunTest(self):
104    """Compile and run the generated test.
105
106    Ensures that the current Test.java in the temporary directory is compiled
107    and executed under the current execution mode. On success, transfers the
108    generated output to the file self.output_file in the temporary directory.
109
110    Most nonzero return codes are assumed non-divergent, since systems may
111    exit in different ways. This is enforced by normalizing return codes.
112
113    Returns:
114      normalized return code
115    """
116
117
118class TestRunnerWithHostCompilation(TestRunner):
119  """Abstract test runner that supports compilation on host."""
120
121  def  __init__(self, dexer, debug_info):
122    """Constructor for the runner with host compilation.
123
124    Args:
125      dexer: string, defines dexer
126      debug_info: boolean, if True include debugging info
127    """
128    self._dexer = dexer
129    self._debug_info = debug_info
130    self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.',
131                       'Test.java']
132
133  def CompileOnHost(self):
134    if self._dexer == 'dx' or self._dexer == 'd8':
135      dbg = '-g' if self._debug_info else '-g:none'
136      if RunCommand(['javac', '--release=8', dbg, 'Test.java'],
137                    out=None, err=None, timeout=30) == RetCode.SUCCESS:
138        dx = 'dx' if self._dexer == 'dx' else 'd8-compat-dx'
139        retc = RunCommand([dx, '--dex', '--output=classes.dex'] + glob('*.class'),
140                          out=None, err='dxerr.txt', timeout=30)
141      else:
142        retc = RetCode.NOTCOMPILED
143    elif self._dexer == 'jack':
144      retc = RunCommand(['jack'] + self._jack_args,
145                        out=None, err='jackerr.txt', timeout=30)
146    else:
147      raise FatalError('Unknown dexer: ' + self._dexer)
148    return retc
149
150
151class TestRunnerRIOnHost(TestRunner):
152  """Concrete test runner of the reference implementation on host."""
153
154  def  __init__(self, debug_info):
155    """Constructor for the runner with host compilation.
156
157    Args:
158      debug_info: boolean, if True include debugging info
159    """
160    self._debug_info = debug_info
161
162  @property
163  def description(self):
164    return 'RI on host'
165
166  @property
167  def id(self):
168    return 'RI'
169
170  def CompileAndRunTest(self):
171    dbg = '-g' if self._debug_info else '-g:none'
172    if RunCommand(['javac', '--release=8', dbg, 'Test.java'],
173                  out=None, err=None, timeout=30) == RetCode.SUCCESS:
174      retc = RunCommand(['java', 'Test'], self.output_file, err=None)
175    else:
176      retc = RetCode.NOTCOMPILED
177    return retc
178
179  def GetBisectionSearchArgs(self):
180    return None
181
182
183class TestRunnerArtOnHost(TestRunnerWithHostCompilation):
184  """Abstract test runner of Art on host."""
185
186  def  __init__(self, dexer, debug_info, extra_args=None):
187    """Constructor for the Art on host tester.
188
189    Args:
190      dexer: string, defines dexer
191      debug_info: boolean, if True include debugging info
192      extra_args: list of strings, extra arguments for dalvikvm
193    """
194    super().__init__(dexer, debug_info)
195    self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex']
196    if extra_args is not None:
197      self._art_cmd += extra_args
198    self._art_cmd.append('Test')
199
200  def CompileAndRunTest(self):
201    if self.CompileOnHost() == RetCode.SUCCESS:
202      retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt')
203    else:
204      retc = RetCode.NOTCOMPILED
205    return retc
206
207
208class TestRunnerArtIntOnHost(TestRunnerArtOnHost):
209  """Concrete test runner of interpreter mode Art on host."""
210
211  def  __init__(self, dexer, debug_info):
212    """Constructor for the Art on host tester (interpreter).
213
214    Args:
215      dexer: string, defines dexer
216      debug_info: boolean, if True include debugging info
217   """
218    super().__init__(dexer, debug_info, ['-Xint'])
219
220  @property
221  def description(self):
222    return 'Art interpreter on host'
223
224  @property
225  def id(self):
226    return 'HInt'
227
228  def GetBisectionSearchArgs(self):
229    return None
230
231
232class TestRunnerArtOptOnHost(TestRunnerArtOnHost):
233  """Concrete test runner of optimizing compiler mode Art on host."""
234
235  def  __init__(self, dexer, debug_info):
236    """Constructor for the Art on host tester (optimizing).
237
238    Args:
239      dexer: string, defines dexer
240      debug_info: boolean, if True include debugging info
241   """
242    super().__init__(dexer, debug_info, None)
243
244  @property
245  def description(self):
246    return 'Art optimizing on host'
247
248  @property
249  def id(self):
250    return 'HOpt'
251
252  def GetBisectionSearchArgs(self):
253    cmd_str = CommandListToCommandString(
254        self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:])
255    return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
256
257
258class TestRunnerArtOnTarget(TestRunnerWithHostCompilation):
259  """Abstract test runner of Art on target."""
260
261  def  __init__(self, dexer, debug_info, device, extra_args=None):
262    """Constructor for the Art on target tester.
263
264    Args:
265      dexer: string, defines dexer
266      debug_info: boolean, if True include debugging info
267      device: string, target device serial number (or None)
268      extra_args: list of strings, extra arguments for dalvikvm
269    """
270    super().__init__(dexer, debug_info)
271    self._test_env = DeviceTestEnv('jfuzz_', specific_device=device)
272    self._dalvik_cmd = ['dalvikvm']
273    if extra_args is not None:
274      self._dalvik_cmd += extra_args
275    self._device = device
276    self._device_classpath = None
277
278  def CompileAndRunTest(self):
279    if self.CompileOnHost() == RetCode.SUCCESS:
280      self._device_classpath = self._test_env.PushClasspath('classes.dex')
281      cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']
282      (output, retc) = self._test_env.RunCommand(
283          cmd, {'ANDROID_LOG_TAGS': '*:s'})
284      with open(self.output_file, 'w') as run_out:
285        run_out.write(output)
286    else:
287      retc = RetCode.NOTCOMPILED
288    return retc
289
290  def GetBisectionSearchArgs(self):
291    cmd_str = CommandListToCommandString(
292        self._dalvik_cmd + ['-cp',self._device_classpath, 'Test'])
293    cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
294    if self._device:
295      cmd += ['--device-serial', self._device]
296    else:
297      cmd.append('--device')
298    return cmd
299
300
301class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget):
302  """Concrete test runner of interpreter mode Art on target."""
303
304  def  __init__(self, dexer, debug_info, device):
305    """Constructor for the Art on target tester (interpreter).
306
307    Args:
308      dexer: string, defines dexer
309      debug_info: boolean, if True include debugging info
310      device: string, target device serial number (or None)
311    """
312    super().__init__(dexer, debug_info, device, ['-Xint'])
313
314  @property
315  def description(self):
316    return 'Art interpreter on target'
317
318  @property
319  def id(self):
320    return 'TInt'
321
322  def GetBisectionSearchArgs(self):
323    return None
324
325
326class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget):
327  """Concrete test runner of optimizing compiler mode Art on target."""
328
329  def  __init__(self, dexer, debug_info, device):
330    """Constructor for the Art on target tester (optimizing).
331
332    Args:
333      dexer: string, defines dexer
334      debug_info: boolean, if True include debugging info
335      device: string, target device serial number (or None)
336    """
337    super().__init__(dexer, debug_info, device, None)
338
339  @property
340  def description(self):
341    return 'Art optimizing on target'
342
343  @property
344  def id(self):
345    return 'TOpt'
346
347  def GetBisectionSearchArgs(self):
348    cmd_str = CommandListToCommandString(
349        self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'])
350    cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
351    if self._device:
352      cmd += ['--device-serial', self._device]
353    else:
354      cmd.append('--device')
355    return cmd
356
357
358#
359# Tester class.
360#
361
362
363class JFuzzTester(object):
364  """Tester that runs JFuzz many times and report divergences."""
365
366  def  __init__(self, num_tests, device, mode1, mode2, jfuzz_args,
367                report_script, true_divergence_only, dexer, debug_info):
368    """Constructor for the tester.
369
370    Args:
371      num_tests: int, number of tests to run
372      device: string, target device serial number (or None)
373      mode1: string, execution mode for first runner
374      mode2: string, execution mode for second runner
375      jfuzz_args: list of strings, additional arguments for jfuzz
376      report_script: string, path to script called for each divergence
377      true_divergence_only: boolean, if True don't bisect timeout divergences
378      dexer: string, defines dexer
379      debug_info: boolean, if True include debugging info
380    """
381    self._num_tests = num_tests
382    self._device = device
383    self._runner1 = GetExecutionModeRunner(dexer, debug_info, device, mode1)
384    self._runner2 = GetExecutionModeRunner(dexer, debug_info, device, mode2)
385    self._jfuzz_args = jfuzz_args
386    self._report_script = report_script
387    self._true_divergence_only = true_divergence_only
388    self._dexer = dexer
389    self._debug_info = debug_info
390    self._save_dir = None
391    self._results_dir = None
392    self._jfuzz_dir = None
393    # Statistics.
394    self._test = 0
395    self._num_success = 0
396    self._num_not_compiled = 0
397    self._num_not_run = 0
398    self._num_timed_out = 0
399    self._num_divergences = 0
400
401  def __enter__(self):
402    """On entry, enters new temp directory after saving current directory.
403
404    Raises:
405      FatalError: error when temp directory cannot be constructed
406    """
407    self._save_dir = os.getcwd()
408    self._results_dir = mkdtemp(dir='/tmp/')
409    self._jfuzz_dir = mkdtemp(dir=self._results_dir)
410    if self._results_dir is None or self._jfuzz_dir is None:
411      raise FatalError('Cannot obtain temp directory')
412    os.chdir(self._jfuzz_dir)
413    return self
414
415  def __exit__(self, etype, evalue, etraceback):
416    """On exit, re-enters previously saved current directory and cleans up."""
417    os.chdir(self._save_dir)
418    shutil.rmtree(self._jfuzz_dir)
419    if self._num_divergences == 0:
420      shutil.rmtree(self._results_dir)
421
422  def Run(self):
423    """Runs JFuzz many times and report divergences."""
424    print()
425    print('**\n**** JFuzz Testing\n**')
426    print()
427    print('#Tests    :', self._num_tests)
428    print('Device    :', self._device)
429    print('Directory :', self._results_dir)
430    print('Exec-mode1:', self._runner1.description)
431    print('Exec-mode2:', self._runner2.description)
432    print('Dexer     :', self._dexer)
433    print('Debug-info:', self._debug_info)
434    print()
435    self.ShowStats()
436    for self._test in range(1, self._num_tests + 1):
437      self.RunJFuzzTest()
438      self.ShowStats()
439    if self._num_divergences == 0:
440      print('\n\nsuccess (no divergences)\n')
441    else:
442      print('\n\nfailure (divergences)\n')
443
444  def ShowStats(self):
445    """Shows current statistics (on same line) while tester is running."""
446    print('\rTests:', self._test,
447          'Success:', self._num_success,
448          'Not-compiled:', self._num_not_compiled,
449          'Not-run:', self._num_not_run,
450          'Timed-out:', self._num_timed_out,
451          'Divergences:', self._num_divergences,
452          end='')
453    sys.stdout.flush()
454
455  def RunJFuzzTest(self):
456    """Runs a single JFuzz test, comparing two execution modes."""
457    self.ConstructTest()
458    retc1 = self._runner1.CompileAndRunTest()
459    retc2 = self._runner2.CompileAndRunTest()
460    self.CheckForDivergence(retc1, retc2)
461    self.CleanupTest()
462
463  def ConstructTest(self):
464    """Use JFuzz to generate next Test.java test.
465
466    Raises:
467      FatalError: error when jfuzz fails
468    """
469    if (RunCommand(['jfuzz'] + self._jfuzz_args, out='Test.java', err=None)
470          != RetCode.SUCCESS):
471      raise FatalError('Unexpected error while running JFuzz')
472
473  def CheckForDivergence(self, retc1, retc2):
474    """Checks for divergences and updates statistics.
475
476    Args:
477      retc1: int, normalized return code of first runner
478      retc2: int, normalized return code of second runner
479    """
480    if retc1 == retc2:
481      # No divergence in return code.
482      if retc1 == RetCode.SUCCESS:
483        # Both compilations and runs were successful, inspect generated output.
484        runner1_out = self._runner1.output_file
485        runner2_out = self._runner2.output_file
486        if not filecmp.cmp(runner1_out, runner2_out, shallow=False):
487          # Divergence in output.
488          self.ReportDivergence(retc1, retc2, is_output_divergence=True)
489        else:
490          # No divergence in output.
491          self._num_success += 1
492      elif retc1 == RetCode.TIMEOUT:
493        self._num_timed_out += 1
494      elif retc1 == RetCode.NOTCOMPILED:
495        self._num_not_compiled += 1
496      else:
497        self._num_not_run += 1
498    else:
499      # Divergence in return code.
500      if self._true_divergence_only:
501        # When only true divergences are requested, any divergence in return
502        # code where one is a time out is treated as a regular time out.
503        if RetCode.TIMEOUT in (retc1, retc2):
504          self._num_timed_out += 1
505          return
506        # When only true divergences are requested, a runtime crash in just
507        # the RI is treated as if not run at all.
508        if retc1 == RetCode.ERROR and retc2 == RetCode.SUCCESS:
509          if self._runner1.GetBisectionSearchArgs() is None:
510            self._num_not_run += 1
511            return
512      self.ReportDivergence(retc1, retc2, is_output_divergence=False)
513
514  def GetCurrentDivergenceDir(self):
515    return self._results_dir + '/divergence' + str(self._num_divergences)
516
517  def ReportDivergence(self, retc1, retc2, is_output_divergence):
518    """Reports and saves a divergence."""
519    self._num_divergences += 1
520    print('\n' + str(self._num_divergences), end='')
521    if is_output_divergence:
522      print(' divergence in output')
523    else:
524      print(' divergence in return code: ' + retc1.name + ' vs. ' +
525            retc2.name)
526    # Save.
527    ddir = self.GetCurrentDivergenceDir()
528    os.mkdir(ddir)
529    for f in glob('*.txt') + ['Test.java']:
530      shutil.copy(f, ddir)
531    # Maybe run bisection bug search.
532    if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES:
533      self.MaybeBisectDivergence(retc1, retc2, is_output_divergence)
534    # Call reporting script.
535    if self._report_script:
536      self.RunReportScript(retc1, retc2, is_output_divergence)
537
538  def RunReportScript(self, retc1, retc2, is_output_divergence):
539    """Runs report script."""
540    try:
541      title = "Divergence between {0} and {1} (found with fuzz testing)".format(
542          self._runner1.description, self._runner2.description)
543      # Prepare divergence comment.
544      jfuzz_cmd_and_version = subprocess.check_output(
545          ['grep', '-o', 'jfuzz.*', 'Test.java'], universal_newlines=True)
546      (jfuzz_cmd_str, jfuzz_ver) = jfuzz_cmd_and_version.split('(')
547      # Strip right parenthesis and new line.
548      jfuzz_ver = jfuzz_ver[:-2]
549      jfuzz_args = ['\'-{0}\''.format(arg)
550                    for arg in jfuzz_cmd_str.strip().split(' -')][1:]
551      wrapped_args = ['--jfuzz_arg={0}'.format(opt) for opt in jfuzz_args]
552      repro_cmd_str = (os.path.basename(__file__) +
553                       ' --num_tests=1 --dexer=' + self._dexer +
554                       (' --debug_info ' if self._debug_info else ' ') +
555                       ' '.join(wrapped_args))
556      comment = 'jfuzz {0}\nReproduce test:\n{1}\nReproduce divergence:\n{2}\n'.format(
557          jfuzz_ver, jfuzz_cmd_str, repro_cmd_str)
558      if is_output_divergence:
559        (output, _, _) = RunCommandForOutput(
560            ['diff', self._runner1.output_file, self._runner2.output_file],
561            None, subprocess.PIPE, subprocess.STDOUT)
562        comment += 'Diff:\n' + output
563      else:
564        comment += '{0} vs {1}\n'.format(retc1, retc2)
565      # Prepare report script command.
566      script_cmd = [self._report_script, title, comment]
567      ddir = self.GetCurrentDivergenceDir()
568      bisection_out_files = glob(ddir + '/*_bisection_out.txt')
569      if bisection_out_files:
570        script_cmd += ['--bisection_out', bisection_out_files[0]]
571      subprocess.check_call(script_cmd, stdout=DEVNULL, stderr=DEVNULL)
572    except subprocess.CalledProcessError as err:
573      print('Failed to run report script.\n', err)
574
575  def RunBisectionSearch(self, args, expected_retcode, expected_output,
576                         runner_id):
577    ddir = self.GetCurrentDivergenceDir()
578    outfile_path = ddir + '/' + runner_id + '_bisection_out.txt'
579    logfile_path = ddir + '/' + runner_id + '_bisection_log.txt'
580    errfile_path = ddir + '/' + runner_id + '_bisection_err.txt'
581    args = list(args) + ['--logfile', logfile_path, '--cleanup']
582    args += ['--expected-retcode', expected_retcode.name]
583    if expected_output:
584      args += ['--expected-output', expected_output]
585    bisection_search_path = os.path.join(
586        GetEnvVariableOrError('ANDROID_BUILD_TOP'),
587        'art/tools/bisection_search/bisection_search.py')
588    if RunCommand([bisection_search_path] + args, out=outfile_path,
589                  err=errfile_path, timeout=300) == RetCode.TIMEOUT:
590      print('Bisection search TIMEOUT')
591
592  def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence):
593    bisection_args1 = self._runner1.GetBisectionSearchArgs()
594    bisection_args2 = self._runner2.GetBisectionSearchArgs()
595    if is_output_divergence:
596      maybe_output1 = self._runner1.output_file
597      maybe_output2 = self._runner2.output_file
598    else:
599      maybe_output1 = maybe_output2 = None
600    if bisection_args1 is not None:
601      self.RunBisectionSearch(bisection_args1, retc2, maybe_output2,
602                              self._runner1.id)
603    if bisection_args2 is not None:
604      self.RunBisectionSearch(bisection_args2, retc1, maybe_output1,
605                              self._runner2.id)
606
607  def CleanupTest(self):
608    """Cleans up after a single test run."""
609    for file_name in os.listdir(self._jfuzz_dir):
610      file_path = os.path.join(self._jfuzz_dir, file_name)
611      if os.path.isfile(file_path):
612        os.unlink(file_path)
613      elif os.path.isdir(file_path):
614        shutil.rmtree(file_path)
615
616
617def main():
618  # Handle arguments.
619  parser = argparse.ArgumentParser()
620  parser.add_argument('--num_tests', default=10000, type=int,
621                      help='number of tests to run')
622  parser.add_argument('--device', help='target device serial number')
623  parser.add_argument('--mode1', default='ri',
624                      help='execution mode 1 (default: ri)')
625  parser.add_argument('--mode2', default='hopt',
626                      help='execution mode 2 (default: hopt)')
627  parser.add_argument('--report_script',
628                      help='script called for each divergence')
629  parser.add_argument('--jfuzz_arg', default=[], dest='jfuzz_args',
630                      action='append',
631                      help='argument for jfuzz')
632  parser.add_argument('--true_divergence', default=False, action='store_true',
633                      help='do not bisect timeout divergences')
634  parser.add_argument('--dexer', default='dx', type=str,
635                      help='defines dexer as dx, d8, or jack (default: dx)')
636  parser.add_argument('--debug_info', default=False, action='store_true',
637                      help='include debugging info')
638  args = parser.parse_args()
639  if args.mode1 == args.mode2:
640    raise FatalError('Identical execution modes given')
641  # Run the JFuzz tester.
642  with JFuzzTester(args.num_tests,
643                   args.device,
644                   args.mode1, args.mode2,
645                   args.jfuzz_args,
646                   args.report_script,
647                   args.true_divergence,
648                   args.dexer,
649                   args.debug_info) as fuzzer:
650    fuzzer.Run()
651
652if __name__ == '__main__':
653  main()
654