1#!/usr/bin/env python3.4
2#
3# Copyright (C) 2016 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#   http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import abc
18import argparse
19import filecmp
20import os
21import shlex
22import shutil
23import subprocess
24import sys
25
26from glob import glob
27from subprocess import DEVNULL
28from tempfile import mkdtemp
29
30sys.path.append(os.path.dirname(os.path.dirname(
31    os.path.realpath(__file__))))
32
33from common.common import RetCode
34from common.common import CommandListToCommandString
35from common.common import FatalError
36from common.common import GetJackClassPath
37from common.common import GetEnvVariableOrError
38from common.common import RunCommand
39from common.common import RunCommandForOutput
40from common.common import DeviceTestEnv
41
42# Return codes supported by bisection bug search.
43BISECTABLE_RET_CODES = (RetCode.SUCCESS, RetCode.ERROR, RetCode.TIMEOUT)
44
45
46def GetExecutionModeRunner(use_dx, device, mode):
47  """Returns a runner for the given execution mode.
48
49  Args:
50    use_dx: boolean, if True use dx rather than jack
51    device: string, target device serial number (or None)
52    mode: string, execution mode
53  Returns:
54    TestRunner with given execution mode
55  Raises:
56    FatalError: error for unknown execution mode
57  """
58  if mode == 'ri':
59    return TestRunnerRIOnHost()
60  if mode == 'hint':
61    return TestRunnerArtIntOnHost(use_dx)
62  if mode == 'hopt':
63    return TestRunnerArtOptOnHost(use_dx)
64  if mode == 'tint':
65    return TestRunnerArtIntOnTarget(use_dx, device)
66  if mode == 'topt':
67    return TestRunnerArtOptOnTarget(use_dx, device)
68  raise FatalError('Unknown execution mode')
69
70
71#
72# Execution mode classes.
73#
74
75
76class TestRunner(object):
77  """Abstraction for running a test in a particular execution mode."""
78  __meta_class__ = abc.ABCMeta
79
80  @abc.abstractproperty
81  def description(self):
82    """Returns a description string of the execution mode."""
83
84  @abc.abstractproperty
85  def id(self):
86    """Returns a short string that uniquely identifies the execution mode."""
87
88  @property
89  def output_file(self):
90    return self.id + '_out.txt'
91
92  @abc.abstractmethod
93  def GetBisectionSearchArgs(self):
94    """Get arguments to pass to bisection search tool.
95
96    Returns:
97      list of strings - arguments for bisection search tool, or None if
98      runner is not bisectable
99    """
100
101  @abc.abstractmethod
102  def CompileAndRunTest(self):
103    """Compile and run the generated test.
104
105    Ensures that the current Test.java in the temporary directory is compiled
106    and executed under the current execution mode. On success, transfers the
107    generated output to the file self.output_file in the temporary directory.
108
109    Most nonzero return codes are assumed non-divergent, since systems may
110    exit in different ways. This is enforced by normalizing return codes.
111
112    Returns:
113      normalized return code
114    """
115
116
117class TestRunnerWithHostCompilation(TestRunner):
118  """Abstract test runner that supports compilation on host."""
119
120  def  __init__(self, use_dx):
121    """Constructor for the runner with host compilation.
122
123    Args:
124      use_dx: boolean, if True use dx rather than jack
125    """
126    self._jack_args = ['-cp', GetJackClassPath(), '--output-dex', '.',
127                       'Test.java']
128    self._use_dx = use_dx
129
130  def CompileOnHost(self):
131    if self._use_dx:
132      if RunCommand(['javac', 'Test.java'],
133                    out=None, err=None, timeout=30) == RetCode.SUCCESS:
134        retc = RunCommand(['dx', '--dex', '--output=classes.dex'] + glob('*.class'),
135                          out=None, err='dxerr.txt', timeout=30)
136      else:
137        retc = RetCode.NOTCOMPILED
138    else:
139      retc = RunCommand(['jack'] + self._jack_args,
140                        out=None, err='jackerr.txt', timeout=30)
141    return retc
142
143
144class TestRunnerRIOnHost(TestRunner):
145  """Concrete test runner of the reference implementation on host."""
146
147  @property
148  def description(self):
149    return 'RI on host'
150
151  @property
152  def id(self):
153    return 'RI'
154
155  def CompileAndRunTest(self):
156    if RunCommand(['javac', 'Test.java'],
157                  out=None, err=None, timeout=30) == RetCode.SUCCESS:
158      retc = RunCommand(['java', 'Test'], self.output_file, err=None)
159    else:
160      retc = RetCode.NOTCOMPILED
161    return retc
162
163  def GetBisectionSearchArgs(self):
164    return None
165
166
167class TestRunnerArtOnHost(TestRunnerWithHostCompilation):
168  """Abstract test runner of Art on host."""
169
170  def  __init__(self, use_dx, extra_args=None):
171    """Constructor for the Art on host tester.
172
173    Args:
174      use_dx: boolean, if True use dx rather than jack
175      extra_args: list of strings, extra arguments for dalvikvm
176    """
177    super().__init__(use_dx)
178    self._art_cmd = ['/bin/bash', 'art', '-cp', 'classes.dex']
179    if extra_args is not None:
180      self._art_cmd += extra_args
181    self._art_cmd.append('Test')
182
183  def CompileAndRunTest(self):
184    if self.CompileOnHost() == RetCode.SUCCESS:
185      retc = RunCommand(self._art_cmd, self.output_file, 'arterr.txt')
186    else:
187      retc = RetCode.NOTCOMPILED
188    return retc
189
190
191class TestRunnerArtIntOnHost(TestRunnerArtOnHost):
192  """Concrete test runner of interpreter mode Art on host."""
193
194  def  __init__(self, use_dx):
195    """Constructor for the Art on host tester (interpreter).
196
197    Args:
198      use_dx: boolean, if True use dx rather than jack
199   """
200    super().__init__(use_dx, ['-Xint'])
201
202  @property
203  def description(self):
204    return 'Art interpreter on host'
205
206  @property
207  def id(self):
208    return 'HInt'
209
210  def GetBisectionSearchArgs(self):
211    return None
212
213
214class TestRunnerArtOptOnHost(TestRunnerArtOnHost):
215  """Concrete test runner of optimizing compiler mode Art on host."""
216
217  def  __init__(self, use_dx):
218    """Constructor for the Art on host tester (optimizing).
219
220    Args:
221      use_dx: boolean, if True use dx rather than jack
222   """
223    super().__init__(use_dx, None)
224
225  @property
226  def description(self):
227    return 'Art optimizing on host'
228
229  @property
230  def id(self):
231    return 'HOpt'
232
233  def GetBisectionSearchArgs(self):
234    cmd_str = CommandListToCommandString(
235        self._art_cmd[0:2] + ['{ARGS}'] + self._art_cmd[2:])
236    return ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
237
238
239class TestRunnerArtOnTarget(TestRunnerWithHostCompilation):
240  """Abstract test runner of Art on target."""
241
242  def  __init__(self, use_dx, device, extra_args=None):
243    """Constructor for the Art on target tester.
244
245    Args:
246      use_dx: boolean, if True use dx rather than jack
247      device: string, target device serial number (or None)
248      extra_args: list of strings, extra arguments for dalvikvm
249    """
250    super().__init__(use_dx)
251    self._test_env = DeviceTestEnv('jfuzz_', specific_device=device)
252    self._dalvik_cmd = ['dalvikvm']
253    if extra_args is not None:
254      self._dalvik_cmd += extra_args
255    self._device = device
256    self._device_classpath = None
257
258  def CompileAndRunTest(self):
259    if self.CompileOnHost() == RetCode.SUCCESS:
260      self._device_classpath = self._test_env.PushClasspath('classes.dex')
261      cmd = self._dalvik_cmd + ['-cp', self._device_classpath, 'Test']
262      (output, retc) = self._test_env.RunCommand(
263          cmd, {'ANDROID_LOG_TAGS': '*:s'})
264      with open(self.output_file, 'w') as run_out:
265        run_out.write(output)
266    else:
267      retc = RetCode.NOTCOMPILED
268    return retc
269
270  def GetBisectionSearchArgs(self):
271    cmd_str = CommandListToCommandString(
272        self._dalvik_cmd + ['-cp',self._device_classpath, 'Test'])
273    cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
274    if self._device:
275      cmd += ['--device-serial', self._device]
276    else:
277      cmd.append('--device')
278    return cmd
279
280
281class TestRunnerArtIntOnTarget(TestRunnerArtOnTarget):
282  """Concrete test runner of interpreter mode Art on target."""
283
284  def  __init__(self, use_dx, device):
285    """Constructor for the Art on target tester (interpreter).
286
287    Args:
288      use_dx: boolean, if True use dx rather than jack
289      device: string, target device serial number (or None)
290    """
291    super().__init__(use_dx, device, ['-Xint'])
292
293  @property
294  def description(self):
295    return 'Art interpreter on target'
296
297  @property
298  def id(self):
299    return 'TInt'
300
301  def GetBisectionSearchArgs(self):
302    return None
303
304
305class TestRunnerArtOptOnTarget(TestRunnerArtOnTarget):
306  """Concrete test runner of optimizing compiler mode Art on target."""
307
308  def  __init__(self, use_dx, device):
309    """Constructor for the Art on target tester (optimizing).
310
311    Args:
312      use_dx: boolean, if True use dx rather than jack
313      device: string, target device serial number (or None)
314    """
315    super().__init__(use_dx, device, None)
316
317  @property
318  def description(self):
319    return 'Art optimizing on target'
320
321  @property
322  def id(self):
323    return 'TOpt'
324
325  def GetBisectionSearchArgs(self):
326    cmd_str = CommandListToCommandString(
327        self._dalvik_cmd + ['-cp', self._device_classpath, 'Test'])
328    cmd = ['--raw-cmd={0}'.format(cmd_str), '--timeout', str(30)]
329    if self._device:
330      cmd += ['--device-serial', self._device]
331    else:
332      cmd.append('--device')
333    return cmd
334
335
336#
337# Tester class.
338#
339
340
341class JFuzzTester(object):
342  """Tester that runs JFuzz many times and report divergences."""
343
344  def  __init__(self, num_tests, device, mode1, mode2, jfuzz_args,
345                report_script, true_divergence_only, use_dx):
346    """Constructor for the tester.
347
348    Args:
349      num_tests: int, number of tests to run
350      device: string, target device serial number (or None)
351      mode1: string, execution mode for first runner
352      mode2: string, execution mode for second runner
353      jfuzz_args: list of strings, additional arguments for jfuzz
354      report_script: string, path to script called for each divergence
355      true_divergence_only: boolean, if True don't bisect timeout divergences
356      use_dx: boolean, if True use dx rather than jack
357    """
358    self._num_tests = num_tests
359    self._device = device
360    self._runner1 = GetExecutionModeRunner(use_dx, device, mode1)
361    self._runner2 = GetExecutionModeRunner(use_dx, device, mode2)
362    self._jfuzz_args = jfuzz_args
363    self._report_script = report_script
364    self._true_divergence_only = true_divergence_only
365    self._use_dx = use_dx
366    self._save_dir = None
367    self._results_dir = None
368    self._jfuzz_dir = None
369    # Statistics.
370    self._test = 0
371    self._num_success = 0
372    self._num_not_compiled = 0
373    self._num_not_run = 0
374    self._num_timed_out = 0
375    self._num_divergences = 0
376
377  def __enter__(self):
378    """On entry, enters new temp directory after saving current directory.
379
380    Raises:
381      FatalError: error when temp directory cannot be constructed
382    """
383    self._save_dir = os.getcwd()
384    self._results_dir = mkdtemp(dir='/tmp/')
385    self._jfuzz_dir = mkdtemp(dir=self._results_dir)
386    if self._results_dir is None or self._jfuzz_dir is None:
387      raise FatalError('Cannot obtain temp directory')
388    os.chdir(self._jfuzz_dir)
389    return self
390
391  def __exit__(self, etype, evalue, etraceback):
392    """On exit, re-enters previously saved current directory and cleans up."""
393    os.chdir(self._save_dir)
394    shutil.rmtree(self._jfuzz_dir)
395    if self._num_divergences == 0:
396      shutil.rmtree(self._results_dir)
397
398  def Run(self):
399    """Runs JFuzz many times and report divergences."""
400    print()
401    print('**\n**** JFuzz Testing\n**')
402    print()
403    print('#Tests    :', self._num_tests)
404    print('Device    :', self._device)
405    print('Directory :', self._results_dir)
406    print('Exec-mode1:', self._runner1.description)
407    print('Exec-mode2:', self._runner2.description)
408    print('Compiler  :', 'dx' if self._use_dx else 'jack')
409    print()
410    self.ShowStats()
411    for self._test in range(1, self._num_tests + 1):
412      self.RunJFuzzTest()
413      self.ShowStats()
414    if self._num_divergences == 0:
415      print('\n\nsuccess (no divergences)\n')
416    else:
417      print('\n\nfailure (divergences)\n')
418
419  def ShowStats(self):
420    """Shows current statistics (on same line) while tester is running."""
421    print('\rTests:', self._test,
422          'Success:', self._num_success,
423          'Not-compiled:', self._num_not_compiled,
424          'Not-run:', self._num_not_run,
425          'Timed-out:', self._num_timed_out,
426          'Divergences:', self._num_divergences,
427          end='')
428    sys.stdout.flush()
429
430  def RunJFuzzTest(self):
431    """Runs a single JFuzz test, comparing two execution modes."""
432    self.ConstructTest()
433    retc1 = self._runner1.CompileAndRunTest()
434    retc2 = self._runner2.CompileAndRunTest()
435    self.CheckForDivergence(retc1, retc2)
436    self.CleanupTest()
437
438  def ConstructTest(self):
439    """Use JFuzz to generate next Test.java test.
440
441    Raises:
442      FatalError: error when jfuzz fails
443    """
444    if (RunCommand(['jfuzz'] + self._jfuzz_args, out='Test.java', err=None)
445          != RetCode.SUCCESS):
446      raise FatalError('Unexpected error while running JFuzz')
447
448  def CheckForDivergence(self, retc1, retc2):
449    """Checks for divergences and updates statistics.
450
451    Args:
452      retc1: int, normalized return code of first runner
453      retc2: int, normalized return code of second runner
454    """
455    if retc1 == retc2:
456      # No divergence in return code.
457      if retc1 == RetCode.SUCCESS:
458        # Both compilations and runs were successful, inspect generated output.
459        runner1_out = self._runner1.output_file
460        runner2_out = self._runner2.output_file
461        if not filecmp.cmp(runner1_out, runner2_out, shallow=False):
462          # Divergence in output.
463          self.ReportDivergence(retc1, retc2, is_output_divergence=True)
464        else:
465          # No divergence in output.
466          self._num_success += 1
467      elif retc1 == RetCode.TIMEOUT:
468        self._num_timed_out += 1
469      elif retc1 == RetCode.NOTCOMPILED:
470        self._num_not_compiled += 1
471      else:
472        self._num_not_run += 1
473    elif self._true_divergence_only and RetCode.TIMEOUT in (retc1, retc2):
474      # When only true divergences are requested, any divergence in return
475      # code where one is a time out is treated as a regular time out.
476      self._num_timed_out += 1
477    else:
478      # Divergence in return code.
479      self.ReportDivergence(retc1, retc2, is_output_divergence=False)
480
481  def GetCurrentDivergenceDir(self):
482    return self._results_dir + '/divergence' + str(self._num_divergences)
483
484  def ReportDivergence(self, retc1, retc2, is_output_divergence):
485    """Reports and saves a divergence."""
486    self._num_divergences += 1
487    print('\n' + str(self._num_divergences), end='')
488    if is_output_divergence:
489      print(' divergence in output')
490    else:
491      print(' divergence in return code: ' + retc1.name + ' vs. ' +
492            retc2.name)
493    # Save.
494    ddir = self.GetCurrentDivergenceDir()
495    os.mkdir(ddir)
496    for f in glob('*.txt') + ['Test.java']:
497      shutil.copy(f, ddir)
498    # Maybe run bisection bug search.
499    if retc1 in BISECTABLE_RET_CODES and retc2 in BISECTABLE_RET_CODES:
500      self.MaybeBisectDivergence(retc1, retc2, is_output_divergence)
501    # Call reporting script.
502    if self._report_script:
503      self.RunReportScript(retc1, retc2, is_output_divergence)
504
505  def RunReportScript(self, retc1, retc2, is_output_divergence):
506    """Runs report script."""
507    try:
508      title = "Divergence between {0} and {1} (found with fuzz testing)".format(
509          self._runner1.description, self._runner2.description)
510      # Prepare divergence comment.
511      jfuzz_cmd_and_version = subprocess.check_output(
512          ['grep', '-o', 'jfuzz.*', 'Test.java'], universal_newlines=True)
513      (jfuzz_cmd_str, jfuzz_ver) = jfuzz_cmd_and_version.split('(')
514      # Strip right parenthesis and new line.
515      jfuzz_ver = jfuzz_ver[:-2]
516      jfuzz_args = ['\'-{0}\''.format(arg)
517                    for arg in jfuzz_cmd_str.strip().split(' -')][1:]
518      wrapped_args = ['--jfuzz_arg={0}'.format(opt) for opt in jfuzz_args]
519      repro_cmd_str = (os.path.basename(__file__) + ' --num_tests 1 ' +
520                       ' '.join(wrapped_args))
521      comment = 'jfuzz {0}\nReproduce test:\n{1}\nReproduce divergence:\n{2}\n'.format(
522          jfuzz_ver, jfuzz_cmd_str, repro_cmd_str)
523      if is_output_divergence:
524        (output, _, _) = RunCommandForOutput(
525            ['diff', self._runner1.output_file, self._runner2.output_file],
526            None, subprocess.PIPE, subprocess.STDOUT)
527        comment += 'Diff:\n' + output
528      else:
529        comment += '{0} vs {1}\n'.format(retc1, retc2)
530      # Prepare report script command.
531      script_cmd = [self._report_script, title, comment]
532      ddir = self.GetCurrentDivergenceDir()
533      bisection_out_files = glob(ddir + '/*_bisection_out.txt')
534      if bisection_out_files:
535        script_cmd += ['--bisection_out', bisection_out_files[0]]
536      subprocess.check_call(script_cmd, stdout=DEVNULL, stderr=DEVNULL)
537    except subprocess.CalledProcessError as err:
538      print('Failed to run report script.\n', err)
539
540  def RunBisectionSearch(self, args, expected_retcode, expected_output,
541                         runner_id):
542    ddir = self.GetCurrentDivergenceDir()
543    outfile_path = ddir + '/' + runner_id + '_bisection_out.txt'
544    logfile_path = ddir + '/' + runner_id + '_bisection_log.txt'
545    errfile_path = ddir + '/' + runner_id + '_bisection_err.txt'
546    args = list(args) + ['--logfile', logfile_path, '--cleanup']
547    args += ['--expected-retcode', expected_retcode.name]
548    if expected_output:
549      args += ['--expected-output', expected_output]
550    bisection_search_path = os.path.join(
551        GetEnvVariableOrError('ANDROID_BUILD_TOP'),
552        'art/tools/bisection_search/bisection_search.py')
553    if RunCommand([bisection_search_path] + args, out=outfile_path,
554                  err=errfile_path, timeout=300) == RetCode.TIMEOUT:
555      print('Bisection search TIMEOUT')
556
557  def MaybeBisectDivergence(self, retc1, retc2, is_output_divergence):
558    bisection_args1 = self._runner1.GetBisectionSearchArgs()
559    bisection_args2 = self._runner2.GetBisectionSearchArgs()
560    if is_output_divergence:
561      maybe_output1 = self._runner1.output_file
562      maybe_output2 = self._runner2.output_file
563    else:
564      maybe_output1 = maybe_output2 = None
565    if bisection_args1 is not None:
566      self.RunBisectionSearch(bisection_args1, retc2, maybe_output2,
567                              self._runner1.id)
568    if bisection_args2 is not None:
569      self.RunBisectionSearch(bisection_args2, retc1, maybe_output1,
570                              self._runner2.id)
571
572  def CleanupTest(self):
573    """Cleans up after a single test run."""
574    for file_name in os.listdir(self._jfuzz_dir):
575      file_path = os.path.join(self._jfuzz_dir, file_name)
576      if os.path.isfile(file_path):
577        os.unlink(file_path)
578      elif os.path.isdir(file_path):
579        shutil.rmtree(file_path)
580
581
582def main():
583  # Handle arguments.
584  parser = argparse.ArgumentParser()
585  parser.add_argument('--num_tests', default=10000,
586                      type=int, help='number of tests to run')
587  parser.add_argument('--device', help='target device serial number')
588  parser.add_argument('--mode1', default='ri',
589                      help='execution mode 1 (default: ri)')
590  parser.add_argument('--mode2', default='hopt',
591                      help='execution mode 2 (default: hopt)')
592  parser.add_argument('--report_script', help='script called for each'
593                                              ' divergence')
594  parser.add_argument('--jfuzz_arg', default=[], dest='jfuzz_args',
595                      action='append', help='argument for jfuzz')
596  parser.add_argument('--true_divergence', default=False, action='store_true',
597                      help='don\'t bisect timeout divergences')
598  parser.add_argument('--use_dx', default=False, action='store_true',
599                      help='use old-style dx (rather than jack)')
600  args = parser.parse_args()
601  if args.mode1 == args.mode2:
602    raise FatalError('Identical execution modes given')
603  # Run the JFuzz tester.
604  with JFuzzTester(args.num_tests,
605                   args.device, args.mode1, args.mode2,
606                   args.jfuzz_args, args.report_script,
607                   args.true_divergence, args.use_dx) as fuzzer:
608    fuzzer.Run()
609
610if __name__ == '__main__':
611  main()
612