1# Copyright 2012 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28import copy
29import os
30import re
31import shlex
32
33from ..outproc import base as outproc
34from ..local import command
35from ..local import statusfile
36from ..local import utils
37
38FLAGS_PATTERN = re.compile(r"//\s+Flags:(.*)")
39
40
41
42class TestCase(object):
43  def __init__(self, suite, path, name, test_config):
44    self.suite = suite        # TestSuite object
45
46    self.path = path          # string, e.g. 'div-mod', 'test-api/foo'
47    self.name = name          # string that identifies test in the status file
48
49    self.variant = None       # name of the used testing variant
50    self.variant_flags = []   # list of strings, flags specific to this test
51
52    # Fields used by the test processors.
53    self.origin = None # Test that this test is subtest of.
54    self.processor = None # Processor that created this subtest.
55    self.procid = '%s/%s' % (self.suite.name, self.name) # unique id
56    self.keep_output = False # Can output of this test be dropped
57
58    # Test config contains information needed to build the command.
59    self._test_config = test_config
60    self._random_seed = None # Overrides test config value if not None
61
62    # Outcomes
63    self._statusfile_outcomes = None
64    self.expected_outcomes = None
65    self._statusfile_flags = None
66
67    self._prepare_outcomes()
68
69  def create_subtest(self, processor, subtest_id, variant=None, flags=None,
70                     keep_output=False, random_seed=None):
71    subtest = copy.copy(self)
72    subtest.origin = self
73    subtest.processor = processor
74    subtest.procid += '.%s' % subtest_id
75    subtest.keep_output |= keep_output
76    if random_seed:
77      subtest._random_seed = random_seed
78    if flags:
79      subtest.variant_flags = subtest.variant_flags + flags
80    if variant is not None:
81      assert self.variant is None
82      subtest.variant = variant
83      subtest._prepare_outcomes()
84    return subtest
85
86  def _prepare_outcomes(self, force_update=True):
87    if force_update or self._statusfile_outcomes is None:
88      def is_flag(outcome):
89        return outcome.startswith('--')
90      def not_flag(outcome):
91        return not is_flag(outcome)
92
93      outcomes = self.suite.statusfile.get_outcomes(self.name, self.variant)
94      self._statusfile_outcomes = filter(not_flag, outcomes)
95      self._statusfile_flags = filter(is_flag, outcomes)
96    self.expected_outcomes = (
97      self._parse_status_file_outcomes(self._statusfile_outcomes))
98
99  def _parse_status_file_outcomes(self, outcomes):
100    if (statusfile.FAIL_SLOPPY in outcomes and
101        '--use-strict' not in self.variant_flags):
102      return outproc.OUTCOMES_FAIL
103
104    expected_outcomes = []
105    if (statusfile.FAIL in outcomes or
106        statusfile.FAIL_OK in outcomes):
107      expected_outcomes.append(statusfile.FAIL)
108    if statusfile.CRASH in outcomes:
109      expected_outcomes.append(statusfile.CRASH)
110
111    # Do not add PASS if there is nothing else. Empty outcomes are converted to
112    # the global [PASS].
113    if expected_outcomes and statusfile.PASS in outcomes:
114      expected_outcomes.append(statusfile.PASS)
115
116    # Avoid creating multiple instances of a list with a single FAIL.
117    if expected_outcomes == outproc.OUTCOMES_FAIL:
118      return outproc.OUTCOMES_FAIL
119    return expected_outcomes or outproc.OUTCOMES_PASS
120
121  @property
122  def do_skip(self):
123    return statusfile.SKIP in self._statusfile_outcomes
124
125  @property
126  def is_slow(self):
127    return statusfile.SLOW in self._statusfile_outcomes
128
129  @property
130  def is_fail_ok(self):
131    return statusfile.FAIL_OK in self._statusfile_outcomes
132
133  @property
134  def is_pass_or_fail(self):
135    return (statusfile.PASS in self._statusfile_outcomes and
136            statusfile.FAIL in self._statusfile_outcomes and
137            statusfile.CRASH not in self._statusfile_outcomes)
138
139  @property
140  def only_standard_variant(self):
141    return statusfile.NO_VARIANTS in self._statusfile_outcomes
142
143  def get_command(self):
144    params = self._get_cmd_params()
145    env = self._get_cmd_env()
146    shell, shell_flags = self._get_shell_with_flags()
147    timeout = self._get_timeout(params)
148    return self._create_cmd(shell, shell_flags + params, env, timeout)
149
150  def _get_cmd_params(self):
151    """Gets command parameters and combines them in the following order:
152      - files [empty by default]
153      - random seed
154      - extra flags (from command line)
155      - user flags (variant/fuzzer flags)
156      - mode flags (based on chosen mode)
157      - source flags (from source code) [empty by default]
158      - test-suite flags
159      - statusfile flags
160
161    The best way to modify how parameters are created is to only override
162    methods for getting partial parameters.
163    """
164    return (
165        self._get_files_params() +
166        self._get_random_seed_flags() +
167        self._get_extra_flags() +
168        self._get_variant_flags() +
169        self._get_mode_flags() +
170        self._get_source_flags() +
171        self._get_suite_flags() +
172        self._get_statusfile_flags()
173    )
174
175  def _get_cmd_env(self):
176    return {}
177
178  def _get_files_params(self):
179    return []
180
181  def _get_random_seed_flags(self):
182    return ['--random-seed=%d' % self.random_seed]
183
184  @property
185  def random_seed(self):
186    return self._random_seed or self._test_config.random_seed
187
188  def _get_extra_flags(self):
189    return self._test_config.extra_flags
190
191  def _get_variant_flags(self):
192    return self.variant_flags
193
194  def _get_statusfile_flags(self):
195    """Gets runtime flags from a status file.
196
197    Every outcome that starts with "--" is a flag.
198    """
199    return self._statusfile_flags
200
201  def _get_mode_flags(self):
202    return self._test_config.mode_flags
203
204  def _get_source_flags(self):
205    return []
206
207  def _get_suite_flags(self):
208    return []
209
210  def _get_shell_with_flags(self):
211    shell = self.get_shell()
212    shell_flags = []
213    if shell == 'd8':
214      shell_flags.append('--test')
215    if utils.IsWindows():
216      shell += '.exe'
217    return shell, shell_flags
218
219  def _get_timeout(self, params):
220    timeout = self._test_config.timeout
221    if "--stress-opt" in params:
222      timeout *= 4
223    if "--noenable-vfp3" in params:
224      timeout *= 2
225
226    # TODO(majeski): make it slow outcome dependent.
227    timeout *= 2
228    return timeout
229
230  def get_shell(self):
231    return 'd8'
232
233  def _get_suffix(self):
234    return '.js'
235
236  def _create_cmd(self, shell, params, env, timeout):
237    return command.Command(
238      cmd_prefix=self._test_config.command_prefix,
239      shell=os.path.abspath(os.path.join(self._test_config.shell_dir, shell)),
240      args=params,
241      env=env,
242      timeout=timeout,
243      verbose=self._test_config.verbose,
244      resources_func=self._get_resources,
245    )
246
247  def _parse_source_flags(self, source=None):
248    source = source or self.get_source()
249    flags = []
250    for match in re.findall(FLAGS_PATTERN, source):
251      flags += shlex.split(match.strip())
252    return flags
253
254  def is_source_available(self):
255    return self._get_source_path() is not None
256
257  def get_source(self):
258    with open(self._get_source_path()) as f:
259      return f.read()
260
261  def _get_source_path(self):
262    return None
263
264  def _get_resources(self):
265    """Returns a list of absolute paths with additional files needed by the
266    test case.
267
268    Used to push additional files to Android devices.
269    """
270    return []
271
272  @property
273  def output_proc(self):
274    if self.expected_outcomes is outproc.OUTCOMES_PASS:
275      return outproc.DEFAULT
276    return outproc.OutProc(self.expected_outcomes)
277
278  def __cmp__(self, other):
279    # Make sure that test cases are sorted correctly if sorted without
280    # key function. But using a key function is preferred for speed.
281    return cmp(
282        (self.suite.name, self.name, self.variant),
283        (other.suite.name, other.name, other.variant)
284    )
285
286  def __str__(self):
287    return self.suite.name + '/' + self.name
288