1# Copyright (c) 2018 Google LLC
2#
3# Licensed under the Apache License, Version 2.0 (the "License");
4# you may not use this file except in compliance with the License.
5# You may obtain a copy of the License at
6#
7#     http://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS,
11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12# See the License for the specific language governing permissions and
13# limitations under the License.
14"""Manages and runs tests from the current working directory.
15
16This will traverse the current working directory and look for python files that
17contain subclasses of SpirvTest.
18
19If a class has an @inside_spirv_testsuite decorator, an instance of that
20class will be created and serve as a test case in that testsuite.  The test
21case is then run by the following steps:
22
23  1. A temporary directory will be created.
24  2. The spirv_args member variable will be inspected and all placeholders in it
25     will be expanded by calling instantiate_for_spirv_args() on placeholders.
26     The transformed list elements are then supplied as arguments to the spirv-*
27     tool under test.
28  3. If the environment member variable exists, its write() method will be
29     invoked.
30  4. All expected_* member variables will be inspected and all placeholders in
31     them will be expanded by calling instantiate_for_expectation() on those
32     placeholders. After placeholder expansion, if the expected_* variable is
33     a list, its element will be joined together with '' to form a single
34     string. These expected_* variables are to be used by the check_*() methods.
35  5. The spirv-* tool will be run with the arguments supplied in spirv_args.
36  6. All check_*() member methods will be called by supplying a TestStatus as
37     argument. Each check_*() method is expected to return a (Success, Message)
38     pair where Success is a boolean indicating success and Message is an error
39     message.
40  7. If any check_*() method fails, the error message is output and the
41     current test case fails.
42
43If --leave-output was not specified, all temporary files and directories will
44be deleted.
45"""
46
47import argparse
48import fnmatch
49import inspect
50import os
51import shutil
52import subprocess
53import sys
54import tempfile
55from collections import defaultdict
56from placeholder import PlaceHolder
57
58EXPECTED_BEHAVIOR_PREFIX = 'expected_'
59VALIDATE_METHOD_PREFIX = 'check_'
60
61
62def get_all_variables(instance):
63  """Returns the names of all the variables in instance."""
64  return [v for v in dir(instance) if not callable(getattr(instance, v))]
65
66
67def get_all_methods(instance):
68  """Returns the names of all methods in instance."""
69  return [m for m in dir(instance) if callable(getattr(instance, m))]
70
71
72def get_all_superclasses(cls):
73  """Returns all superclasses of a given class. Omits root 'object' superclass.
74
75    Returns:
76      A list of superclasses of the given class. The order guarantees that
77      * A Base class precedes its derived classes, e.g., for "class B(A)", it
78        will be [..., A, B, ...].
79      * When there are multiple base classes, base classes declared first
80        precede those declared later, e.g., for "class C(A, B), it will be
81        [..., A, B, C, ...]
82    """
83  classes = []
84  for superclass in cls.__bases__:
85    for c in get_all_superclasses(superclass):
86      if c is not object and c not in classes:
87        classes.append(c)
88  for superclass in cls.__bases__:
89    if superclass is not object and superclass not in classes:
90      classes.append(superclass)
91
92  return classes
93
94
95def get_all_test_methods(test_class):
96  """Gets all validation methods.
97
98    Returns:
99      A list of validation methods. The order guarantees that
100      * A method defined in superclass precedes one defined in subclass,
101        e.g., for "class A(B)", methods defined in B precedes those defined
102        in A.
103      * If a subclass has more than one superclass, e.g., "class C(A, B)",
104        then methods defined in A precedes those defined in B.
105    """
106  classes = get_all_superclasses(test_class)
107  classes.append(test_class)
108  all_tests = [
109      m for c in classes for m in get_all_methods(c)
110      if m.startswith(VALIDATE_METHOD_PREFIX)
111  ]
112  unique_tests = []
113  for t in all_tests:
114    if t not in unique_tests:
115      unique_tests.append(t)
116  return unique_tests
117
118
119class SpirvTest:
120  """Base class for spirv test cases.
121
122    Subclasses define test cases' facts (shader source code, spirv command,
123    result validation), which will be used by the TestCase class for running
124    tests. Subclasses should define spirv_args (specifying spirv_tool command
125    arguments), and at least one check_*() method (for result validation) for
126    a full-fledged test case. All check_*() methods should take a TestStatus
127    parameter and return a (Success, Message) pair, in which Success is a
128    boolean indicating success and Message is an error message. The test passes
129    iff all check_*() methods returns true.
130
131    Often, a test case class will delegate the check_* behaviors by inheriting
132    from other classes.
133    """
134
135  def name(self):
136    return self.__class__.__name__
137
138
139class TestStatus:
140  """A struct for holding run status of a test case."""
141
142  def __init__(self, test_manager, returncode, stdout, stderr, directory,
143               inputs, input_filenames):
144    self.test_manager = test_manager
145    self.returncode = returncode
146    # Some of our MacOS bots still run Python 2, so need to be backwards
147    # compatible here.
148    if type(stdout) is not str:
149      if sys.version_info[0] == 2:
150       self.stdout = stdout.decode('utf-8')
151      elif sys.version_info[0] == 3:
152        self.stdout = str(stdout, encoding='utf-8') if stdout is not None else stdout
153      else:
154        raise Exception('Unable to determine if running Python 2 or 3 from {}'.format(sys.version_info))
155    else:
156      self.stdout = stdout
157
158    if type(stderr) is not str:
159      if sys.version_info[0] == 2:
160       self.stderr = stderr.decode('utf-8')
161      elif sys.version_info[0] == 3:
162        self.stderr = str(stderr, encoding='utf-8') if stderr is not None else stderr
163      else:
164        raise Exception('Unable to determine if running Python 2 or 3 from {}'.format(sys.version_info))
165    else:
166      self.stderr = stderr
167
168    # temporary directory where the test runs
169    self.directory = directory
170    # List of inputs, as PlaceHolder objects.
171    self.inputs = inputs
172    # the names of input shader files (potentially including paths)
173    self.input_filenames = input_filenames
174
175
176class SpirvTestException(Exception):
177  """SpirvTest exception class."""
178  pass
179
180
181def inside_spirv_testsuite(testsuite_name):
182  """Decorator for subclasses of SpirvTest.
183
184    This decorator checks that a class meets the requirements (see below)
185    for a test case class, and then puts the class in a certain testsuite.
186    * The class needs to be a subclass of SpirvTest.
187    * The class needs to have spirv_args defined as a list.
188    * The class needs to define at least one check_*() methods.
189    * All expected_* variables required by check_*() methods can only be
190      of bool, str, or list type.
191    * Python runtime will throw an exception if the expected_* member
192      attributes required by check_*() methods are missing.
193    """
194
195  def actual_decorator(cls):
196    if not inspect.isclass(cls):
197      raise SpirvTestException('Test case should be a class')
198    if not issubclass(cls, SpirvTest):
199      raise SpirvTestException(
200          'All test cases should be subclasses of SpirvTest')
201    if 'spirv_args' not in get_all_variables(cls):
202      raise SpirvTestException('No spirv_args found in the test case')
203    if not isinstance(cls.spirv_args, list):
204      raise SpirvTestException('spirv_args needs to be a list')
205    if not any(
206        [m.startswith(VALIDATE_METHOD_PREFIX) for m in get_all_methods(cls)]):
207      raise SpirvTestException('No check_*() methods found in the test case')
208    if not all(
209        [isinstance(v, (bool, str, list)) for v in get_all_variables(cls)]):
210      raise SpirvTestException(
211          'expected_* variables are only allowed to be bool, str, or '
212          'list type.')
213    cls.parent_testsuite = testsuite_name
214    return cls
215
216  return actual_decorator
217
218
219class TestManager:
220  """Manages and runs a set of tests."""
221
222  def __init__(self, executable_path, assembler_path, disassembler_path):
223    self.executable_path = executable_path
224    self.assembler_path = assembler_path
225    self.disassembler_path = disassembler_path
226    self.num_successes = 0
227    self.num_failures = 0
228    self.num_tests = 0
229    self.leave_output = False
230    self.tests = defaultdict(list)
231
232  def notify_result(self, test_case, success, message):
233    """Call this to notify the manager of the results of a test run."""
234    self.num_successes += 1 if success else 0
235    self.num_failures += 0 if success else 1
236    counter_string = str(self.num_successes + self.num_failures) + '/' + str(
237        self.num_tests)
238    print('%-10s %-40s ' % (counter_string, test_case.test.name()) +
239          ('Passed' if success else '-Failed-'))
240    if not success:
241      print(' '.join(test_case.command))
242      print(message)
243
244  def add_test(self, testsuite, test):
245    """Add this to the current list of test cases."""
246    self.tests[testsuite].append(TestCase(test, self))
247    self.num_tests += 1
248
249  def run_tests(self):
250    for suite in self.tests:
251      print('SPIRV tool test suite: "{suite}"'.format(suite=suite))
252      for x in self.tests[suite]:
253        x.runTest()
254
255
256class TestCase:
257  """A single test case that runs in its own directory."""
258
259  def __init__(self, test, test_manager):
260    self.test = test
261    self.test_manager = test_manager
262    self.inputs = []  # inputs, as PlaceHolder objects.
263    self.file_shaders = []  # filenames of shader files.
264    self.stdin_shader = None  # text to be passed to spirv_tool as stdin
265
266  def setUp(self):
267    """Creates environment and instantiates placeholders for the test case."""
268
269    self.directory = tempfile.mkdtemp(dir=os.getcwd())
270    spirv_args = self.test.spirv_args
271    # Instantiate placeholders in spirv_args
272    self.test.spirv_args = [
273        arg.instantiate_for_spirv_args(self)
274        if isinstance(arg, PlaceHolder) else arg for arg in self.test.spirv_args
275    ]
276    # Get all shader files' names
277    self.inputs = [arg for arg in spirv_args if isinstance(arg, PlaceHolder)]
278    self.file_shaders = [arg.filename for arg in self.inputs]
279
280    if 'environment' in get_all_variables(self.test):
281      self.test.environment.write(self.directory)
282
283    expectations = [
284        v for v in get_all_variables(self.test)
285        if v.startswith(EXPECTED_BEHAVIOR_PREFIX)
286    ]
287    # Instantiate placeholders in expectations
288    for expectation_name in expectations:
289      expectation = getattr(self.test, expectation_name)
290      if isinstance(expectation, list):
291        expanded_expections = [
292            element.instantiate_for_expectation(self)
293            if isinstance(element, PlaceHolder) else element
294            for element in expectation
295        ]
296        setattr(self.test, expectation_name, expanded_expections)
297      elif isinstance(expectation, PlaceHolder):
298        setattr(self.test, expectation_name,
299                expectation.instantiate_for_expectation(self))
300
301  def tearDown(self):
302    """Removes the directory if we were not instructed to do otherwise."""
303    if not self.test_manager.leave_output:
304      shutil.rmtree(self.directory)
305
306  def runTest(self):
307    """Sets up and runs a test, reports any failures and then cleans up."""
308    self.setUp()
309    success = False
310    message = ''
311    try:
312      self.command = [self.test_manager.executable_path]
313      self.command.extend(self.test.spirv_args)
314
315      process = subprocess.Popen(
316          args=self.command,
317          stdin=subprocess.PIPE,
318          stdout=subprocess.PIPE,
319          stderr=subprocess.PIPE,
320          cwd=self.directory)
321      output = process.communicate(self.stdin_shader)
322      test_status = TestStatus(self.test_manager, process.returncode, output[0],
323                               output[1], self.directory, self.inputs,
324                               self.file_shaders)
325      run_results = [
326          getattr(self.test, test_method)(test_status)
327          for test_method in get_all_test_methods(self.test.__class__)
328      ]
329      success, message = zip(*run_results)
330      success = all(success)
331      message = '\n'.join(message)
332    except Exception as e:
333      success = False
334      message = str(e)
335    self.test_manager.notify_result(
336        self, success,
337        message + '\nSTDOUT:\n%s\nSTDERR:\n%s' % (output[0], output[1]))
338    self.tearDown()
339
340
341def main():
342  parser = argparse.ArgumentParser()
343  parser.add_argument(
344      'spirv_tool',
345      metavar='path/to/spirv_tool',
346      type=str,
347      nargs=1,
348      help='Path to the spirv-* tool under test')
349  parser.add_argument(
350      'spirv_as',
351      metavar='path/to/spirv-as',
352      type=str,
353      nargs=1,
354      help='Path to spirv-as')
355  parser.add_argument(
356      'spirv_dis',
357      metavar='path/to/spirv-dis',
358      type=str,
359      nargs=1,
360      help='Path to spirv-dis')
361  parser.add_argument(
362      '--leave-output',
363      action='store_const',
364      const=1,
365      help='Do not clean up temporary directories')
366  parser.add_argument(
367      '--test-dir', nargs=1, help='Directory to gather the tests from')
368  args = parser.parse_args()
369  default_path = sys.path
370  root_dir = os.getcwd()
371  if args.test_dir:
372    root_dir = args.test_dir[0]
373  manager = TestManager(args.spirv_tool[0], args.spirv_as[0], args.spirv_dis[0])
374  if args.leave_output:
375    manager.leave_output = True
376  for root, _, filenames in os.walk(root_dir):
377    for filename in fnmatch.filter(filenames, '*.py'):
378      if filename.endswith('nosetest.py'):
379        # Skip nose tests, which are for testing functions of
380        # the test framework.
381        continue
382      sys.path = default_path
383      sys.path.append(root)
384      mod = __import__(os.path.splitext(filename)[0])
385      for _, obj, in inspect.getmembers(mod):
386        if inspect.isclass(obj) and hasattr(obj, 'parent_testsuite'):
387          manager.add_test(obj.parent_testsuite, obj())
388  manager.run_tests()
389  if manager.num_failures > 0:
390    sys.exit(-1)
391
392
393if __name__ == '__main__':
394  main()
395