1# Copyright (c) 2018 Google LLC 2# 3# Licensed under the Apache License, Version 2.0 (the "License"); 4# you may not use this file except in compliance with the License. 5# You may obtain a copy of the License at 6# 7# http://www.apache.org/licenses/LICENSE-2.0 8# 9# Unless required by applicable law or agreed to in writing, software 10# distributed under the License is distributed on an "AS IS" BASIS, 11# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12# See the License for the specific language governing permissions and 13# limitations under the License. 14"""Manages and runs tests from the current working directory. 15 16This will traverse the current working directory and look for python files that 17contain subclasses of SpirvTest. 18 19If a class has an @inside_spirv_testsuite decorator, an instance of that 20class will be created and serve as a test case in that testsuite. The test 21case is then run by the following steps: 22 23 1. A temporary directory will be created. 24 2. The spirv_args member variable will be inspected and all placeholders in it 25 will be expanded by calling instantiate_for_spirv_args() on placeholders. 26 The transformed list elements are then supplied as arguments to the spirv-* 27 tool under test. 28 3. If the environment member variable exists, its write() method will be 29 invoked. 30 4. All expected_* member variables will be inspected and all placeholders in 31 them will be expanded by calling instantiate_for_expectation() on those 32 placeholders. After placeholder expansion, if the expected_* variable is 33 a list, its element will be joined together with '' to form a single 34 string. These expected_* variables are to be used by the check_*() methods. 35 5. The spirv-* tool will be run with the arguments supplied in spirv_args. 36 6. All check_*() member methods will be called by supplying a TestStatus as 37 argument. Each check_*() method is expected to return a (Success, Message) 38 pair where Success is a boolean indicating success and Message is an error 39 message. 40 7. If any check_*() method fails, the error message is output and the 41 current test case fails. 42 43If --leave-output was not specified, all temporary files and directories will 44be deleted. 45""" 46 47import argparse 48import fnmatch 49import inspect 50import os 51import shutil 52import subprocess 53import sys 54import tempfile 55from collections import defaultdict 56from placeholder import PlaceHolder 57 58EXPECTED_BEHAVIOR_PREFIX = 'expected_' 59VALIDATE_METHOD_PREFIX = 'check_' 60 61 62def get_all_variables(instance): 63 """Returns the names of all the variables in instance.""" 64 return [v for v in dir(instance) if not callable(getattr(instance, v))] 65 66 67def get_all_methods(instance): 68 """Returns the names of all methods in instance.""" 69 return [m for m in dir(instance) if callable(getattr(instance, m))] 70 71 72def get_all_superclasses(cls): 73 """Returns all superclasses of a given class. Omits root 'object' superclass. 74 75 Returns: 76 A list of superclasses of the given class. The order guarantees that 77 * A Base class precedes its derived classes, e.g., for "class B(A)", it 78 will be [..., A, B, ...]. 79 * When there are multiple base classes, base classes declared first 80 precede those declared later, e.g., for "class C(A, B), it will be 81 [..., A, B, C, ...] 82 """ 83 classes = [] 84 for superclass in cls.__bases__: 85 for c in get_all_superclasses(superclass): 86 if c is not object and c not in classes: 87 classes.append(c) 88 for superclass in cls.__bases__: 89 if superclass is not object and superclass not in classes: 90 classes.append(superclass) 91 92 return classes 93 94 95def get_all_test_methods(test_class): 96 """Gets all validation methods. 97 98 Returns: 99 A list of validation methods. The order guarantees that 100 * A method defined in superclass precedes one defined in subclass, 101 e.g., for "class A(B)", methods defined in B precedes those defined 102 in A. 103 * If a subclass has more than one superclass, e.g., "class C(A, B)", 104 then methods defined in A precedes those defined in B. 105 """ 106 classes = get_all_superclasses(test_class) 107 classes.append(test_class) 108 all_tests = [ 109 m for c in classes for m in get_all_methods(c) 110 if m.startswith(VALIDATE_METHOD_PREFIX) 111 ] 112 unique_tests = [] 113 for t in all_tests: 114 if t not in unique_tests: 115 unique_tests.append(t) 116 return unique_tests 117 118 119class SpirvTest: 120 """Base class for spirv test cases. 121 122 Subclasses define test cases' facts (shader source code, spirv command, 123 result validation), which will be used by the TestCase class for running 124 tests. Subclasses should define spirv_args (specifying spirv_tool command 125 arguments), and at least one check_*() method (for result validation) for 126 a full-fledged test case. All check_*() methods should take a TestStatus 127 parameter and return a (Success, Message) pair, in which Success is a 128 boolean indicating success and Message is an error message. The test passes 129 iff all check_*() methods returns true. 130 131 Often, a test case class will delegate the check_* behaviors by inheriting 132 from other classes. 133 """ 134 135 def name(self): 136 return self.__class__.__name__ 137 138 139class TestStatus: 140 """A struct for holding run status of a test case.""" 141 142 def __init__(self, test_manager, returncode, stdout, stderr, directory, 143 inputs, input_filenames): 144 self.test_manager = test_manager 145 self.returncode = returncode 146 # Some of our MacOS bots still run Python 2, so need to be backwards 147 # compatible here. 148 if type(stdout) is not str: 149 if sys.version_info[0] == 2: 150 self.stdout = stdout.decode('utf-8') 151 elif sys.version_info[0] == 3: 152 self.stdout = str(stdout, encoding='utf-8') if stdout is not None else stdout 153 else: 154 raise Exception('Unable to determine if running Python 2 or 3 from {}'.format(sys.version_info)) 155 else: 156 self.stdout = stdout 157 158 if type(stderr) is not str: 159 if sys.version_info[0] == 2: 160 self.stderr = stderr.decode('utf-8') 161 elif sys.version_info[0] == 3: 162 self.stderr = str(stderr, encoding='utf-8') if stderr is not None else stderr 163 else: 164 raise Exception('Unable to determine if running Python 2 or 3 from {}'.format(sys.version_info)) 165 else: 166 self.stderr = stderr 167 168 # temporary directory where the test runs 169 self.directory = directory 170 # List of inputs, as PlaceHolder objects. 171 self.inputs = inputs 172 # the names of input shader files (potentially including paths) 173 self.input_filenames = input_filenames 174 175 176class SpirvTestException(Exception): 177 """SpirvTest exception class.""" 178 pass 179 180 181def inside_spirv_testsuite(testsuite_name): 182 """Decorator for subclasses of SpirvTest. 183 184 This decorator checks that a class meets the requirements (see below) 185 for a test case class, and then puts the class in a certain testsuite. 186 * The class needs to be a subclass of SpirvTest. 187 * The class needs to have spirv_args defined as a list. 188 * The class needs to define at least one check_*() methods. 189 * All expected_* variables required by check_*() methods can only be 190 of bool, str, or list type. 191 * Python runtime will throw an exception if the expected_* member 192 attributes required by check_*() methods are missing. 193 """ 194 195 def actual_decorator(cls): 196 if not inspect.isclass(cls): 197 raise SpirvTestException('Test case should be a class') 198 if not issubclass(cls, SpirvTest): 199 raise SpirvTestException( 200 'All test cases should be subclasses of SpirvTest') 201 if 'spirv_args' not in get_all_variables(cls): 202 raise SpirvTestException('No spirv_args found in the test case') 203 if not isinstance(cls.spirv_args, list): 204 raise SpirvTestException('spirv_args needs to be a list') 205 if not any( 206 [m.startswith(VALIDATE_METHOD_PREFIX) for m in get_all_methods(cls)]): 207 raise SpirvTestException('No check_*() methods found in the test case') 208 if not all( 209 [isinstance(v, (bool, str, list)) for v in get_all_variables(cls)]): 210 raise SpirvTestException( 211 'expected_* variables are only allowed to be bool, str, or ' 212 'list type.') 213 cls.parent_testsuite = testsuite_name 214 return cls 215 216 return actual_decorator 217 218 219class TestManager: 220 """Manages and runs a set of tests.""" 221 222 def __init__(self, executable_path, assembler_path, disassembler_path): 223 self.executable_path = executable_path 224 self.assembler_path = assembler_path 225 self.disassembler_path = disassembler_path 226 self.num_successes = 0 227 self.num_failures = 0 228 self.num_tests = 0 229 self.leave_output = False 230 self.tests = defaultdict(list) 231 232 def notify_result(self, test_case, success, message): 233 """Call this to notify the manager of the results of a test run.""" 234 self.num_successes += 1 if success else 0 235 self.num_failures += 0 if success else 1 236 counter_string = str(self.num_successes + self.num_failures) + '/' + str( 237 self.num_tests) 238 print('%-10s %-40s ' % (counter_string, test_case.test.name()) + 239 ('Passed' if success else '-Failed-')) 240 if not success: 241 print(' '.join(test_case.command)) 242 print(message) 243 244 def add_test(self, testsuite, test): 245 """Add this to the current list of test cases.""" 246 self.tests[testsuite].append(TestCase(test, self)) 247 self.num_tests += 1 248 249 def run_tests(self): 250 for suite in self.tests: 251 print('SPIRV tool test suite: "{suite}"'.format(suite=suite)) 252 for x in self.tests[suite]: 253 x.runTest() 254 255 256class TestCase: 257 """A single test case that runs in its own directory.""" 258 259 def __init__(self, test, test_manager): 260 self.test = test 261 self.test_manager = test_manager 262 self.inputs = [] # inputs, as PlaceHolder objects. 263 self.file_shaders = [] # filenames of shader files. 264 self.stdin_shader = None # text to be passed to spirv_tool as stdin 265 266 def setUp(self): 267 """Creates environment and instantiates placeholders for the test case.""" 268 269 self.directory = tempfile.mkdtemp(dir=os.getcwd()) 270 spirv_args = self.test.spirv_args 271 # Instantiate placeholders in spirv_args 272 self.test.spirv_args = [ 273 arg.instantiate_for_spirv_args(self) 274 if isinstance(arg, PlaceHolder) else arg for arg in self.test.spirv_args 275 ] 276 # Get all shader files' names 277 self.inputs = [arg for arg in spirv_args if isinstance(arg, PlaceHolder)] 278 self.file_shaders = [arg.filename for arg in self.inputs] 279 280 if 'environment' in get_all_variables(self.test): 281 self.test.environment.write(self.directory) 282 283 expectations = [ 284 v for v in get_all_variables(self.test) 285 if v.startswith(EXPECTED_BEHAVIOR_PREFIX) 286 ] 287 # Instantiate placeholders in expectations 288 for expectation_name in expectations: 289 expectation = getattr(self.test, expectation_name) 290 if isinstance(expectation, list): 291 expanded_expections = [ 292 element.instantiate_for_expectation(self) 293 if isinstance(element, PlaceHolder) else element 294 for element in expectation 295 ] 296 setattr(self.test, expectation_name, expanded_expections) 297 elif isinstance(expectation, PlaceHolder): 298 setattr(self.test, expectation_name, 299 expectation.instantiate_for_expectation(self)) 300 301 def tearDown(self): 302 """Removes the directory if we were not instructed to do otherwise.""" 303 if not self.test_manager.leave_output: 304 shutil.rmtree(self.directory) 305 306 def runTest(self): 307 """Sets up and runs a test, reports any failures and then cleans up.""" 308 self.setUp() 309 success = False 310 message = '' 311 try: 312 self.command = [self.test_manager.executable_path] 313 self.command.extend(self.test.spirv_args) 314 315 process = subprocess.Popen( 316 args=self.command, 317 stdin=subprocess.PIPE, 318 stdout=subprocess.PIPE, 319 stderr=subprocess.PIPE, 320 cwd=self.directory) 321 output = process.communicate(self.stdin_shader) 322 test_status = TestStatus(self.test_manager, process.returncode, output[0], 323 output[1], self.directory, self.inputs, 324 self.file_shaders) 325 run_results = [ 326 getattr(self.test, test_method)(test_status) 327 for test_method in get_all_test_methods(self.test.__class__) 328 ] 329 success, message = zip(*run_results) 330 success = all(success) 331 message = '\n'.join(message) 332 except Exception as e: 333 success = False 334 message = str(e) 335 self.test_manager.notify_result( 336 self, success, 337 message + '\nSTDOUT:\n%s\nSTDERR:\n%s' % (output[0], output[1])) 338 self.tearDown() 339 340 341def main(): 342 parser = argparse.ArgumentParser() 343 parser.add_argument( 344 'spirv_tool', 345 metavar='path/to/spirv_tool', 346 type=str, 347 nargs=1, 348 help='Path to the spirv-* tool under test') 349 parser.add_argument( 350 'spirv_as', 351 metavar='path/to/spirv-as', 352 type=str, 353 nargs=1, 354 help='Path to spirv-as') 355 parser.add_argument( 356 'spirv_dis', 357 metavar='path/to/spirv-dis', 358 type=str, 359 nargs=1, 360 help='Path to spirv-dis') 361 parser.add_argument( 362 '--leave-output', 363 action='store_const', 364 const=1, 365 help='Do not clean up temporary directories') 366 parser.add_argument( 367 '--test-dir', nargs=1, help='Directory to gather the tests from') 368 args = parser.parse_args() 369 default_path = sys.path 370 root_dir = os.getcwd() 371 if args.test_dir: 372 root_dir = args.test_dir[0] 373 manager = TestManager(args.spirv_tool[0], args.spirv_as[0], args.spirv_dis[0]) 374 if args.leave_output: 375 manager.leave_output = True 376 for root, _, filenames in os.walk(root_dir): 377 for filename in fnmatch.filter(filenames, '*.py'): 378 if filename.endswith('nosetest.py'): 379 # Skip nose tests, which are for testing functions of 380 # the test framework. 381 continue 382 sys.path = default_path 383 sys.path.append(root) 384 mod = __import__(os.path.splitext(filename)[0]) 385 for _, obj, in inspect.getmembers(mod): 386 if inspect.isclass(obj) and hasattr(obj, 'parent_testsuite'): 387 manager.add_test(obj.parent_testsuite, obj()) 388 manager.run_tests() 389 if manager.num_failures > 0: 390 sys.exit(-1) 391 392 393if __name__ == '__main__': 394 main() 395