1# Copyright 2016 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5import os 6import string 7import sys 8import tempfile 9import unittest 10import json 11 12from telemetry import decorators 13from telemetry import project_config 14from telemetry.core import util 15from telemetry.testing import browser_test_context 16from telemetry.testing import browser_test_runner 17from telemetry.testing import options_for_unittests 18from telemetry.testing import run_browser_tests 19from telemetry.testing import serially_executed_browser_test_case 20 21 22class BrowserTestRunnerTest(unittest.TestCase): 23 24 def _ExtractTestResults(self, test_result): 25 delimiter = test_result['path_delimiter'] 26 failures = [] 27 successes = [] 28 def _IsLeafNode(node): 29 test_dict = node[1] 30 return ('expected' in test_dict and 31 isinstance(test_dict['expected'], basestring)) 32 node_queues = [] 33 for t in test_result['tests']: 34 node_queues.append((t, test_result['tests'][t])) 35 while node_queues: 36 node = node_queues.pop() 37 full_test_name, test_dict = node 38 if _IsLeafNode(node): 39 if all(res not in test_dict['expected'].split() for res in 40 test_dict['actual'].split()): 41 failures.append(full_test_name) 42 else: 43 successes.append(full_test_name) 44 else: 45 for k in test_dict: 46 node_queues.append( 47 ('%s%s%s' % (full_test_name, delimiter, k), 48 test_dict[k])) 49 return successes, failures 50 51 def baseTest(self, test_filter, 52 failures, successes, test_name='SimpleTest'): 53 config = project_config.ProjectConfig( 54 top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'), 55 client_configs=[], 56 benchmark_dirs=[ 57 os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests')] 58 ) 59 temp_file = tempfile.NamedTemporaryFile(delete=False) 60 temp_file.close() 61 temp_file_name = temp_file.name 62 try: 63 browser_test_runner.Run( 64 config, 65 [test_name, 66 '--write-full-results-to=%s' % temp_file_name, 67 '--test-filter=%s' % test_filter]) 68 with open(temp_file_name) as f: 69 test_result = json.load(f) 70 71 actual_successes, actual_failures = self._ExtractTestResults(test_result) 72 self.assertEquals(set(actual_failures), set(failures)) 73 self.assertEquals(set(actual_successes), set(successes)) 74 finally: 75 os.remove(temp_file_name) 76 77 @decorators.Disabled('chromeos') # crbug.com/696553 78 def testJsonOutputFormatNegativeFilter(self): 79 self.baseTest( 80 '^(add|multiplier).*', 81 ['browser_tests.simple_numeric_test.SimpleTest.add_1_and_2', 82 'browser_tests.simple_numeric_test.SimpleTest.add_7_and_3', 83 'browser_tests.simple_numeric_test.SimpleTest.multiplier_simple_2'], 84 ['browser_tests.simple_numeric_test.SimpleTest.add_2_and_3', 85 'browser_tests.simple_numeric_test.SimpleTest.multiplier_simple', 86 'browser_tests.simple_numeric_test.SimpleTest.multiplier_simple_3']) 87 88 @decorators.Disabled('chromeos') # crbug.com/696553 89 def testJsonOutputWhenSetupClassFailed(self): 90 self.baseTest( 91 '.*', 92 ['browser_tests.failed_tests.SetUpClassFailedTest.dummy_test_0', 93 'browser_tests.failed_tests.SetUpClassFailedTest.dummy_test_1', 94 'browser_tests.failed_tests.SetUpClassFailedTest.dummy_test_2'], 95 [], test_name='SetUpClassFailedTest') 96 97 @decorators.Disabled('chromeos') # crbug.com/696553 98 def testJsonOutputWhenTearDownClassFailed(self): 99 self.baseTest( 100 '.*', 101 ['browser_tests.failed_tests.TearDownClassFailedTest.dummy_test_0', 102 'browser_tests.failed_tests.TearDownClassFailedTest.dummy_test_1', 103 'browser_tests.failed_tests.TearDownClassFailedTest.dummy_test_2'], 104 [], test_name='TearDownClassFailedTest') 105 106 @decorators.Disabled('chromeos') # crbug.com/696553 107 def testSetUpProcessCalledOnce(self): 108 self.baseTest( 109 '.*', 110 [], 111 ['browser_tests.process_tests.FailIfSetUpProcessCalledTwice.Dummy_0', 112 'browser_tests.process_tests.FailIfSetUpProcessCalledTwice.Dummy_1', 113 'browser_tests.process_tests.FailIfSetUpProcessCalledTwice.Dummy_2'], 114 test_name='FailIfSetUpProcessCalledTwice') 115 116 @decorators.Disabled('chromeos') # crbug.com/696553 117 def testTearDownProcessCalledOnce(self): 118 self.baseTest( 119 '.*', 120 [], 121 ['browser_tests.process_tests.FailIfTearDownProcessCalledTwice.Dummy_0', 122 'browser_tests.process_tests.FailIfTearDownProcessCalledTwice.Dummy_1', 123 'browser_tests.process_tests.FailIfTearDownProcessCalledTwice.Dummy_2'], 124 test_name='FailIfTearDownProcessCalledTwice') 125 126 @decorators.Disabled('chromeos') # crbug.com/696553 127 def testJsonOutputFormatPositiveFilter(self): 128 self.baseTest( 129 '(TestSimple|TestException).*', 130 ['browser_tests.simple_numeric_test.SimpleTest.TestException', 131 'browser_tests.simple_numeric_test.SimpleTest.TestSimple'], []) 132 133 @decorators.Disabled('chromeos') # crbug.com/696553 134 def testExecutingTestsInSortedOrder(self): 135 alphabetical_tests = [] 136 prefix = 'browser_tests.simple_numeric_test.SimpleTest.Alphabetical_' 137 for i in xrange(20): 138 alphabetical_tests.append(prefix + str(i)) 139 for c in string.uppercase[:26]: 140 alphabetical_tests.append(prefix + c) 141 for c in string.lowercase[:26]: 142 alphabetical_tests.append(prefix + c) 143 alphabetical_tests.sort() 144 self.baseTest( 145 'Alphabetical', [], alphabetical_tests) 146 147 def shardingRangeTestHelper(self, total_shards, num_tests): 148 shard_ranges = [] 149 for shard_index in xrange(0, total_shards): 150 shard_ranges.append(run_browser_tests._TestRangeForShard( 151 total_shards, shard_index, num_tests)) 152 # Make assertions about ranges 153 num_tests_run = 0 154 for i in xrange(0, len(shard_ranges)): 155 cur_range = shard_ranges[i] 156 if i < num_tests: 157 self.assertGreater(cur_range[1], cur_range[0]) 158 num_tests_run += (cur_range[1] - cur_range[0]) 159 else: 160 # Not enough tests to go around all of the shards. 161 self.assertEquals(cur_range[0], cur_range[1]) 162 # Make assertions about non-overlapping ranges 163 for i in xrange(1, len(shard_ranges)): 164 prev_range = shard_ranges[i - 1] 165 cur_range = shard_ranges[i] 166 self.assertEquals(prev_range[1], cur_range[0]) 167 # Assert that we run all of the tests (very important) 168 self.assertEquals(num_tests_run, num_tests) 169 170 def testShardsWithPrimeNumTests(self): 171 for total_shards in xrange(1, 20): 172 # Nice non-prime number 173 self.shardingRangeTestHelper(total_shards, 101) 174 175 def testShardsWithDivisibleNumTests(self): 176 for total_shards in xrange(1, 6): 177 self.shardingRangeTestHelper(total_shards, 8) 178 179 def testShardBoundaryConditions(self): 180 self.shardingRangeTestHelper(1, 0) 181 self.shardingRangeTestHelper(1, 1) 182 self.shardingRangeTestHelper(2, 1) 183 184 def baseShardingTest(self, total_shards, shard_index, failures, successes, 185 opt_abbr_input_json_file=None, 186 opt_test_filter='', 187 opt_filter_tests_after_sharding=False): 188 config = project_config.ProjectConfig( 189 top_level_dir=os.path.join(util.GetTelemetryDir(), 'examples'), 190 client_configs=[], 191 benchmark_dirs=[ 192 os.path.join(util.GetTelemetryDir(), 'examples', 'browser_tests')] 193 ) 194 temp_file = tempfile.NamedTemporaryFile(delete=False) 195 temp_file.close() 196 temp_file_name = temp_file.name 197 opt_args = [] 198 if opt_abbr_input_json_file: 199 opt_args += [ 200 '--read-abbreviated-json-results-from=%s' % opt_abbr_input_json_file] 201 if opt_test_filter: 202 opt_args += [ 203 '--test-filter=%s' % opt_test_filter] 204 if opt_filter_tests_after_sharding: 205 opt_args += ['--filter-tests-after-sharding'] 206 try: 207 browser_test_runner.Run( 208 config, 209 ['SimpleShardingTest', 210 '--write-full-results-to=%s' % temp_file_name, 211 '--total-shards=%d' % total_shards, 212 '--shard-index=%d' % shard_index] + opt_args) 213 with open(temp_file_name) as f: 214 test_result = json.load(f) 215 actual_successes, actual_failures = self._ExtractTestResults(test_result) 216 self.assertEquals(set(actual_failures), set(failures)) 217 self.assertEquals(set(actual_successes), set(successes)) 218 finally: 219 os.remove(temp_file_name) 220 221 @decorators.Disabled('chromeos') # crbug.com/696553 222 def testShardedTestRun(self): 223 self.baseShardingTest(3, 0, [], [ 224 'browser_tests.simple_sharding_test.SimpleShardingTest.Test1', 225 'browser_tests.simple_sharding_test.SimpleShardingTest.Test2', 226 'browser_tests.simple_sharding_test.SimpleShardingTest.Test3', 227 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_0', 228 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_1', 229 ]) 230 self.baseShardingTest(3, 1, [], [ 231 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_2', 232 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_3', 233 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_4', 234 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_5', 235 ]) 236 self.baseShardingTest(3, 2, [], [ 237 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_6', 238 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_7', 239 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_8', 240 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_9', 241 ]) 242 243 def writeMockTestResultsFile(self): 244 mock_test_results = { 245 'passes': [ 246 'Test1', 247 'Test2', 248 'Test3', 249 'passing_test_0', 250 'passing_test_1', 251 'passing_test_2', 252 'passing_test_3', 253 'passing_test_4', 254 'passing_test_5', 255 'passing_test_6', 256 'passing_test_7', 257 'passing_test_8', 258 'passing_test_9', 259 ], 260 'failures': [], 261 'valid': True, 262 'times': { 263 'Test1': 3.0, 264 'Test2': 3.0, 265 'Test3': 3.0, 266 'passing_test_0': 3.0, 267 'passing_test_1': 2.0, 268 'passing_test_2': 2.0, 269 'passing_test_3': 2.0, 270 'passing_test_4': 2.0, 271 'passing_test_5': 1.0, 272 'passing_test_6': 1.0, 273 'passing_test_7': 1.0, 274 'passing_test_8': 1.0, 275 'passing_test_9': 0.5, 276 } 277 } 278 temp_file = tempfile.NamedTemporaryFile(delete=False) 279 temp_file.close() 280 temp_file_name = temp_file.name 281 with open(temp_file_name, 'w') as f: 282 json.dump(mock_test_results, f) 283 return temp_file_name 284 285 @decorators.Disabled('chromeos') # crbug.com/696553 286 def testSplittingShardsByTimes(self): 287 temp_file_name = self.writeMockTestResultsFile() 288 # It seems that the sorting order of the first four tests above is: 289 # passing_test_0, Test1, Test2, Test3 290 # This is probably because the relative order of the "fixed" tests 291 # (starting with "Test") and the generated ones ("passing_") is 292 # not well defined, and the sorting is stable afterward. The 293 # expectations have been adjusted for this fact. 294 try: 295 self.baseShardingTest( 296 4, 0, [], 297 ['browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_0', 298 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_1', 299 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_5', 300 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_9' 301 ], temp_file_name) 302 self.baseShardingTest( 303 4, 1, [], 304 ['browser_tests.simple_sharding_test.SimpleShardingTest.Test1', 305 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_2', 306 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_6' 307 ], temp_file_name) 308 self.baseShardingTest( 309 4, 2, [], 310 ['browser_tests.simple_sharding_test.SimpleShardingTest.Test2', 311 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_3', 312 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_7' 313 ], temp_file_name) 314 self.baseShardingTest( 315 4, 3, [], 316 ['browser_tests.simple_sharding_test.SimpleShardingTest.Test3', 317 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_4', 318 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_8' 319 ], temp_file_name) 320 finally: 321 os.remove(temp_file_name) 322 323 @decorators.Disabled('chromeos') # crbug.com/696553 324 def testFilteringAfterSharding(self): 325 temp_file_name = self.writeMockTestResultsFile() 326 try: 327 self.baseShardingTest( 328 4, 1, [], 329 ['browser_tests.simple_sharding_test.SimpleShardingTest.Test1', 330 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_2', 331 'browser_tests.simple_sharding_test.SimpleShardingTest.passing_test_6' 332 ], temp_file_name, 333 opt_test_filter='(Test1|passing_test_2|passing_test_6)', 334 opt_filter_tests_after_sharding=True) 335 finally: 336 os.remove(temp_file_name) 337 338 def testMedianComputation(self): 339 self.assertEquals(2.0, run_browser_tests._MedianTestTime( 340 {'test1': 2.0, 'test2': 7.0, 'test3': 1.0})) 341 self.assertEquals(2.0, run_browser_tests._MedianTestTime( 342 {'test1': 2.0})) 343 self.assertEquals(0.0, run_browser_tests._MedianTestTime({})) 344 self.assertEqual(4.0, run_browser_tests._MedianTestTime( 345 {'test1': 2.0, 'test2': 6.0, 'test3': 1.0, 'test4': 8.0})) 346 347 348class Algebra( 349 serially_executed_browser_test_case.SeriallyExecutedBrowserTestCase): 350 351 @classmethod 352 def GenerateTestCases_Simple(cls, options): 353 del options # Unused. 354 yield 'testOne', (1, 2) 355 yield 'testTwo', (3, 3) 356 357 def Simple(self, x, y): 358 self.assertEquals(x, y) 359 360 def TestNumber(self): 361 self.assertEquals(0, 1) 362 363 364class ErrorneousGeometric( 365 serially_executed_browser_test_case.SeriallyExecutedBrowserTestCase): 366 367 @classmethod 368 def GenerateTestCases_Compare(cls, options): 369 del options # Unused. 370 assert False, 'I am a problematic generator' 371 yield 'testBasic', ('square', 'circle') 372 373 def Compare(self, x, y): 374 self.assertEquals(x, y) 375 376 def TestAngle(self): 377 self.assertEquals(90, 450) 378 379class TestLoadAllTestModules(unittest.TestCase): 380 def testLoadAllTestsInModule(self): 381 context = browser_test_context.TypTestContext() 382 context.finder_options = options_for_unittests.GetCopy() 383 context.test_class = Algebra 384 context.test_case_ids_to_run.add( 385 'telemetry.testing.browser_test_runner_unittest.Algebra.TestNumber') 386 context.test_case_ids_to_run.add( 387 'telemetry.testing.browser_test_runner_unittest.Algebra.testOne') 388 context.Freeze() 389 browser_test_context._global_test_context = context 390 try: 391 # This should not invoke GenerateTestCases of ErrorneousGeometric class, 392 # otherwise that would throw Exception. 393 tests = serially_executed_browser_test_case.LoadAllTestsInModule( 394 sys.modules[__name__]) 395 self.assertEquals(sorted([t.id() for t in tests]), 396 ['telemetry.testing.browser_test_runner_unittest.Algebra.TestNumber', 397 'telemetry.testing.browser_test_runner_unittest.Algebra.testOne']) 398 finally: 399 browser_test_context._global_test_context = None 400