1#!/usr/bin/env python 2# Copyright (c) 2017 The WebRTC project authors. All Rights Reserved. 3# 4# Use of this source code is governed by a BSD-style license 5# that can be found in the LICENSE file in the root of the source 6# tree. An additional intellectual property rights grant can be found 7# in the file PATENTS. All contributing project authors may 8# be found in the AUTHORS file in the root of the source tree. 9 10''' Runs various chrome tests through valgrind_test.py.''' 11 12import glob 13import logging 14import multiprocessing 15import optparse 16import os 17import stat 18import subprocess 19import sys 20 21import logging_utils 22import path_utils 23 24import common 25import valgrind_test 26 27class TestNotFound(Exception): pass 28 29class MultipleGTestFiltersSpecified(Exception): pass 30 31class BuildDirNotFound(Exception): pass 32 33class BuildDirAmbiguous(Exception): pass 34 35class ExecutableNotFound(Exception): pass 36 37class BadBinary(Exception): pass 38 39class ChromeTests: 40 SLOW_TOOLS = ["memcheck", "drmemory"] 41 LAYOUT_TESTS_DEFAULT_CHUNK_SIZE = 300 42 43 def __init__(self, options, args, test): 44 if ':' in test: 45 (self._test, self._gtest_filter) = test.split(':', 1) 46 else: 47 self._test = test 48 self._gtest_filter = options.gtest_filter 49 50 if self._test not in self._test_list: 51 raise TestNotFound("Unknown test: %s" % test) 52 53 if options.gtest_filter and options.gtest_filter != self._gtest_filter: 54 raise MultipleGTestFiltersSpecified("Can not specify both --gtest_filter " 55 "and --test %s" % test) 56 57 self._options = options 58 self._args = args 59 60 script_dir = path_utils.ScriptDir() 61 # Compute the top of the tree (the "source dir") from the script dir (where 62 # this script lives). We assume that the script dir is in tools/valgrind/ 63 # relative to the top of the tree. 64 self._source_dir = os.path.dirname(os.path.dirname(script_dir)) 65 # since this path is used for string matching, make sure it's always 66 # an absolute Unix-style path 67 self._source_dir = os.path.abspath(self._source_dir).replace('\\', '/') 68 valgrind_test_script = os.path.join(script_dir, "valgrind_test.py") 69 self._command_preamble = ["--source-dir=%s" % (self._source_dir)] 70 71 if not self._options.build_dir: 72 dirs = [ 73 os.path.join(self._source_dir, "xcodebuild", "Debug"), 74 os.path.join(self._source_dir, "out", "Debug"), 75 os.path.join(self._source_dir, "build", "Debug"), 76 ] 77 build_dir = [d for d in dirs if os.path.isdir(d)] 78 if len(build_dir) > 1: 79 raise BuildDirAmbiguous("Found more than one suitable build dir:\n" 80 "%s\nPlease specify just one " 81 "using --build-dir" % ", ".join(build_dir)) 82 elif build_dir: 83 self._options.build_dir = build_dir[0] 84 else: 85 self._options.build_dir = None 86 87 if self._options.build_dir: 88 build_dir = os.path.abspath(self._options.build_dir) 89 self._command_preamble += ["--build-dir=%s" % (self._options.build_dir)] 90 91 def _EnsureBuildDirFound(self): 92 if not self._options.build_dir: 93 raise BuildDirNotFound("Oops, couldn't find a build dir, please " 94 "specify it manually using --build-dir") 95 96 def _DefaultCommand(self, tool, exe=None, valgrind_test_args=None): 97 '''Generates the default command array that most tests will use.''' 98 if exe and common.IsWindows(): 99 exe += '.exe' 100 101 cmd = list(self._command_preamble) 102 103 # Find all suppressions matching the following pattern: 104 # tools/valgrind/TOOL/suppressions[_PLATFORM].txt 105 # and list them with --suppressions= prefix. 106 script_dir = path_utils.ScriptDir() 107 tool_name = tool.ToolName(); 108 suppression_file = os.path.join(script_dir, tool_name, "suppressions.txt") 109 if os.path.exists(suppression_file): 110 cmd.append("--suppressions=%s" % suppression_file) 111 # Platform-specific suppression 112 for platform in common.PlatformNames(): 113 platform_suppression_file = \ 114 os.path.join(script_dir, tool_name, 'suppressions_%s.txt' % platform) 115 if os.path.exists(platform_suppression_file): 116 cmd.append("--suppressions=%s" % platform_suppression_file) 117 118 if tool_name == "drmemory": 119 if self._options.drmemory_ops: 120 # prepending " " to avoid Dr. Memory's option confusing optparse 121 cmd += ["--drmemory_ops", " " + self._options.drmemory_ops] 122 123 if self._options.valgrind_tool_flags: 124 cmd += self._options.valgrind_tool_flags.split(" ") 125 if self._options.keep_logs: 126 cmd += ["--keep_logs"] 127 if valgrind_test_args != None: 128 for arg in valgrind_test_args: 129 cmd.append(arg) 130 if exe: 131 self._EnsureBuildDirFound() 132 exe_path = os.path.join(self._options.build_dir, exe) 133 if not os.path.exists(exe_path): 134 raise ExecutableNotFound("Couldn't find '%s'" % exe_path) 135 136 # Make sure we don't try to test ASan-built binaries 137 # with other dynamic instrumentation-based tools. 138 # TODO(timurrrr): also check TSan and MSan? 139 # `nm` might not be available, so use try-except. 140 try: 141 # Do not perform this check on OS X, as 'nm' on 10.6 can't handle 142 # binaries built with Clang 3.5+. 143 if not common.IsMac(): 144 nm_output = subprocess.check_output(["nm", exe_path]) 145 if nm_output.find("__asan_init") != -1: 146 raise BadBinary("You're trying to run an executable instrumented " 147 "with AddressSanitizer under %s. Please provide " 148 "an uninstrumented executable." % tool_name) 149 except OSError: 150 pass 151 152 cmd.append(exe_path) 153 # Valgrind runs tests slowly, so slow tests hurt more; show elapased time 154 # so we can find the slowpokes. 155 cmd.append("--gtest_print_time") 156 # Built-in test launcher for gtest-based executables runs tests using 157 # multiple process by default. Force the single-process mode back. 158 cmd.append("--single-process-tests") 159 if self._options.gtest_repeat: 160 cmd.append("--gtest_repeat=%s" % self._options.gtest_repeat) 161 if self._options.gtest_shuffle: 162 cmd.append("--gtest_shuffle") 163 if self._options.gtest_break_on_failure: 164 cmd.append("--gtest_break_on_failure") 165 if self._options.test_launcher_bot_mode: 166 cmd.append("--test-launcher-bot-mode") 167 if self._options.test_launcher_total_shards is not None: 168 cmd.append("--test-launcher-total-shards=%d" 169 % self._options.test_launcher_total_shards) 170 if self._options.test_launcher_shard_index is not None: 171 cmd.append("--test-launcher-shard-index=%d" 172 % self._options.test_launcher_shard_index) 173 return cmd 174 175 def Run(self): 176 ''' Runs the test specified by command-line argument --test ''' 177 logging.info("running test %s" % (self._test)) 178 return self._test_list[self._test](self) 179 180 def _AppendGtestFilter(self, tool, name, cmd): 181 '''Append an appropriate --gtest_filter flag to the googletest binary 182 invocation. 183 If the user passed their own filter mentioning only one test, just use 184 it. Otherwise, filter out tests listed in the appropriate gtest_exclude 185 files. 186 ''' 187 if (self._gtest_filter and 188 ":" not in self._gtest_filter and 189 "?" not in self._gtest_filter and 190 "*" not in self._gtest_filter): 191 cmd.append("--gtest_filter=%s" % self._gtest_filter) 192 return 193 194 filters = [] 195 gtest_files_dir = os.path.join(path_utils.ScriptDir(), "gtest_exclude") 196 197 gtest_filter_files = [ 198 os.path.join(gtest_files_dir, name + ".gtest-%s.txt" % tool.ToolName())] 199 # Use ".gtest.txt" files only for slow tools, as they now contain 200 # Valgrind- and Dr.Memory-specific filters. 201 # TODO(glider): rename the files to ".gtest_slow.txt" 202 if tool.ToolName() in ChromeTests.SLOW_TOOLS: 203 gtest_filter_files += [os.path.join(gtest_files_dir, name + ".gtest.txt")] 204 for platform_suffix in common.PlatformNames(): 205 gtest_filter_files += [ 206 os.path.join(gtest_files_dir, name + ".gtest_%s.txt" % platform_suffix), 207 os.path.join(gtest_files_dir, name + ".gtest-%s_%s.txt" % \ 208 (tool.ToolName(), platform_suffix))] 209 logging.info("Reading gtest exclude filter files:") 210 for filename in gtest_filter_files: 211 # strip the leading absolute path (may be very long on the bot) 212 # and the following / or \. 213 readable_filename = filename.replace("\\", "/") # '\' on Windows 214 readable_filename = readable_filename.replace(self._source_dir, "")[1:] 215 if not os.path.exists(filename): 216 logging.info(" \"%s\" - not found" % readable_filename) 217 continue 218 logging.info(" \"%s\" - OK" % readable_filename) 219 f = open(filename, 'r') 220 for line in f.readlines(): 221 if line.startswith("#") or line.startswith("//") or line.isspace(): 222 continue 223 line = line.rstrip() 224 test_prefixes = ["FLAKY", "FAILS"] 225 for p in test_prefixes: 226 # Strip prefixes from the test names. 227 line = line.replace(".%s_" % p, ".") 228 # Exclude the original test name. 229 filters.append(line) 230 if line[-2:] != ".*": 231 # List all possible prefixes if line doesn't end with ".*". 232 for p in test_prefixes: 233 filters.append(line.replace(".", ".%s_" % p)) 234 # Get rid of duplicates. 235 filters = set(filters) 236 gtest_filter = self._gtest_filter 237 if len(filters): 238 if gtest_filter: 239 gtest_filter += ":" 240 if gtest_filter.find("-") < 0: 241 gtest_filter += "-" 242 else: 243 gtest_filter = "-" 244 gtest_filter += ":".join(filters) 245 if gtest_filter: 246 cmd.append("--gtest_filter=%s" % gtest_filter) 247 248 @staticmethod 249 def ShowTests(): 250 test_to_names = {} 251 for name, test_function in ChromeTests._test_list.iteritems(): 252 test_to_names.setdefault(test_function, []).append(name) 253 254 name_to_aliases = {} 255 for names in test_to_names.itervalues(): 256 names.sort(key=lambda name: len(name)) 257 name_to_aliases[names[0]] = names[1:] 258 259 print 260 print "Available tests:" 261 print "----------------" 262 for name, aliases in sorted(name_to_aliases.iteritems()): 263 if aliases: 264 print " {} (aka {})".format(name, ', '.join(aliases)) 265 else: 266 print " {}".format(name) 267 268 def SetupLdPath(self, requires_build_dir): 269 if requires_build_dir: 270 self._EnsureBuildDirFound() 271 elif not self._options.build_dir: 272 return 273 274 # Append build_dir to LD_LIBRARY_PATH so external libraries can be loaded. 275 if (os.getenv("LD_LIBRARY_PATH")): 276 os.putenv("LD_LIBRARY_PATH", "%s:%s" % (os.getenv("LD_LIBRARY_PATH"), 277 self._options.build_dir)) 278 else: 279 os.putenv("LD_LIBRARY_PATH", self._options.build_dir) 280 281 def SimpleTest(self, module, name, valgrind_test_args=None, cmd_args=None): 282 tool = valgrind_test.CreateTool(self._options.valgrind_tool) 283 cmd = self._DefaultCommand(tool, name, valgrind_test_args) 284 self._AppendGtestFilter(tool, name, cmd) 285 cmd.extend(['--test-tiny-timeout=1000']) 286 if cmd_args: 287 cmd.extend(cmd_args) 288 289 self.SetupLdPath(True) 290 return tool.Run(cmd, module) 291 292 def RunCmdLine(self): 293 tool = valgrind_test.CreateTool(self._options.valgrind_tool) 294 cmd = self._DefaultCommand(tool, None, self._args) 295 self.SetupLdPath(False) 296 return tool.Run(cmd, None) 297 298 def TestAccessibility(self): 299 return self.SimpleTest("accessibility", "accessibility_unittests") 300 301 def TestAddressInput(self): 302 return self.SimpleTest("addressinput", "libaddressinput_unittests") 303 304 def TestAngle(self): 305 return self.SimpleTest("angle", "angle_unittests") 306 307 def TestAppList(self): 308 return self.SimpleTest("app_list", "app_list_unittests") 309 310 def TestAsh(self): 311 return self.SimpleTest("ash", "ash_unittests") 312 313 def TestAura(self): 314 return self.SimpleTest("aura", "aura_unittests") 315 316 def TestBase(self): 317 return self.SimpleTest("base", "base_unittests") 318 319 def TestBlinkHeap(self): 320 return self.SimpleTest("blink_heap", "blink_heap_unittests") 321 322 def TestBlinkPlatform(self): 323 return self.SimpleTest("blink_platform", "blink_platform_unittests") 324 325 def TestCacheInvalidation(self): 326 return self.SimpleTest("cacheinvalidation", "cacheinvalidation_unittests") 327 328 def TestCast(self): 329 return self.SimpleTest("chrome", "cast_unittests") 330 331 def TestCC(self): 332 return self.SimpleTest("cc", "cc_unittests", 333 cmd_args=[ 334 "--cc-layer-tree-test-long-timeout"]) 335 336 def TestChromeApp(self): 337 return self.SimpleTest("chrome_app", "chrome_app_unittests") 338 339 def TestChromeElf(self): 340 return self.SimpleTest("chrome_elf", "chrome_elf_unittests") 341 342 def TestChromeDriver(self): 343 return self.SimpleTest("chromedriver", "chromedriver_unittests") 344 345 def TestChromeOS(self): 346 return self.SimpleTest("chromeos", "chromeos_unittests") 347 348 def TestComponents(self): 349 return self.SimpleTest("components", "components_unittests") 350 351 def TestCompositor(self): 352 return self.SimpleTest("compositor", "compositor_unittests") 353 354 def TestContent(self): 355 return self.SimpleTest("content", "content_unittests") 356 357 def TestCourgette(self): 358 return self.SimpleTest("courgette", "courgette_unittests") 359 360 def TestCrypto(self): 361 return self.SimpleTest("crypto", "crypto_unittests") 362 363 def TestDevice(self): 364 return self.SimpleTest("device", "device_unittests") 365 366 def TestDisplay(self): 367 return self.SimpleTest("display", "display_unittests") 368 369 def TestEvents(self): 370 return self.SimpleTest("events", "events_unittests") 371 372 def TestExtensions(self): 373 return self.SimpleTest("extensions", "extensions_unittests") 374 375 def TestFFmpegRegressions(self): 376 return self.SimpleTest("chrome", "ffmpeg_regression_tests") 377 378 def TestGCM(self): 379 return self.SimpleTest("gcm", "gcm_unit_tests") 380 381 def TestGfx(self): 382 return self.SimpleTest("gfx", "gfx_unittests") 383 384 def TestGin(self): 385 return self.SimpleTest("gin", "gin_unittests") 386 387 def TestGoogleApis(self): 388 return self.SimpleTest("google_apis", "google_apis_unittests") 389 390 def TestGPU(self): 391 return self.SimpleTest("gpu", "gpu_unittests") 392 393 def TestIpc(self): 394 return self.SimpleTest("ipc", "ipc_tests", 395 valgrind_test_args=["--trace_children"]) 396 397 def TestInstallerUtil(self): 398 return self.SimpleTest("installer_util", "installer_util_unittests") 399 400 def TestInstallStatic(self): 401 return self.SimpleTest("install_static", "install_static_unittests") 402 403 def TestJingle(self): 404 return self.SimpleTest("chrome", "jingle_unittests") 405 406 def TestKeyboard(self): 407 return self.SimpleTest("keyboard", "keyboard_unittests") 408 409 def TestLatency(self): 410 return self.SimpleTest("latency", "latency_unittests") 411 412 def TestMedia(self): 413 return self.SimpleTest("chrome", "media_unittests") 414 415 def TestMessageCenter(self): 416 return self.SimpleTest("message_center", "message_center_unittests") 417 418 def TestMidi(self): 419 return self.SimpleTest("chrome", "midi_unittests") 420 421 def TestMojoCommon(self): 422 return self.SimpleTest("mojo_common", "mojo_common_unittests") 423 424 def TestMojoPublicBindings(self): 425 return self.SimpleTest("mojo_public_bindings", 426 "mojo_public_bindings_unittests") 427 428 def TestMojoPublicSystem(self): 429 return self.SimpleTest("mojo_public_system", 430 "mojo_public_system_unittests") 431 432 def TestMojoPublicSysPerf(self): 433 return self.SimpleTest("mojo_public_sysperf", 434 "mojo_public_system_perftests") 435 436 def TestMojoSystem(self): 437 return self.SimpleTest("mojo_system", "mojo_system_unittests") 438 439 def TestNet(self): 440 return self.SimpleTest("net", "net_unittests") 441 442 def TestNetPerf(self): 443 return self.SimpleTest("net", "net_perftests") 444 445 def TestPhoneNumber(self): 446 return self.SimpleTest("phonenumber", "libphonenumber_unittests") 447 448 def TestPPAPI(self): 449 return self.SimpleTest("chrome", "ppapi_unittests") 450 451 def TestPrinting(self): 452 return self.SimpleTest("chrome", "printing_unittests") 453 454 def TestRemoting(self): 455 return self.SimpleTest("chrome", "remoting_unittests", 456 cmd_args=[ 457 "--ui-test-action-timeout=60000", 458 "--ui-test-action-max-timeout=150000"]) 459 460 def TestSkia(self): 461 return self.SimpleTest("skia", "skia_unittests") 462 463 def TestSql(self): 464 return self.SimpleTest("chrome", "sql_unittests") 465 466 def TestStorage(self): 467 return self.SimpleTest("storage", "storage_unittests") 468 469 def TestLinuxSandbox(self): 470 return self.SimpleTest("sandbox", "sandbox_linux_unittests") 471 472 def TestUnit(self): 473 # http://crbug.com/51716 474 # Disabling all unit tests 475 # Problems reappeared after r119922 476 if common.IsMac() and (self._options.valgrind_tool == "memcheck"): 477 logging.warning("unit_tests are disabled for memcheck on MacOS.") 478 return 0; 479 return self.SimpleTest("chrome", "unit_tests") 480 481 def TestUIBaseUnit(self): 482 return self.SimpleTest("chrome", "ui_base_unittests") 483 484 def TestUIChromeOS(self): 485 return self.SimpleTest("chrome", "ui_chromeos_unittests") 486 487 def TestURL(self): 488 return self.SimpleTest("chrome", "url_unittests") 489 490 def TestViews(self): 491 return self.SimpleTest("views", "views_unittests") 492 493 494 # Valgrind timeouts are in seconds. 495 UI_VALGRIND_ARGS = ["--timeout=14400", "--trace_children", "--indirect"] 496 # UI test timeouts are in milliseconds. 497 UI_TEST_ARGS = ["--ui-test-action-timeout=60000", 498 "--ui-test-action-max-timeout=150000", 499 "--no-sandbox"] 500 501 # TODO(thestig) fine-tune these values. 502 # Valgrind timeouts are in seconds. 503 BROWSER_VALGRIND_ARGS = ["--timeout=50000", "--trace_children", "--indirect"] 504 # Browser test timeouts are in milliseconds. 505 BROWSER_TEST_ARGS = ["--ui-test-action-timeout=400000", 506 "--ui-test-action-max-timeout=800000", 507 "--no-sandbox"] 508 509 def TestBrowser(self): 510 return self.SimpleTest("chrome", "browser_tests", 511 valgrind_test_args=self.BROWSER_VALGRIND_ARGS, 512 cmd_args=self.BROWSER_TEST_ARGS) 513 514 def TestContentBrowser(self): 515 return self.SimpleTest("content", "content_browsertests", 516 valgrind_test_args=self.BROWSER_VALGRIND_ARGS, 517 cmd_args=self.BROWSER_TEST_ARGS) 518 519 def TestInteractiveUI(self): 520 return self.SimpleTest("chrome", "interactive_ui_tests", 521 valgrind_test_args=self.UI_VALGRIND_ARGS, 522 cmd_args=self.UI_TEST_ARGS) 523 524 def TestSyncIntegration(self): 525 return self.SimpleTest("chrome", "sync_integration_tests", 526 valgrind_test_args=self.UI_VALGRIND_ARGS, 527 cmd_args=(["--ui-test-action-max-timeout=450000"])) 528 529 def TestLayoutChunk(self, chunk_num, chunk_size): 530 # Run tests [chunk_num*chunk_size .. (chunk_num+1)*chunk_size) from the 531 # list of tests. Wrap around to beginning of list at end. 532 # If chunk_size is zero, run all tests in the list once. 533 # If a text file is given as argument, it is used as the list of tests. 534 assert((chunk_size == 0) != (len(self._args) == 0)) 535 # Build the ginormous commandline in 'cmd'. 536 # It's going to be roughly 537 # python valgrind_test.py ... 538 # but we'll use the --indirect flag to valgrind_test.py 539 # to avoid valgrinding python. 540 # Start by building the valgrind_test.py commandline. 541 tool = valgrind_test.CreateTool(self._options.valgrind_tool) 542 cmd = self._DefaultCommand(tool) 543 cmd.append("--trace_children") 544 cmd.append("--indirect_webkit_layout") 545 cmd.append("--ignore_exit_code") 546 # Now build script_cmd, the run-webkits-tests commandline. 547 # Store each chunk in its own directory so that we can find the data later 548 chunk_dir = os.path.join("layout", "chunk_%05d" % chunk_num) 549 out_dir = os.path.join(path_utils.ScriptDir(), "latest") 550 out_dir = os.path.join(out_dir, chunk_dir) 551 if os.path.exists(out_dir): 552 old_files = glob.glob(os.path.join(out_dir, "*.txt")) 553 for f in old_files: 554 os.remove(f) 555 else: 556 os.makedirs(out_dir) 557 script = os.path.join(self._source_dir, "third_party", "WebKit", "Tools", 558 "Scripts", "run-webkit-tests") 559 # http://crbug.com/260627: After the switch to content_shell from DRT, each 560 # test now brings up 3 processes. Under Valgrind, they become memory bound 561 # and can eventually OOM if we don't reduce the total count. 562 # It'd be nice if content_shell automatically throttled the startup of new 563 # tests if we're low on memory. 564 jobs = max(1, int(multiprocessing.cpu_count() * 0.3)) 565 script_cmd = ["python", script, "-v", 566 # run a separate DumpRenderTree for each test 567 "--batch-size=1", 568 "--fully-parallel", 569 "--child-processes=%d" % jobs, 570 "--time-out-ms=800000", 571 "--no-retry-failures", # retrying takes too much time 572 # http://crbug.com/176908: Don't launch a browser when done. 573 "--no-show-results", 574 "--nocheck-sys-deps", 575 "--additional-driver-flag=--no-sandbox"] 576 # Pass build mode to run-webkit-tests. We aren't passed it directly, 577 # so parse it out of build_dir. run-webkit-tests can only handle 578 # the two values "Release" and "Debug". 579 # TODO(Hercules): unify how all our scripts pass around build mode 580 # (--mode / --target / --build-dir / --debug) 581 if self._options.build_dir: 582 build_root, mode = os.path.split(self._options.build_dir) 583 script_cmd.extend(["--build-directory", build_root, "--target", mode]) 584 if (chunk_size > 0): 585 script_cmd.append("--run-chunk=%d:%d" % (chunk_num, chunk_size)) 586 if len(self._args): 587 # if the arg is a txt file, then treat it as a list of tests 588 if os.path.isfile(self._args[0]) and self._args[0][-4:] == ".txt": 589 script_cmd.append("--test-list=%s" % self._args[0]) 590 else: 591 script_cmd.extend(self._args) 592 self._AppendGtestFilter(tool, "layout", script_cmd) 593 # Now run script_cmd with the wrapper in cmd 594 cmd.extend(["--"]) 595 cmd.extend(script_cmd) 596 597 # Layout tests often times fail quickly, but the buildbot remains green. 598 # Detect this situation when running with the default chunk size. 599 if chunk_size == self.LAYOUT_TESTS_DEFAULT_CHUNK_SIZE: 600 min_runtime_in_seconds=120 601 else: 602 min_runtime_in_seconds=0 603 ret = tool.Run(cmd, "layout", min_runtime_in_seconds=min_runtime_in_seconds) 604 return ret 605 606 607 def TestLayout(self): 608 # A "chunk file" is maintained in the local directory so that each test 609 # runs a slice of the layout tests of size chunk_size that increments with 610 # each run. Since tests can be added and removed from the layout tests at 611 # any time, this is not going to give exact coverage, but it will allow us 612 # to continuously run small slices of the layout tests under valgrind rather 613 # than having to run all of them in one shot. 614 chunk_size = self._options.num_tests 615 if chunk_size == 0 or len(self._args): 616 return self.TestLayoutChunk(0, 0) 617 chunk_num = 0 618 chunk_file = os.path.join("valgrind_layout_chunk.txt") 619 logging.info("Reading state from " + chunk_file) 620 try: 621 f = open(chunk_file) 622 if f: 623 chunk_str = f.read() 624 if len(chunk_str): 625 chunk_num = int(chunk_str) 626 # This should be enough so that we have a couple of complete runs 627 # of test data stored in the archive (although note that when we loop 628 # that we almost guaranteed won't be at the end of the test list) 629 if chunk_num > 10000: 630 chunk_num = 0 631 f.close() 632 except IOError, (errno, strerror): 633 logging.error("error reading from file %s (%d, %s)" % (chunk_file, 634 errno, strerror)) 635 # Save the new chunk size before running the tests. Otherwise if a 636 # particular chunk hangs the bot, the chunk number will never get 637 # incremented and the bot will be wedged. 638 logging.info("Saving state to " + chunk_file) 639 try: 640 f = open(chunk_file, "w") 641 chunk_num += 1 642 f.write("%d" % chunk_num) 643 f.close() 644 except IOError, (errno, strerror): 645 logging.error("error writing to file %s (%d, %s)" % (chunk_file, errno, 646 strerror)) 647 # Since we're running small chunks of the layout tests, it's important to 648 # mark the ones that have errors in them. These won't be visible in the 649 # summary list for long, but will be useful for someone reviewing this bot. 650 return self.TestLayoutChunk(chunk_num, chunk_size) 651 652 # The known list of tests. 653 # Recognise the original abbreviations as well as full executable names. 654 _test_list = { 655 "cmdline" : RunCmdLine, 656 "addressinput": TestAddressInput, 657 "libaddressinput_unittests": TestAddressInput, 658 "accessibility": TestAccessibility, 659 "angle": TestAngle, "angle_unittests": TestAngle, 660 "app_list": TestAppList, "app_list_unittests": TestAppList, 661 "ash": TestAsh, "ash_unittests": TestAsh, 662 "aura": TestAura, "aura_unittests": TestAura, 663 "base": TestBase, "base_unittests": TestBase, 664 "blink_heap": TestBlinkHeap, 665 "blink_platform": TestBlinkPlatform, 666 "browser": TestBrowser, "browser_tests": TestBrowser, 667 "cacheinvalidation": TestCacheInvalidation, 668 "cacheinvalidation_unittests": TestCacheInvalidation, 669 "cast": TestCast, "cast_unittests": TestCast, 670 "cc": TestCC, "cc_unittests": TestCC, 671 "chrome_app": TestChromeApp, 672 "chrome_elf": TestChromeElf, 673 "chromedriver": TestChromeDriver, 674 "chromeos": TestChromeOS, "chromeos_unittests": TestChromeOS, 675 "components": TestComponents,"components_unittests": TestComponents, 676 "compositor": TestCompositor,"compositor_unittests": TestCompositor, 677 "content": TestContent, "content_unittests": TestContent, 678 "content_browsertests": TestContentBrowser, 679 "courgette": TestCourgette, "courgette_unittests": TestCourgette, 680 "crypto": TestCrypto, "crypto_unittests": TestCrypto, 681 "device": TestDevice, "device_unittests": TestDevice, 682 "display": TestDisplay, "display_unittests": TestDisplay, 683 "events": TestEvents, "events_unittests": TestEvents, 684 "extensions": TestExtensions, "extensions_unittests": TestExtensions, 685 "ffmpeg_regression_tests": TestFFmpegRegressions, 686 "gcm": TestGCM, "gcm_unit_tests": TestGCM, 687 "gin": TestGin, "gin_unittests": TestGin, 688 "gfx": TestGfx, "gfx_unittests": TestGfx, 689 "google_apis": TestGoogleApis, 690 "gpu": TestGPU, "gpu_unittests": TestGPU, 691 "ipc": TestIpc, "ipc_tests": TestIpc, 692 "installer_util": TestInstallerUtil, 693 "installer_util_unittests": TestInstallerUtil, 694 "install_static_unittests": TestInstallStatic, 695 "interactive_ui": TestInteractiveUI, 696 "jingle": TestJingle, "jingle_unittests": TestJingle, 697 "keyboard": TestKeyboard, "keyboard_unittests": TestKeyboard, 698 "latency": TestLatency, "latency_unittests": TestLatency, 699 "layout": TestLayout, "layout_tests": TestLayout, 700 "media": TestMedia, "media_unittests": TestMedia, 701 "message_center": TestMessageCenter, 702 "message_center_unittests" : TestMessageCenter, 703 "midi": TestMidi, "midi_unittests": TestMidi, 704 "mojo_common": TestMojoCommon, 705 "mojo_common_unittests": TestMojoCommon, 706 "mojo_system": TestMojoSystem, 707 "mojo_system_unittests": TestMojoSystem, 708 "mojo_public_system": TestMojoPublicSystem, 709 "mojo_public_system_unittests": TestMojoPublicSystem, 710 "mojo_public_bindings": TestMojoPublicBindings, 711 "mojo_public_bindings_unittests": TestMojoPublicBindings, 712 "mojo_public_sysperf": TestMojoPublicSysPerf, 713 "net": TestNet, "net_unittests": TestNet, 714 "net_perf": TestNetPerf, "net_perftests": TestNetPerf, 715 "phonenumber": TestPhoneNumber, 716 "libphonenumber_unittests": TestPhoneNumber, 717 "ppapi": TestPPAPI, "ppapi_unittests": TestPPAPI, 718 "printing": TestPrinting, "printing_unittests": TestPrinting, 719 "remoting": TestRemoting, "remoting_unittests": TestRemoting, 720 "sandbox": TestLinuxSandbox, "sandbox_linux_unittests": TestLinuxSandbox, 721 "skia": TestSkia, "skia_unittests": TestSkia, 722 "sql": TestSql, "sql_unittests": TestSql, 723 "storage": TestStorage, "storage_unittests": TestStorage, 724 "sync_integration_tests": TestSyncIntegration, 725 "sync_integration": TestSyncIntegration, 726 "ui_base_unit": TestUIBaseUnit, "ui_base_unittests": TestUIBaseUnit, 727 "ui_chromeos": TestUIChromeOS, "ui_chromeos_unittests": TestUIChromeOS, 728 "unit": TestUnit, "unit_tests": TestUnit, 729 "url": TestURL, "url_unittests": TestURL, 730 "views": TestViews, "views_unittests": TestViews, 731 "webkit": TestLayout, 732 } 733 734 735def _main(): 736 parser = optparse.OptionParser("usage: %prog -b <dir> -t <test> " 737 "[-t <test> ...]") 738 739 parser.add_option("--help-tests", dest="help_tests", action="store_true", 740 default=False, help="List all available tests") 741 parser.add_option("-b", "--build-dir", 742 help="the location of the compiler output") 743 parser.add_option("--target", help="Debug or Release") 744 parser.add_option("-t", "--test", action="append", default=[], 745 help="which test to run, supports test:gtest_filter format " 746 "as well.") 747 parser.add_option("--baseline", action="store_true", default=False, 748 help="generate baseline data instead of validating") 749 parser.add_option("-f", "--force", action="store_true", default=False, 750 help="run a broken test anyway") 751 parser.add_option("--gtest_filter", 752 help="additional arguments to --gtest_filter") 753 parser.add_option("--gtest_repeat", help="argument for --gtest_repeat") 754 parser.add_option("--gtest_shuffle", action="store_true", default=False, 755 help="Randomize tests' orders on every iteration.") 756 parser.add_option("--gtest_break_on_failure", action="store_true", 757 default=False, 758 help="Drop in to debugger on assertion failure. Also " 759 "useful for forcing tests to exit with a stack dump " 760 "on the first assertion failure when running with " 761 "--gtest_repeat=-1") 762 parser.add_option("-v", "--verbose", action="store_true", default=False, 763 help="verbose output - enable debug log messages") 764 parser.add_option("--tool", dest="valgrind_tool", default="memcheck", 765 help="specify a valgrind tool to run the tests under") 766 parser.add_option("--tool_flags", dest="valgrind_tool_flags", default="", 767 help="specify custom flags for the selected valgrind tool") 768 parser.add_option("--keep_logs", action="store_true", default=False, 769 help="store memory tool logs in the <tool>.logs directory " 770 "instead of /tmp.\nThis can be useful for tool " 771 "developers/maintainers.\nPlease note that the <tool>" 772 ".logs directory will be clobbered on tool startup.") 773 parser.add_option("-n", "--num_tests", type="int", 774 default=ChromeTests.LAYOUT_TESTS_DEFAULT_CHUNK_SIZE, 775 help="for layout tests: # of subtests per run. 0 for all.") 776 parser.add_option("--test-launcher-bot-mode", action="store_true", 777 help="run the tests with --test-launcher-bot-mode") 778 parser.add_option("--test-launcher-total-shards", type=int, 779 help="run the tests with --test-launcher-total-shards") 780 parser.add_option("--test-launcher-shard-index", type=int, 781 help="run the tests with --test-launcher-shard-index") 782 parser.add_option("--drmemory_ops", 783 help="extra options passed to Dr. Memory") 784 785 options, args = parser.parse_args() 786 787 # Bake target into build_dir. 788 if options.target and options.build_dir: 789 assert (options.target != 790 os.path.basename(os.path.dirname(options.build_dir))) 791 options.build_dir = os.path.join(os.path.abspath(options.build_dir), 792 options.target) 793 794 if options.verbose: 795 logging_utils.config_root(logging.DEBUG) 796 else: 797 logging_utils.config_root() 798 799 if options.help_tests: 800 ChromeTests.ShowTests() 801 return 0 802 803 if not options.test: 804 parser.error("--test not specified") 805 806 if len(options.test) != 1 and options.gtest_filter: 807 parser.error("--gtest_filter and multiple tests don't make sense together") 808 809 BROKEN_TESTS = { 810 'drmemory_light': [ 811 'addressinput', 812 'aura', 813 'base_unittests', 814 'cc', 815 'components', # x64 only? 816 'content', 817 'gfx', 818 'mojo_public_bindings', 819 ], 820 'drmemory_full': [ 821 'addressinput', 822 'aura', 823 'base_unittests', 824 'blink_heap', 825 'blink_platform', 826 'browser_tests', 827 'cast', 828 'cc', 829 'chromedriver', 830 'compositor', 831 'content', 832 'content_browsertests', 833 'device', 834 'events', 835 'extensions', 836 'gfx', 837 'google_apis', 838 'gpu', 839 'ipc_tests', 840 'jingle', 841 'keyboard', 842 'media', 843 'midi', 844 'mojo_common', 845 'mojo_public_bindings', 846 'mojo_public_sysperf', 847 'mojo_public_system', 848 'mojo_system', 849 'net', 850 'remoting', 851 'unit', 852 'url', 853 ], 854 } 855 856 for t in options.test: 857 if t in BROKEN_TESTS[options.valgrind_tool] and not options.force: 858 logging.info("Skipping broken %s test %s -- see crbug.com/633693" % 859 (options.valgrind_tool, t)) 860 return 0 861 862 tests = ChromeTests(options, args, t) 863 ret = tests.Run() 864 if ret: return ret 865 return 0 866 867 868if __name__ == "__main__": 869 sys.exit(_main()) 870