1#pylint: disable-msg=W0611 2# Copyright (c) 2012 The Chromium OS Authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6import cgi 7import collections 8import HTMLParser 9import logging 10import os 11import re 12 13from xml.parsers import expat 14 15import common 16 17from autotest_lib.client.common_lib import error 18from autotest_lib.client.common_lib import global_config 19from autotest_lib.client.common_lib.cros.graphite import autotest_stats 20from autotest_lib.server import site_utils 21from autotest_lib.server.cros.dynamic_suite import constants 22from autotest_lib.server.cros.dynamic_suite import job_status 23from autotest_lib.server.cros.dynamic_suite import reporting_utils 24from autotest_lib.server.cros.dynamic_suite import tools 25from autotest_lib.site_utils import gmail_lib 26 27# Try importing the essential bug reporting libraries. 28try: 29 from autotest_lib.site_utils import phapi_lib 30except ImportError, e: 31 fundamental_libs = False 32 logging.debug('Bug filing disabled. %s', e) 33else: 34 fundamental_libs = True 35 36EMAIL_COUNT_KEY = 'emails.test_failure.%s' 37BUG_CONFIG_SECTION = 'BUG_REPORTING' 38 39CHROMIUM_EMAIL_ADDRESS = global_config.global_config.get_config_value( 40 BUG_CONFIG_SECTION, 'chromium_email_address', default='') 41EMAIL_CREDS_FILE = global_config.global_config.get_config_value( 42 'NOTIFICATIONS', 'gmail_api_credentials_test_failure', default=None) 43 44 45class Bug(object): 46 """Holds the minimum information needed to make a dedupable bug report.""" 47 48 def __init__(self, title, summary, search_marker=None, labels=None, 49 owner='', cc=None): 50 """ 51 Initializes Bug object. 52 53 @param title: The title of the bug. 54 @param summary: The summary of the bug. 55 @param search_marker: The string used to determine if a bug is a 56 duplicate report or not. All Bugs with the same 57 search_marker are considered to be for the same 58 bug. Make this None if you do not want to dedupe. 59 @param labels: The labels that the filed bug will have. 60 @param owner: The owner/asignee of this bug. Typically left blank. 61 @param cc: Who to cc'd for this bug. 62 """ 63 self._title = title 64 self._summary = summary 65 self._search_marker = search_marker 66 self.owner = owner 67 68 self.labels = labels if labels is not None else [] 69 self.cc = cc if cc is not None else [] 70 71 72 def title(self): 73 """Combines information about this bug into a title string.""" 74 return self._title 75 76 77 def summary(self): 78 """Combines information about this bug into a summary string.""" 79 return self._summary 80 81 82 def search_marker(self): 83 """Return an Anchor that we can use to dedupe this exact bug.""" 84 return self._search_marker 85 86 87class TestBug(Bug): 88 """ 89 Wrap up all information needed to make an intelligent report about an 90 issue. Each TestBug has a search marker associated with it that can be 91 used to find similar reports. 92 """ 93 94 def __init__(self, build, chrome_version, suite, result): 95 """ 96 @param build: The build type, of the form <board>/<milestone>-<release>. 97 eg: x86-mario-release/R25-4321.0.0 98 @param chrome_version: The chrome version associated with the build. 99 eg: 28.0.1498.1 100 @param suite: The name of the suite that this test run is a part of. 101 @param result: The status of the job associated with this issue. 102 This contains the status, job id, test name, hostname 103 and reason for issue. 104 """ 105 self.build = build 106 self.chrome_version = chrome_version 107 self.suite = suite 108 self.name = tools.get_test_name(build, suite, result.test_name) 109 self.reason = result.reason 110 # The result_owner is used to find results and logs. 111 self.result_owner = result.owner 112 self.hostname = result.hostname 113 self.job_id = result.id 114 115 # Aborts, server/client job failures or a test failure without a 116 # reason field need lab attention. Lab bugs for the aborted case 117 # are disabled till crbug.com/188217 is resolved. 118 self.lab_error = job_status.is_for_infrastructure_fail(result) 119 120 # The owner is who the bug is assigned to. 121 self.owner = '' 122 self.cc = [] 123 124 if result.is_warn(): 125 self.labels = ['Test-Warning'] 126 self.status = 'Warning' 127 else: 128 self.labels = [] 129 self.status = 'Failure' 130 131 132 def title(self): 133 """Combines information about this bug into a title string.""" 134 return '[%s] %s %s on %s' % (self.suite, self.name, 135 self.status, self.build) 136 137 138 def summary(self): 139 """Combines information about this bug into a summary string.""" 140 141 links = self._get_links_for_failure() 142 template = ('This report is automatically generated to track the ' 143 'following %(status)s:\n' 144 'Test: %(test)s.\n' 145 'Suite: %(suite)s.\n' 146 'Chrome Version: %(chrome_version)s.\n' 147 'Build: %(build)s.\n\nReason:\n%(reason)s.\n' 148 'build artifacts: %(build_artifacts)s.\n' 149 'results log: %(results_log)s.\n' 150 'status log: %(status_log)s.\n' 151 'buildbot stages: %(buildbot_stages)s.\n' 152 'job link: %(job)s.\n\n' 153 'You may want to check the test retry dashboard in case ' 154 'this is a flakey test: %(retry_url)s\n') 155 156 specifics = { 157 'status': self.status, 158 'test': self.name, 159 'suite': self.suite, 160 'build': self.build, 161 'chrome_version': self.chrome_version, 162 'reason': self.reason, 163 'build_artifacts': links.artifacts, 164 'results_log': links.results, 165 'status_log': links.status_log, 166 'buildbot_stages': links.buildbot, 167 'job': links.job, 168 'retry_url': links.retry_url, 169 } 170 171 return template % specifics 172 173 174 def search_marker(self): 175 """Return an Anchor that we can use to dedupe this exact bug.""" 176 board = '' 177 try: 178 board = site_utils.ParseBuildName(self.build)[0] 179 except site_utils.ParseBuildNameException as e: 180 logging.error(str(e)) 181 182 # Substitute the board name for a placeholder. We try both build and 183 # release board name variants. 184 reason = self.reason 185 if board: 186 for b in (board, board.replace('_', '-')): 187 reason = reason.replace(b, 'BOARD_PLACEHOLDER') 188 189 return "%s(%s,%s,%s)" % ('Test%s' % self.status, self.suite, 190 self.name, reason) 191 192 193 def _get_links_for_failure(self): 194 """Returns a named tuple of links related to this failure.""" 195 links = collections.namedtuple('links', ('results,' 196 'status_log,' 197 'artifacts,' 198 'buildbot,' 199 'job,' 200 'retry_url')) 201 return links(reporting_utils.link_result_logs( 202 self.job_id, self.result_owner, self.hostname), 203 reporting_utils.link_status_log( 204 self.job_id, self.result_owner, self.hostname), 205 reporting_utils.link_build_artifacts(self.build), 206 reporting_utils.link_buildbot_stages(self.build), 207 reporting_utils.link_job(self.job_id), 208 reporting_utils.link_retry_url(self.name)) 209 210 211class MachineKillerBug(Bug): 212 """Wrap up information needed to report a test killing a machine.""" 213 214 # Label used by the bug-filer to categorize machine killers 215 _MACHINE_KILLER_LABEL = 'machine-killer' 216 # Address to which this bug will be cc'd 217 _CC_ADDRESS = global_config.global_config.get_config_value( 218 'SCHEDULER', 'notify_email_errors', default='') 219 220 221 def __init__(self, job_id, job_name, machine): 222 """Initialize MachineKillerBug. 223 224 @param job_id: The id of the job, this should be an afe job id. 225 @param job_name: the name of the job 226 @param machine: The hostname of a machine that has been put 227 in Repair Failed by the job. 228 229 """ 230 # Name of test job may contain information like build and suite. 231 # e.g. lumpy-release/R31-1234.0.0/bvt/dummy_Pass_SERVER_JOB 232 # Try to split job_name with '/' and use the last part 233 # as test name. Note this assumes test name must not contains '/'. 234 self._test_name = job_name.rsplit('/', 1)[-1] 235 self._job_id = job_id 236 self._machine = machine 237 self.owner='' 238 self.cc=[self._CC_ADDRESS] 239 self.labels=[self._MACHINE_KILLER_LABEL] 240 241 242 def title(self): 243 return ('%s suspected of putting machines in Repair Failed state.' 244 % self._test_name) 245 246 def summary(self): 247 """Combines information about this bug into a summary string.""" 248 249 template = ('This bug has been automatically filed to track the ' 250 'following issue:\n\n' 251 'Test: %(test)s.\n' 252 'Machine: %(machine)s.\n' 253 'Issue: It is suspected that the test has put the ' 254 'machine in the Repair Failed State.\n' 255 'Suggested Actions: Investigate to determine if this ' 256 'test is at fault and then either fix or disable the ' 257 'test if appropriate.\n' 258 'Job link: %(job)s.\n') 259 disclaimer = ('\n\nNote that the autofiled count on this bug indicates ' 260 'the number of times we have attempted to repair the ' 261 'machine, not the number of times it has gone into ' 262 'the repair failed state.\n') 263 specifics = { 264 'test': self._test_name, 265 'machine': self._machine, 266 'job': reporting_utils.link_job(self._job_id), 267 } 268 return template % specifics + disclaimer 269 270 271 def search_marker(self): 272 """Returns an Anchor that we can use to dedupe this bug.""" 273 return 'MachineKiller(%s)' % self._test_name 274 275 276class PoolHealthBug(Bug): 277 """Report information about a critical pool of DUTs in the lab.""" 278 279 _POOL_HEALTH_LABELS = ['recoverduts', 'Build-HardwareLab', 'Pri-1'] 280 _CC_ADDRESS = global_config.global_config.get_config_value( 281 'BUG_REPORTING', 'pool_health_cc', 282 type=list, default=[]) 283 284 285 def __init__(self, pool, board, dead_hostnames): 286 """Initialize a PoolHealthBug. 287 288 @param pool: The name of the pool in critical condition. 289 @param board: The board in critical condition. 290 @param dead_hostnames: A list of unusable machines with the given 291 board, in the given pool. 292 """ 293 self._pool = pool 294 self._board = board 295 self._dead_hostnames = dead_hostnames 296 self.owner = '' 297 self.cc = self._CC_ADDRESS 298 self.labels = self._POOL_HEALTH_LABELS 299 300 301 def title(self): 302 return ('pool: %s, board: %s in a critical state.' % 303 (self._pool, self._board)) 304 305 306 def summary(self): 307 """Combines information about this bug into a summary string.""" 308 309 template = ('This bug has been automatically filed to track the ' 310 'following issue:\n' 311 'Pool: %(pool)s.\n' 312 'Board: %(board)s.\n' 313 'Dead hosts: %(dead_hosts)s.\n' 314 'Issue: The pool is in a critical condition and cannot ' 315 'complete build verification tests in a timely manner.\n' 316 'Suggested Actions: Recover the devices ASAP.') 317 specifics = { 318 'pool': self._pool, 319 'board': self._board, 320 'dead_hosts': ','.join(self._dead_hostnames), 321 } 322 return template % specifics 323 324 325 def search_marker(self): 326 """Returns an Anchor that we can use to dedupe this bug.""" 327 return 'PoolHealthBug(%s, %s)' % (self._pool, self._board) 328 329class SuiteSchedulerBug(Bug): 330 """Bug filed for suite scheduler.""" 331 332 _SUITE_SCHEDULER_LABELS = ['Build-HardwareLab', 'Pri-1', 'suite_scheduler'] 333 334 def __init__(self, suite, build, board, control_file_exception): 335 self._suite = suite 336 self._build = build 337 self._board = board 338 self._exception = control_file_exception 339 # TODO(fdeng): fix get_sheriffs crbug.com/483254 340 lab_deputies = site_utils.get_sheriffs(lab_only=True) 341 self.owner = lab_deputies[0] if lab_deputies else '' 342 self.labels = self._SUITE_SCHEDULER_LABELS 343 self.cc = lab_deputies[1:] if lab_deputies else [] 344 345 346 def title(self): 347 """Return Title of the bug""" 348 if isinstance(self._exception, error.ControlFileNotFound): 349 t = 'Missing control file' 350 else: 351 t = 'Problem with getting control file' 352 return '[suite scheduler] %s for suite: "%s", build: %s' % ( 353 t, self._suite, self._build) 354 355 356 def summary(self): 357 """Combines information about this bug into a summary string.""" 358 template = ('Suite scheduler could not schedule suite due to ' 359 'a control file problem:\n\n' 360 'Suite:\t%(suite)s\n' 361 'Build:\t%(build)s\n' 362 'Board:\t%(board)s (The problem may happen for other ' 363 'boards as well, only the first board is reported.)\n' 364 'Diagnose:\n%(diagnose)s\n') 365 366 if isinstance(self._exception, error.ControlFileNotFound): 367 diagnose = ( 368 '\tThe suite\'s control file does not exist in the build.\n' 369 '\tDo you expect the suite to run for the said build?\n' 370 '\t- If yes, please add/backport the control file to ' 371 'the build,\n' 372 '\t- If not, please fix the entry for this suite in ' 373 'suite_scheduler.ini so that it specifies the ' 374 'right builds to run;\n' 375 '\t and request a push to prod.') 376 else: 377 diagnose = ('\tNo suggestion. Please ask infra deputy ' 378 'to triage.\n%s\n') % str(self._exception) 379 specifics = {'suite': self._suite, 380 'build': self._build, 381 'board': self._board, 382 'error': type(self._exception), 383 'diagnose': diagnose,} 384 return template % specifics 385 386 387 def search_marker(self): 388 """Returns an Anchor that we can use to dedupe this bug.""" 389 # TODO(fdeng): flaky deduping behavior, see crbug.com/486895 390 return 'SuiteSchedulerBug(%s, %s)' % ( 391 self._suite, type(self._exception)) 392 393 394class Reporter(object): 395 """ 396 Files external reports about bugs that happened inside autotest. 397 """ 398 # Credentials for access to the project hosting api 399 _project_name = global_config.global_config.get_config_value( 400 BUG_CONFIG_SECTION, 'project_name', default='') 401 _oauth_credentials = global_config.global_config.get_config_value( 402 BUG_CONFIG_SECTION, 'credentials', default='') 403 404 # AUTOFILED_COUNT is a label prefix used to indicate how 405 # many times we think we've updated an issue automatically. 406 AUTOFILED_COUNT = 'autofiled-count-' 407 _PREDEFINED_LABELS = ['autofiled', '%s%d' % (AUTOFILED_COUNT, 1), 408 'OS-Chrome', 'Type-Bug', 409 'Restrict-View-Google'] 410 411 _SEARCH_MARKER = 'ANCHOR ' 412 413 414 @classmethod 415 def get_creds_abspath(cls): 416 """Returns the abspath of the bug filer credentials file. 417 418 @return: A path to the oauth2 credentials file. 419 """ 420 return site_utils.get_creds_abspath(cls._oauth_credentials) 421 422 423 def __init__(self): 424 if not fundamental_libs: 425 logging.warning("Bug filing disabled due to missing imports.") 426 return 427 try: 428 self._phapi_client = phapi_lib.ProjectHostingApiClient( 429 self.get_creds_abspath(), self._project_name) 430 except phapi_lib.ProjectHostingApiException as e: 431 logging.error('Unable to create project hosting api client: %s', e) 432 self._phapi_client = None 433 434 435 def _check_tracker(self): 436 """Returns True if we have a tracker object to use for filing bugs.""" 437 return fundamental_libs and self._phapi_client 438 439 440 def get_bug_tracker_client(self): 441 """Returns the client used to communicate with the project hosting api. 442 443 @return: The instance of the ProjectHostingApiClient associated with 444 this reporter. 445 """ 446 if self._check_tracker(): 447 return self._phapi_client 448 raise phapi_lib.ProjectHostingApiException('Project hosting client not ' 449 'initialized for project:%s, using auth file: %s' % 450 (self._project_name, self.get_creds_abspath())) 451 452 453 def _get_lab_error_template(self): 454 """Return the lab error template. 455 456 @return: A dictionary representing the bug options for an issue that 457 requires investigation from the lab team. 458 """ 459 lab_sheriff = site_utils.get_sheriffs(lab_only=True) 460 return {'labels': ['Build-HardwareLab'], 461 'owner': lab_sheriff[0] if lab_sheriff else '',} 462 463 464 def _format_issue_options(self, override, **kwargs): 465 """ 466 Override the default issue configuration with a suite specific 467 configuration when one is specified in the suite's bug_template. 468 The bug_template is specified in the suite control file. After 469 overriding the correct options, format them in a way that's understood 470 by the project hosting api. 471 472 @param override: Suite specific dictionary with issue config operations. 473 @param kwargs: Keyword args containing the default issue config options. 474 @return: A dictionary which contains the suite specific options, and the 475 default option when a suite specific option isn't specified. 476 """ 477 if override: 478 kwargs.update((k,v) for k,v in override.iteritems() if v) 479 480 kwargs['labels'] = list(set(kwargs['labels'] + self._PREDEFINED_LABELS)) 481 kwargs['cc'] = list(map(lambda cc: {'name': cc}, 482 set(kwargs['cc'] + kwargs['sheriffs']))) 483 484 # The existence of an owner key will cause the api to try and match 485 # the value under the key to a member of the project, resulting in a 486 # 404 or 500 Http response when the owner is invalid. 487 if (CHROMIUM_EMAIL_ADDRESS not in kwargs['owner']): 488 del(kwargs['owner']) 489 else: 490 kwargs['owner'] = {'name': kwargs['owner']} 491 return kwargs 492 493 494 def _anchor_summary(self, bug): 495 """ 496 Creates the summary that can be used for bug deduplication. 497 498 Only attaches the anchor if the search_marker on the bug is not None. 499 500 @param: The bug to create the anchored summary for. 501 502 @return the summary with the anchor appened if the search marker is not 503 None, otherwise return the summary. 504 """ 505 if bug.search_marker() is None: 506 return bug.summary() 507 else: 508 return '%s\n\n%s%s\n' % (bug.summary(), self._SEARCH_MARKER, 509 bug.search_marker()) 510 511 512 def _create_bug_report(self, bug, bug_template={}, sheriffs=[]): 513 """ 514 Creates a new bug report. 515 516 @param bug: The Bug instance to create the report for. 517 @param bug_template: A template of options to use for filing bugs. 518 @param sheriffs: A list of chromium email addresses (of sheriffs) 519 to cc on this bug. Since the list of sheriffs is 520 dynamic it needs to be determined at runtime, as 521 opposed to the normal cc list which is available 522 through the bug template. 523 @return: id of the created issue, or None if an issue wasn't created. 524 Note that if either the description or title fields are missing 525 we won't be able to create a bug. 526 """ 527 anchored_summary = self._anchor_summary(bug) 528 529 issue = self._format_issue_options(bug_template, title=bug.title(), 530 description=anchored_summary, labels=bug.labels, 531 status='Untriaged', owner=bug.owner, cc=bug.cc, 532 sheriffs=sheriffs) 533 534 try: 535 filed_bug = self._phapi_client.create_issue(issue) 536 except phapi_lib.ProjectHostingApiException as e: 537 logging.error('Unable to create a bug for issue with title: %s and ' 538 'description %s and owner: %s. To file a new bug you ' 539 'need both a description and a title, and to assign ' 540 'it to an owner, that person must be known to the ' 541 'bug tracker', bug.title(), anchored_summary, 542 issue.get('owner')) 543 else: 544 logging.info('Filing new bug %s, with description %s', 545 filed_bug.get('id'), anchored_summary) 546 return filed_bug.get('id') 547 548 549 def modify_bug_report(self, issue_id, comment, label_update, status=''): 550 """Modifies an existing bug report with a new comment. 551 552 Adds the given comment and applies the given list of label 553 updates. 554 555 @param issue_id Id of the issue to update with. 556 @param comment Comment to update the issue with. 557 @param label_update List with label updates. 558 @param status New status of the issue. 559 """ 560 updates = { 561 'content': comment, 562 'updates': { 'labels': label_update, 'status': status } 563 } 564 try: 565 self._phapi_client.update_issue(issue_id, updates) 566 except phapi_lib.ProjectHostingApiException as e: 567 logging.warning('Unable to update issue %s, comment %s, ' 568 'labels %r, status %s: %s', issue_id, comment, 569 label_update, status, e) 570 else: 571 logging.info('Updated issue %s, comment %s, labels %r, status %s.', 572 issue_id, comment, label_update, status) 573 574 575 def find_issue_by_marker(self, marker): 576 """ 577 Queries the tracker to find if there is a bug filed for this issue. 578 579 1. 'Escape' the string: cgi.escape is the easiest way to achieve this, 580 though it doesn't handle all html escape characters. 581 eg: replace '"<' with '"<' 582 2. Perform an exact search for the escaped string, if this returns an 583 empty issue list perform a more relaxed query and finally fall back 584 to a query devoid of the reason field. Between these 3 queries we 585 should retrieve the super set of all issues that this marker can be 586 in. In most cases the first search should return a result, examples 587 where this might not be the case are when the reason field contains 588 information that varies between test runs. Since the second search 589 has raw escape characters it will match comments too, and the last 590 should match all similar issues regardless. 591 3. Look through the issues for an exact match between clean versions 592 of the marker and summary; for now 'clean' means bereft of numbers. 593 4. If no match is found look through a list of comments for each issue. 594 595 @param marker The marker string to search for to find a duplicate of 596 this issue. 597 @return A phapi_lib.Issue instance of the issue that was found, or 598 None if no issue was found. Also returns None if the marker 599 is None. 600 """ 601 602 if marker is None: 603 logging.info('No search marker specified, will create new issue.') 604 return None 605 606 # Note that this method cannot handle markers which have already been 607 # html escaped, as it will try and unescape them by converting the & 608 # to & again, thereby failing deduplication. 609 marker = HTMLParser.HTMLParser().unescape(marker) 610 html_escaped_marker = cgi.escape(marker, quote=True) 611 612 # The tracker frontend stores summaries and comments as html elements, 613 # specifically, a summary turns into a span and a comment into 614 # preformatted text. Eg: 615 # 1. A summary of >& would become <span>>&</span> 616 # 2. A comment of >& would become <pre>>&</pre> 617 # When searching for exact matches in text, the gdata api gets this 618 # feed and parses all <pre> tags unescaping html, then matching your 619 # exact string to that. However it does not unescape all <span> tags, 620 # presumably for reasons of performance. Therefore a search for the 621 # exact string ">&" would match issue 2, but not issue 1, and a search 622 # for ">&" would match issue 1 but not issue 2. This problem is 623 # further exacerbated when we have quotes within our search string, 624 # which is common when the reason field contains a python dictionary. 625 # 626 # Our searching strategy prioritizes exact matches in the summary, since 627 # the first bug thats filed will have a summary with the anchor. If we 628 # do not find an exact match in any summary we search through all 629 # related issues of the same bug/suite in the hope of finding an exact 630 # match in the comments. Note that the comments are returned as 631 # unescaped text. 632 # 633 # TODO(beeps): when we start merging issues this could return bloated 634 # results, but for now we have to include duplicate issues so that 635 # we can find the original one with the hook. 636 markers = ['"' + self._SEARCH_MARKER + html_escaped_marker + '"', 637 self._SEARCH_MARKER + marker, 638 self._SEARCH_MARKER + ','.join(marker.split(',')[:2])] 639 for decorated_marker in markers: 640 issues = self._phapi_client.get_tracker_issues_by_text( 641 decorated_marker, include_dupes=True) 642 if issues: 643 break 644 645 if not issues: 646 return 647 648 # Breadth first, since open issues/bugs probably < comments/issue. 649 # If we find more than one issue matching a particular anchor assign 650 # a mystery bug with all relevent information on the owner and return 651 # the first matching issue. 652 clean_marker = re.sub('[0-9]+', '', html_escaped_marker) 653 all_issues = [issue for issue in issues 654 if clean_marker in re.sub('[0-9]+', '', issue.summary)] 655 656 if len(all_issues) > 1: 657 issue_ids = [issue.id for issue in all_issues] 658 logging.warning('Multiple results for a specific query. Query: %s, ' 659 'results: %s', marker, issue_ids) 660 661 if all_issues: 662 return all_issues[0] 663 664 unescaped_clean_marker = re.sub('[0-9]+', '', marker) 665 for issue in issues: 666 if any(unescaped_clean_marker in re.sub('[0-9]+', '', comment) 667 for comment in issue.comments): 668 return issue 669 670 671 def _dedupe_issue(self, marker): 672 """Finds an issue, then checks if it has a parent that's still open. 673 674 @param marker: The marker string to search for to find a duplicate of 675 a issue. 676 @return An Issue instance, representing an open issue that is a 677 duplicate of the one being searched for. 678 """ 679 issue = self.find_issue_by_marker(marker) 680 if not issue or issue.state == constants.ISSUE_OPEN: 681 return issue 682 683 # Iterativly look through the chain of parents, until we find one whose 684 # state is 'open' or reach the end of the chain. 685 # It is possible that the chain forms a circle. Record the visited 686 # issues to prevent loop on a circle. 687 visited_issues = set([issue.id]) 688 while issue.merged_into is not None: 689 issue = self._phapi_client.get_tracker_issue_by_id( 690 issue.merged_into) 691 if not issue or issue.id in visited_issues: 692 break 693 elif issue.state == constants.ISSUE_OPEN: 694 logging.debug('Return the active issue %d that duplicated ' 695 'issue(s) have been merged into.', issue.id) 696 return issue 697 else: 698 visited_issues.add(issue.id) 699 logging.debug('All merged issues %s have been closed, marked ' 700 'invalid etc, will create a new issue instead.', 701 list(visited_issues)) 702 return None 703 704 705 def _get_count_labels_and_max(self, issue): 706 """Read the current autofiled count labels and count. 707 708 Automatically filed issues have a label of the form 709 `autofiled-count-<number>` that indicates about how many 710 times the autofiling code has updated the issue. This 711 routine goes through the labels for the given issue to find 712 the existing count label(s). 713 714 Old bugs may not have a count; this routine implicitly 715 assigns those bugs an initial count of one. 716 717 Usually, only one count label should exist. But 718 this method is written to take care of the case 719 where multiple count labels exist. In such case, 720 All the labels and the max count is returned. 721 722 @param issue: Issue whose 'autofiled-count' is to be read. 723 724 @returns: 2-tuple with a list of labels and 725 the max count. 726 """ 727 count_labels = [] 728 count_max = 1 729 is_count_label = lambda l: l.startswith(self.AUTOFILED_COUNT) 730 for label in filter(is_count_label, issue.labels): 731 try: 732 count = int(label[len(self.AUTOFILED_COUNT):]) 733 except ValueError: 734 continue 735 count_max = max(count, count_max) 736 count_labels.append(label) 737 return count_labels, count_max 738 739 740 def _create_autofiled_count_update(self, issue): 741 """Calculate an 'autofiled-count' label update. 742 743 Remove all the existing autofiled count labels 744 and calculate a new count label. 745 746 Updates to issues aren't guaranteed to be atomic, so in 747 some cases count labels may (in theory at least) be dropped 748 or duplicated. 749 750 The return values are a list of label updates and the 751 count value of the new count label. For the label updates, 752 all existing count labels will be prefixed with '-' to 753 remove them, and a new label with a new count will be added 754 to the set. Labels not related to the count aren't updated. 755 756 @param issue Issue whose 'autofiled-count' is to be updated. 757 @return 2-tuple with a list of label updates and the 758 new count value. 759 """ 760 count_labels, count_max = self._get_count_labels_and_max(issue) 761 label_updates = [] 762 for label in count_labels: 763 label_updates.append('-%s' % label) 764 new_count = count_max + 1 765 label_updates.append('%s%d' % (self.AUTOFILED_COUNT, new_count)) 766 return label_updates, new_count 767 768 769 @classmethod 770 def _get_project_label_from_title(cls, title): 771 """Extract a project label for the device being tested from 772 provided bug title. If no project is found, return empty string. 773 774 E.g. For the following bug title: 775 776 [stress] platform_BootDevice Failure on rikku-release/R44-7075.0.0 777 778 we extract 'rikku' and return a string 'Proj-rikku'. 779 780 Note1: For certain boards, they contain the reference name as well: 781 782 veyron_minnie-release/R44-7075.0.0 783 784 in these cases, we only extract and use the subboard (minnie) and not 785 the whole string (veyron_minnie). 786 787 Note2: some builds have different names like tot-release, 788 freon-build, etc. This function needs to handle these cases as well. 789 790 @param title: A string of the bug title, from which to extract 791 the project label for the device being tested. 792 @return '' if no valid label is found, or a label of the 793 form 'proj-samus' if found. 794 """ 795 m = re.search('.* on (?:.*_)?(?P<proj>[^-]*)-[\S]+/.*', title) 796 if m and m.group('proj'): 797 return 'Proj-%s' % m.group('proj') 798 else: 799 return '' 800 801 802 def report(self, bug, bug_template={}, ignore_duplicate=False): 803 """Report an issue to the bug tracker. 804 805 If this issue has happened before, post a comment on the 806 existing bug about it occurring again, and update the 807 'autofiled-count' label. If this is a new issue, create a 808 new bug for it. 809 810 @param bug A Bug instance about the issue. 811 @param bug_template A template dictionary specifying the 812 default bug filing options for an issue 813 with this suite. 814 @param ignore_duplicate: If True, when a duplicate is found, 815 simply ignore the new one rather than 816 posting an update. 817 @return A 2-tuple of the issue id of the issue 818 that was either created or modified, and 819 a count of the number of times the bug 820 has been updated. For a new bug, the 821 count is 1. If we could not file a bug 822 for some reason, the count is 0. 823 """ 824 if not self._check_tracker(): 825 logging.error("Can't file %s", bug.title()) 826 return None, 0 827 828 project_label = self._get_project_label_from_title(bug.title()) 829 830 issue = None 831 try: 832 issue = self._dedupe_issue(bug.search_marker()) 833 except expat.ExpatError as e: 834 # If our search string sends python's xml module into a 835 # state which it believes will lead to an xml syntax 836 # error, it will give up and throw an exception. This 837 # might happen with aborted jobs that contain weird 838 # escape characters in their reason fields. We'd rather 839 # create a new issue than fail in deduplicating such cases. 840 logging.warning('Unable to deduplicate, creating new issue: %s', 841 str(e)) 842 843 if issue and ignore_duplicate: 844 logging.debug('Duplicate found for %s, not filing as requested.', 845 bug.search_marker()) 846 _, bug_count = self._get_count_labels_and_max(issue) 847 return issue.id, bug_count 848 849 if issue: 850 comment = '%s\n\n%s' % (bug.title(), self._anchor_summary(bug)) 851 label_update, bug_count = ( 852 self._create_autofiled_count_update(issue)) 853 if project_label: 854 label_update.append(project_label) 855 self.modify_bug_report(issue.id, comment, label_update) 856 return issue.id, bug_count 857 858 sheriffs = [] 859 860 # TODO(beeps): crbug.com/254256 861 try: 862 if bug.lab_error and bug.suite == 'bvt': 863 lab_error_template = self._get_lab_error_template() 864 if bug_template.get('labels'): 865 lab_error_template['labels'] += bug_template.get('labels') 866 bug_template = lab_error_template 867 elif bug.suite == 'bvt': 868 sheriffs = site_utils.get_sheriffs() 869 except AttributeError: 870 pass 871 872 if project_label: 873 bug_template.get('labels', []).append(project_label) 874 bug_id = self._create_bug_report(bug, bug_template, sheriffs) 875 bug_count = 1 if bug_id else 0 876 return bug_id, bug_count 877 878 879# TODO(beeps): Move this to server/site_utils after crbug.com/281906 is fixed. 880def submit_generic_bug_report(*args, **kwargs): 881 """ 882 Submit a generic bug report. 883 884 See server.cros.dynamic_suite.reporting.Bug for valid arguments. 885 886 @params args: List of arguments to pass to the Bug creation. 887 @params kwargs: Keyword arguments to pass to Bug creation. 888 889 @returns the filed bug's id. 890 """ 891 bug = Bug(*args, **kwargs) 892 reporter = Reporter() 893 return reporter.report(bug)[0] 894 895 896def send_email(bug, bug_template): 897 """Send email to the owner and cc's to notify the TestBug. 898 899 @param bug: TestBug instance. 900 @param bug_template: A template dictionary specifying the default bug 901 filing options for failures in this suite. 902 """ 903 autotest_stats.Counter(EMAIL_COUNT_KEY % 'total').increment() 904 to_set = set(bug.cc) if bug.cc else set() 905 if bug.owner: 906 to_set.add(bug.owner) 907 if bug_template.get('cc'): 908 to_set = to_set.union(bug_template.get('cc')) 909 if bug_template.get('owner'): 910 to_set.add(bug_template.get('owner')) 911 recipients = ', '.join(to_set) 912 try: 913 gmail_lib.send_email( 914 recipients, bug.title(), bug.summary(), retry=False, 915 creds_path=site_utils.get_creds_abspath(EMAIL_CREDS_FILE)) 916 except Exception: 917 autotest_stats.Counter(EMAIL_COUNT_KEY % 'fail').increment() 918 raise 919