1# Copyright (c) 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5
6"""Top-level presubmit script for Skia.
7
8See http://dev.chromium.org/developers/how-tos/depottools/presubmit-scripts
9for more details about the presubmit API built into gcl.
10"""
11
12import collections
13import csv
14import fnmatch
15import os
16import re
17import subprocess
18import sys
19import traceback
20
21
22REVERT_CL_SUBJECT_PREFIX = 'Revert '
23
24SKIA_TREE_STATUS_URL = 'http://skia-tree-status.appspot.com'
25
26# Please add the complete email address here (and not just 'xyz@' or 'xyz').
27PUBLIC_API_OWNERS = (
28    'reed@chromium.org',
29    'reed@google.com',
30    'bsalomon@chromium.org',
31    'bsalomon@google.com',
32    'djsollen@chromium.org',
33    'djsollen@google.com',
34    'hcm@chromium.org',
35    'hcm@google.com',
36)
37
38AUTHORS_FILE_NAME = 'AUTHORS'
39
40DOCS_PREVIEW_URL = 'https://skia.org/?cl='
41GOLD_TRYBOT_URL = 'https://gold.skia.org/search?issue='
42
43SERVICE_ACCOUNT_SUFFIX = [
44    '@%s.iam.gserviceaccount.com' % project for project in [
45        'skia-buildbots.google.com', 'skia-swarming-bots', 'skia-public',
46        'skia-corp.google.com']]
47
48
49def _CheckChangeHasEol(input_api, output_api, source_file_filter=None):
50  """Checks that files end with atleast one \n (LF)."""
51  eof_files = []
52  for f in input_api.AffectedSourceFiles(source_file_filter):
53    contents = input_api.ReadFile(f, 'rb')
54    # Check that the file ends in atleast one newline character.
55    if len(contents) > 1 and contents[-1:] != '\n':
56      eof_files.append(f.LocalPath())
57
58  if eof_files:
59    return [output_api.PresubmitPromptWarning(
60      'These files should end in a newline character:',
61      items=eof_files)]
62  return []
63
64
65def _JsonChecks(input_api, output_api):
66  """Run checks on any modified json files."""
67  failing_files = []
68  for affected_file in input_api.AffectedFiles(None):
69    affected_file_path = affected_file.LocalPath()
70    is_json = affected_file_path.endswith('.json')
71    is_metadata = (affected_file_path.startswith('site/') and
72                   affected_file_path.endswith('/METADATA'))
73    if is_json or is_metadata:
74      try:
75        input_api.json.load(open(affected_file_path, 'r'))
76      except ValueError:
77        failing_files.append(affected_file_path)
78
79  results = []
80  if failing_files:
81    results.append(
82        output_api.PresubmitError(
83            'The following files contain invalid json:\n%s\n\n' %
84                '\n'.join(failing_files)))
85  return results
86
87
88def _IfDefChecks(input_api, output_api):
89  """Ensures if/ifdef are not before includes. See skbug/3362 for details."""
90  comment_block_start_pattern = re.compile('^\s*\/\*.*$')
91  comment_block_middle_pattern = re.compile('^\s+\*.*')
92  comment_block_end_pattern = re.compile('^\s+\*\/.*$')
93  single_line_comment_pattern = re.compile('^\s*//.*$')
94  def is_comment(line):
95    return (comment_block_start_pattern.match(line) or
96            comment_block_middle_pattern.match(line) or
97            comment_block_end_pattern.match(line) or
98            single_line_comment_pattern.match(line))
99
100  empty_line_pattern = re.compile('^\s*$')
101  def is_empty_line(line):
102    return empty_line_pattern.match(line)
103
104  failing_files = []
105  for affected_file in input_api.AffectedSourceFiles(None):
106    affected_file_path = affected_file.LocalPath()
107    if affected_file_path.endswith('.cpp') or affected_file_path.endswith('.h'):
108      f = open(affected_file_path)
109      for line in f.xreadlines():
110        if is_comment(line) or is_empty_line(line):
111          continue
112        # The below will be the first real line after comments and newlines.
113        if line.startswith('#if 0 '):
114          pass
115        elif line.startswith('#if ') or line.startswith('#ifdef '):
116          failing_files.append(affected_file_path)
117        break
118
119  results = []
120  if failing_files:
121    results.append(
122        output_api.PresubmitError(
123            'The following files have #if or #ifdef before includes:\n%s\n\n'
124            'See https://bug.skia.org/3362 for why this should be fixed.' %
125                '\n'.join(failing_files)))
126  return results
127
128
129def _CopyrightChecks(input_api, output_api, source_file_filter=None):
130  results = []
131  year_pattern = r'\d{4}'
132  year_range_pattern = r'%s(-%s)?' % (year_pattern, year_pattern)
133  years_pattern = r'%s(,%s)*,?' % (year_range_pattern, year_range_pattern)
134  copyright_pattern = (
135      r'Copyright (\([cC]\) )?%s \w+' % years_pattern)
136
137  for affected_file in input_api.AffectedSourceFiles(source_file_filter):
138    if 'third_party' in affected_file.LocalPath():
139      continue
140    contents = input_api.ReadFile(affected_file, 'rb')
141    if not re.search(copyright_pattern, contents):
142      results.append(output_api.PresubmitError(
143          '%s is missing a correct copyright header.' % affected_file))
144  return results
145
146
147def _ToolFlags(input_api, output_api):
148  """Make sure `{dm,nanobench}_flags.py test` passes if modified."""
149  results = []
150  sources = lambda x: ('dm_flags.py'        in x.LocalPath() or
151                       'nanobench_flags.py' in x.LocalPath())
152  for f in input_api.AffectedSourceFiles(sources):
153    if 0 != subprocess.call(['python', f.LocalPath(), 'test']):
154      results.append(output_api.PresubmitError('`python %s test` failed' % f))
155  return results
156
157
158def _InfraTests(input_api, output_api):
159  """Run the infra tests."""
160  results = []
161  if not any(f.LocalPath().startswith('infra')
162             for f in input_api.AffectedFiles()):
163    return results
164
165  cmd = ['python', os.path.join('infra', 'bots', 'infra_tests.py')]
166  try:
167    subprocess.check_output(cmd)
168  except subprocess.CalledProcessError as e:
169    results.append(output_api.PresubmitError(
170        '`%s` failed:\n%s' % (' '.join(cmd), e.output)))
171  return results
172
173
174def _CheckGNFormatted(input_api, output_api):
175  """Make sure any .gn files we're changing have been formatted."""
176  results = []
177  for f in input_api.AffectedFiles():
178    if (not f.LocalPath().endswith('.gn') and
179        not f.LocalPath().endswith('.gni')):
180      continue
181
182    gn = 'gn.bat' if 'win32' in sys.platform else 'gn'
183    cmd = [gn, 'format', '--dry-run', f.LocalPath()]
184    try:
185      subprocess.check_output(cmd)
186    except subprocess.CalledProcessError:
187      fix = 'gn format ' + f.LocalPath()
188      results.append(output_api.PresubmitError(
189          '`%s` failed, try\n\t%s' % (' '.join(cmd), fix)))
190  return results
191
192
193class _WarningsAsErrors():
194  def __init__(self, output_api):
195    self.output_api = output_api
196    self.old_warning = None
197  def __enter__(self):
198    self.old_warning = self.output_api.PresubmitPromptWarning
199    self.output_api.PresubmitPromptWarning = self.output_api.PresubmitError
200    return self.output_api
201  def __exit__(self, ex_type, ex_value, ex_traceback):
202    self.output_api.PresubmitPromptWarning = self.old_warning
203
204
205def _CommonChecks(input_api, output_api):
206  """Presubmit checks common to upload and commit."""
207  results = []
208  sources = lambda x: (x.LocalPath().endswith('.h') or
209                       x.LocalPath().endswith('.py') or
210                       x.LocalPath().endswith('.sh') or
211                       x.LocalPath().endswith('.m') or
212                       x.LocalPath().endswith('.mm') or
213                       x.LocalPath().endswith('.go') or
214                       x.LocalPath().endswith('.c') or
215                       x.LocalPath().endswith('.cc') or
216                       x.LocalPath().endswith('.cpp'))
217  results.extend(_CheckChangeHasEol(
218      input_api, output_api, source_file_filter=sources))
219  with _WarningsAsErrors(output_api):
220    results.extend(input_api.canned_checks.CheckChangeHasNoCR(
221        input_api, output_api, source_file_filter=sources))
222    results.extend(input_api.canned_checks.CheckChangeHasNoStrayWhitespace(
223        input_api, output_api, source_file_filter=sources))
224  results.extend(_JsonChecks(input_api, output_api))
225  results.extend(_IfDefChecks(input_api, output_api))
226  results.extend(_CopyrightChecks(input_api, output_api,
227                                  source_file_filter=sources))
228  results.extend(_ToolFlags(input_api, output_api))
229  return results
230
231
232def CheckChangeOnUpload(input_api, output_api):
233  """Presubmit checks for the change on upload.
234
235  The following are the presubmit checks:
236  * Check change has one and only one EOL.
237  """
238  results = []
239  results.extend(_CommonChecks(input_api, output_api))
240  # Run on upload, not commit, since the presubmit bot apparently doesn't have
241  # coverage or Go installed.
242  results.extend(_InfraTests(input_api, output_api))
243
244  results.extend(_CheckGNFormatted(input_api, output_api))
245  return results
246
247
248def _CheckTreeStatus(input_api, output_api, json_url):
249  """Check whether to allow commit.
250
251  Args:
252    input_api: input related apis.
253    output_api: output related apis.
254    json_url: url to download json style status.
255  """
256  tree_status_results = input_api.canned_checks.CheckTreeIsOpen(
257      input_api, output_api, json_url=json_url)
258  if not tree_status_results:
259    # Check for caution state only if tree is not closed.
260    connection = input_api.urllib2.urlopen(json_url)
261    status = input_api.json.loads(connection.read())
262    connection.close()
263    if ('caution' in status['message'].lower() and
264        os.isatty(sys.stdout.fileno())):
265      # Display a prompt only if we are in an interactive shell. Without this
266      # check the commit queue behaves incorrectly because it considers
267      # prompts to be failures.
268      short_text = 'Tree state is: ' + status['general_state']
269      long_text = status['message'] + '\n' + json_url
270      tree_status_results.append(
271          output_api.PresubmitPromptWarning(
272              message=short_text, long_text=long_text))
273  else:
274    # Tree status is closed. Put in message about contacting sheriff.
275    connection = input_api.urllib2.urlopen(
276        SKIA_TREE_STATUS_URL + '/current-sheriff')
277    sheriff_details = input_api.json.loads(connection.read())
278    if sheriff_details:
279      tree_status_results[0]._message += (
280          '\n\nPlease contact the current Skia sheriff (%s) if you are trying '
281          'to submit a build fix\nand do not know how to submit because the '
282          'tree is closed') % sheriff_details['username']
283  return tree_status_results
284
285
286class CodeReview(object):
287  """Abstracts which codereview tool is used for the specified issue."""
288
289  def __init__(self, input_api):
290    self._issue = input_api.change.issue
291    self._gerrit = input_api.gerrit
292
293  def GetOwnerEmail(self):
294    return self._gerrit.GetChangeOwner(self._issue)
295
296  def GetSubject(self):
297    return self._gerrit.GetChangeInfo(self._issue)['subject']
298
299  def GetDescription(self):
300    return self._gerrit.GetChangeDescription(self._issue)
301
302  def IsDryRun(self):
303    return self._gerrit.GetChangeInfo(
304        self._issue)['labels']['Commit-Queue'].get('value', 0) == 1
305
306  def GetReviewers(self):
307    code_review_label = (
308        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
309    return [r['email'] for r in code_review_label.get('all', [])]
310
311  def GetApprovers(self):
312    approvers = []
313    code_review_label = (
314        self._gerrit.GetChangeInfo(self._issue)['labels']['Code-Review'])
315    for m in code_review_label.get('all', []):
316      if m.get("value") == 1:
317        approvers.append(m["email"])
318    return approvers
319
320
321def _CheckOwnerIsInAuthorsFile(input_api, output_api):
322  results = []
323  if input_api.change.issue:
324    cr = CodeReview(input_api)
325
326    owner_email = cr.GetOwnerEmail()
327
328    # Service accounts don't need to be in AUTHORS.
329    for suffix in SERVICE_ACCOUNT_SUFFIX:
330      if owner_email.endswith(suffix):
331        return results
332
333    try:
334      authors_content = ''
335      for line in open(AUTHORS_FILE_NAME):
336        if not line.startswith('#'):
337          authors_content += line
338      email_fnmatches = re.findall('<(.*)>', authors_content)
339      for email_fnmatch in email_fnmatches:
340        if fnmatch.fnmatch(owner_email, email_fnmatch):
341          # Found a match, the user is in the AUTHORS file break out of the loop
342          break
343      else:
344        results.append(
345          output_api.PresubmitError(
346            'The email %s is not in Skia\'s AUTHORS file.\n'
347            'Issue owner, this CL must include an addition to the Skia AUTHORS '
348            'file.'
349            % owner_email))
350    except IOError:
351      # Do not fail if authors file cannot be found.
352      traceback.print_exc()
353      input_api.logging.error('AUTHORS file not found!')
354
355  return results
356
357
358def _CheckLGTMsForPublicAPI(input_api, output_api):
359  """Check LGTMs for public API changes.
360
361  For public API files make sure there is an LGTM from the list of owners in
362  PUBLIC_API_OWNERS.
363  """
364  results = []
365  requires_owner_check = False
366  for affected_file in input_api.AffectedFiles():
367    affected_file_path = affected_file.LocalPath()
368    file_path, file_ext = os.path.splitext(affected_file_path)
369    # We only care about files that end in .h and are under the top-level
370    # include dir, but not include/private.
371    if (file_ext == '.h' and
372        'include' == file_path.split(os.path.sep)[0] and
373        'private' not in file_path):
374      requires_owner_check = True
375
376  if not requires_owner_check:
377    return results
378
379  lgtm_from_owner = False
380  if input_api.change.issue:
381    cr = CodeReview(input_api)
382
383    if re.match(REVERT_CL_SUBJECT_PREFIX, cr.GetSubject(), re.I):
384      # It is a revert CL, ignore the public api owners check.
385      return results
386
387    if cr.IsDryRun():
388      # Ignore public api owners check for dry run CLs since they are not
389      # going to be committed.
390      return results
391
392    if input_api.gerrit:
393      for reviewer in cr.GetReviewers():
394        if reviewer in PUBLIC_API_OWNERS:
395          # If an owner is specified as an reviewer in Gerrit then ignore the
396          # public api owners check.
397          return results
398    else:
399      match = re.search(r'^TBR=(.*)$', cr.GetDescription(), re.M)
400      if match:
401        tbr_section = match.group(1).strip().split(' ')[0]
402        tbr_entries = tbr_section.split(',')
403        for owner in PUBLIC_API_OWNERS:
404          if owner in tbr_entries or owner.split('@')[0] in tbr_entries:
405            # If an owner is specified in the TBR= line then ignore the public
406            # api owners check.
407            return results
408
409    if cr.GetOwnerEmail() in PUBLIC_API_OWNERS:
410      # An owner created the CL that is an automatic LGTM.
411      lgtm_from_owner = True
412
413    for approver in cr.GetApprovers():
414      if approver in PUBLIC_API_OWNERS:
415        # Found an lgtm in a message from an owner.
416        lgtm_from_owner = True
417        break
418
419  if not lgtm_from_owner:
420    results.append(
421        output_api.PresubmitError(
422            "If this CL adds to or changes Skia's public API, you need an LGTM "
423            "from any of %s.  If this CL only removes from or doesn't change "
424            "Skia's public API, please add a short note to the CL saying so. "
425            "Add one of the owners as a reviewer to your CL as well as to the "
426            "TBR= line.  If you don't know if this CL affects Skia's public "
427            "API, treat it like it does." % str(PUBLIC_API_OWNERS)))
428  return results
429
430
431def _FooterExists(footers, key, value):
432  for k, v in footers:
433    if k == key and v == value:
434      return True
435  return False
436
437
438def PostUploadHook(cl, change, output_api):
439  """git cl upload will call this hook after the issue is created/modified.
440
441  This hook does the following:
442  * Adds a link to preview docs changes if there are any docs changes in the CL.
443  * Adds 'No-Try: true' if the CL contains only docs changes.
444  * Adds 'No-Tree-Checks: true' for non master branch changes since they do not
445    need to be gated on the master branch's tree.
446  * Adds 'No-Try: true' for non master branch changes since trybots do not yet
447    work on them.
448  * Adds 'No-Presubmit: true' for non master branch changes since those don't
449    run the presubmit checks.
450  """
451
452  results = []
453  atleast_one_docs_change = False
454  all_docs_changes = True
455  for affected_file in change.AffectedFiles():
456    affected_file_path = affected_file.LocalPath()
457    file_path, _ = os.path.splitext(affected_file_path)
458    if 'site' == file_path.split(os.path.sep)[0]:
459      atleast_one_docs_change = True
460    else:
461      all_docs_changes = False
462    if atleast_one_docs_change and not all_docs_changes:
463      break
464
465  issue = cl.issue
466  if issue:
467    # Skip PostUploadHooks for all auto-commit service account bots. New
468    # patchsets (caused due to PostUploadHooks) invalidates the CQ+2 vote from
469    # the "--use-commit-queue" flag to "git cl upload".
470    for suffix in SERVICE_ACCOUNT_SUFFIX:
471      if cl.GetIssueOwner().endswith(suffix):
472        return results
473
474    original_description_lines, footers = cl.GetDescriptionFooters()
475    new_description_lines = list(original_description_lines)
476
477    # If the change includes only doc changes then add No-Try: true in the
478    # CL's description if it does not exist yet.
479    if all_docs_changes and not _FooterExists(footers, 'No-Try', 'true'):
480      new_description_lines.append('No-Try: true')
481      results.append(
482          output_api.PresubmitNotifyResult(
483              'This change has only doc changes. Automatically added '
484              '\'No-Try: true\' to the CL\'s description'))
485
486    # If there is atleast one docs change then add preview link in the CL's
487    # description if it does not already exist there.
488    docs_preview_link = '%s%s' % (DOCS_PREVIEW_URL, issue)
489    docs_preview_line = 'Docs-Preview: %s' % docs_preview_link
490    if (atleast_one_docs_change and
491        not _FooterExists(footers, 'Docs-Preview', docs_preview_link)):
492      # Automatically add a link to where the docs can be previewed.
493      new_description_lines.append(docs_preview_line)
494      results.append(
495          output_api.PresubmitNotifyResult(
496              'Automatically added a link to preview the docs changes to the '
497              'CL\'s description'))
498
499    # If the target ref is not master then add 'No-Tree-Checks: true' and
500    # 'No-Try: true' to the CL's description if it does not already exist there.
501    target_ref = cl.GetRemoteBranch()[1]
502    if target_ref != 'refs/remotes/origin/master':
503      if not _FooterExists(footers, 'No-Tree-Checks', 'true'):
504        new_description_lines.append('No-Tree-Checks: true')
505        results.append(
506            output_api.PresubmitNotifyResult(
507                'Branch changes do not need to rely on the master branch\'s '
508                'tree status. Automatically added \'No-Tree-Checks: true\' to '
509                'the CL\'s description'))
510      if not _FooterExists(footers, 'No-Try', 'true'):
511        new_description_lines.append('No-Try: true')
512        results.append(
513            output_api.PresubmitNotifyResult(
514                'Trybots do not yet work for non-master branches. '
515                'Automatically added \'No-Try: true\' to the CL\'s '
516                'description'))
517      if not _FooterExists(footers, 'No-Presubmit', 'true'):
518        new_description_lines.append('No-Presubmit: true')
519        results.append(
520            output_api.PresubmitNotifyResult(
521                'Branch changes do not run the presubmit checks.'))
522
523    # If the description has changed update it.
524    if new_description_lines != original_description_lines:
525      # Add a new line separating the new contents from the old contents.
526      new_description_lines.insert(len(original_description_lines), '')
527      cl.UpdateDescriptionFooters(new_description_lines, footers)
528
529    return results
530
531
532def CheckChangeOnCommit(input_api, output_api):
533  """Presubmit checks for the change on commit.
534
535  The following are the presubmit checks:
536  * Check change has one and only one EOL.
537  * Ensures that the Skia tree is open in
538    http://skia-tree-status.appspot.com/. Shows a warning if it is in 'Caution'
539    state and an error if it is in 'Closed' state.
540  """
541  results = []
542  results.extend(_CommonChecks(input_api, output_api))
543  results.extend(
544      _CheckTreeStatus(input_api, output_api, json_url=(
545          SKIA_TREE_STATUS_URL + '/banner-status?format=json')))
546  results.extend(_CheckLGTMsForPublicAPI(input_api, output_api))
547  results.extend(_CheckOwnerIsInAuthorsFile(input_api, output_api))
548  # Checks for the presence of 'DO NOT''SUBMIT' in CL description and in
549  # content of files.
550  results.extend(
551      input_api.canned_checks.CheckDoNotSubmit(input_api, output_api))
552  return results
553