1#!/usr/bin/env python
2#
3# Copyright 2012 the V8 project authors. All rights reserved.
4# Redistribution and use in source and binary forms, with or without
5# modification, are permitted provided that the following conditions are
6# met:
7#
8#     * Redistributions of source code must retain the above copyright
9#       notice, this list of conditions and the following disclaimer.
10#     * Redistributions in binary form must reproduce the above
11#       copyright notice, this list of conditions and the following
12#       disclaimer in the documentation and/or other materials provided
13#       with the distribution.
14#     * Neither the name of Google Inc. nor the names of its
15#       contributors may be used to endorse or promote products derived
16#       from this software without specific prior written permission.
17#
18# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
19# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
20# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
21# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
22# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
23# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
24# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
25# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
26# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
27# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
28# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
29
30try:
31  import hashlib
32  md5er = hashlib.md5
33except ImportError, e:
34  import md5
35  md5er = md5.new
36
37
38import json
39import optparse
40import os
41from os.path import abspath, join, dirname, basename, exists
42import pickle
43import re
44import sys
45import subprocess
46import multiprocessing
47from subprocess import PIPE
48
49from testrunner.local import statusfile
50from testrunner.local import testsuite
51from testrunner.local import utils
52
53# Special LINT rules diverging from default and reason.
54# build/header_guard: Our guards have the form "V8_FOO_H_", not "SRC_FOO_H_".
55# build/include_what_you_use: Started giving false positives for variables
56#   named "string" and "map" assuming that you needed to include STL headers.
57# TODO(bmeurer): Fix and re-enable readability/check
58
59LINT_RULES = """
60-build/header_guard
61+build/include_alpha
62-build/include_what_you_use
63-build/namespaces
64-readability/check
65+readability/streams
66-runtime/references
67""".split()
68
69LINT_OUTPUT_PATTERN = re.compile(r'^.+[:(]\d+[:)]|^Done processing')
70FLAGS_LINE = re.compile("//\s*Flags:.*--([A-z0-9-])+_[A-z0-9].*\n")
71
72def CppLintWorker(command):
73  try:
74    process = subprocess.Popen(command, stderr=subprocess.PIPE)
75    process.wait()
76    out_lines = ""
77    error_count = -1
78    while True:
79      out_line = process.stderr.readline()
80      if out_line == '' and process.poll() != None:
81        if error_count == -1:
82          print "Failed to process %s" % command.pop()
83          return 1
84        break
85      m = LINT_OUTPUT_PATTERN.match(out_line)
86      if m:
87        out_lines += out_line
88        error_count += 1
89    sys.stdout.write(out_lines)
90    return error_count
91  except KeyboardInterrupt:
92    process.kill()
93  except:
94    print('Error running cpplint.py. Please make sure you have depot_tools' +
95          ' in your $PATH. Lint check skipped.')
96    process.kill()
97
98
99class FileContentsCache(object):
100
101  def __init__(self, sums_file_name):
102    self.sums = {}
103    self.sums_file_name = sums_file_name
104
105  def Load(self):
106    try:
107      sums_file = None
108      try:
109        sums_file = open(self.sums_file_name, 'r')
110        self.sums = pickle.load(sums_file)
111      except:
112        # Cannot parse pickle for any reason. Not much we can do about it.
113        pass
114    finally:
115      if sums_file:
116        sums_file.close()
117
118  def Save(self):
119    try:
120      sums_file = open(self.sums_file_name, 'w')
121      pickle.dump(self.sums, sums_file)
122    except:
123      # Failed to write pickle. Try to clean-up behind us.
124      if sums_file:
125        sums_file.close()
126      try:
127        os.unlink(self.sums_file_name)
128      except:
129        pass
130    finally:
131      sums_file.close()
132
133  def FilterUnchangedFiles(self, files):
134    changed_or_new = []
135    for file in files:
136      try:
137        handle = open(file, "r")
138        file_sum = md5er(handle.read()).digest()
139        if not file in self.sums or self.sums[file] != file_sum:
140          changed_or_new.append(file)
141          self.sums[file] = file_sum
142      finally:
143        handle.close()
144    return changed_or_new
145
146  def RemoveFile(self, file):
147    if file in self.sums:
148      self.sums.pop(file)
149
150
151class SourceFileProcessor(object):
152  """
153  Utility class that can run through a directory structure, find all relevant
154  files and invoke a custom check on the files.
155  """
156
157  def Run(self, path):
158    all_files = []
159    for file in self.GetPathsToSearch():
160      all_files += self.FindFilesIn(join(path, file))
161    if not self.ProcessFiles(all_files, path):
162      return False
163    return True
164
165  def IgnoreDir(self, name):
166    return (name.startswith('.') or
167            name in ('buildtools', 'data', 'gmock', 'gtest', 'kraken',
168                     'octane', 'sunspider'))
169
170  def IgnoreFile(self, name):
171    return name.startswith('.')
172
173  def FindFilesIn(self, path):
174    result = []
175    for (root, dirs, files) in os.walk(path):
176      for ignored in [x for x in dirs if self.IgnoreDir(x)]:
177        dirs.remove(ignored)
178      for file in files:
179        if not self.IgnoreFile(file) and self.IsRelevant(file):
180          result.append(join(root, file))
181    return result
182
183
184class CppLintProcessor(SourceFileProcessor):
185  """
186  Lint files to check that they follow the google code style.
187  """
188
189  def IsRelevant(self, name):
190    return name.endswith('.cc') or name.endswith('.h')
191
192  def IgnoreDir(self, name):
193    return (super(CppLintProcessor, self).IgnoreDir(name)
194              or (name == 'third_party'))
195
196  IGNORE_LINT = ['flag-definitions.h']
197
198  def IgnoreFile(self, name):
199    return (super(CppLintProcessor, self).IgnoreFile(name)
200              or (name in CppLintProcessor.IGNORE_LINT))
201
202  def GetPathsToSearch(self):
203    return ['src', 'include', 'samples', join('test', 'cctest'),
204            join('test', 'unittests')]
205
206  def GetCpplintScript(self, prio_path):
207    for path in [prio_path] + os.environ["PATH"].split(os.pathsep):
208      path = path.strip('"')
209      cpplint = os.path.join(path, "cpplint.py")
210      if os.path.isfile(cpplint):
211        return cpplint
212
213    return None
214
215  def ProcessFiles(self, files, path):
216    good_files_cache = FileContentsCache('.cpplint-cache')
217    good_files_cache.Load()
218    files = good_files_cache.FilterUnchangedFiles(files)
219    if len(files) == 0:
220      print 'No changes in files detected. Skipping cpplint check.'
221      return True
222
223    filters = ",".join([n for n in LINT_RULES])
224    command = [sys.executable, 'cpplint.py', '--filter', filters]
225    cpplint = self.GetCpplintScript(join(path, "tools"))
226    if cpplint is None:
227      print('Could not find cpplint.py. Make sure '
228            'depot_tools is installed and in the path.')
229      sys.exit(1)
230
231    command = [sys.executable, cpplint, '--filter', filters]
232
233    commands = join([command + [file] for file in files])
234    count = multiprocessing.cpu_count()
235    pool = multiprocessing.Pool(count)
236    try:
237      results = pool.map_async(CppLintWorker, commands).get(999999)
238    except KeyboardInterrupt:
239      print "\nCaught KeyboardInterrupt, terminating workers."
240      sys.exit(1)
241
242    for i in range(len(files)):
243      if results[i] > 0:
244        good_files_cache.RemoveFile(files[i])
245
246    total_errors = sum(results)
247    print "Total errors found: %d" % total_errors
248    good_files_cache.Save()
249    return total_errors == 0
250
251
252COPYRIGHT_HEADER_PATTERN = re.compile(
253    r'Copyright [\d-]*20[0-1][0-9] the V8 project authors. All rights reserved.')
254
255class SourceProcessor(SourceFileProcessor):
256  """
257  Check that all files include a copyright notice and no trailing whitespaces.
258  """
259
260  RELEVANT_EXTENSIONS = ['.js', '.cc', '.h', '.py', '.c',
261                         '.status', '.gyp', '.gypi']
262
263  # Overwriting the one in the parent class.
264  def FindFilesIn(self, path):
265    if os.path.exists(path+'/.git'):
266      output = subprocess.Popen('git ls-files --full-name',
267                                stdout=PIPE, cwd=path, shell=True)
268      result = []
269      for file in output.stdout.read().split():
270        for dir_part in os.path.dirname(file).replace(os.sep, '/').split('/'):
271          if self.IgnoreDir(dir_part):
272            break
273        else:
274          if (self.IsRelevant(file) and os.path.exists(file)
275              and not self.IgnoreFile(file)):
276            result.append(join(path, file))
277      if output.wait() == 0:
278        return result
279    return super(SourceProcessor, self).FindFilesIn(path)
280
281  def IsRelevant(self, name):
282    for ext in SourceProcessor.RELEVANT_EXTENSIONS:
283      if name.endswith(ext):
284        return True
285    return False
286
287  def GetPathsToSearch(self):
288    return ['.']
289
290  def IgnoreDir(self, name):
291    return (super(SourceProcessor, self).IgnoreDir(name) or
292            name in ('third_party', 'gyp', 'out', 'obj', 'DerivedSources'))
293
294  IGNORE_COPYRIGHTS = ['box2d.js',
295                       'cpplint.py',
296                       'copy.js',
297                       'corrections.js',
298                       'crypto.js',
299                       'daemon.py',
300                       'earley-boyer.js',
301                       'fannkuch.js',
302                       'fasta.js',
303                       'jsmin.py',
304                       'libraries.cc',
305                       'libraries-empty.cc',
306                       'lua_binarytrees.js',
307                       'memops.js',
308                       'poppler.js',
309                       'primes.js',
310                       'raytrace.js',
311                       'regexp-pcre.js',
312                       'sqlite.js',
313                       'sqlite-change-heap.js',
314                       'sqlite-pointer-masking.js',
315                       'sqlite-safe-heap.js',
316                       'gnuplot-4.6.3-emscripten.js',
317                       'zlib.js']
318  IGNORE_TABS = IGNORE_COPYRIGHTS + ['unicode-test.js', 'html-comments.js']
319
320  def EndOfDeclaration(self, line):
321    return line == "}" or line == "};"
322
323  def StartOfDeclaration(self, line):
324    return line.find("//") == 0 or \
325           line.find("/*") == 0 or \
326           line.find(") {") != -1
327
328  def ProcessContents(self, name, contents):
329    result = True
330    base = basename(name)
331    if not base in SourceProcessor.IGNORE_TABS:
332      if '\t' in contents:
333        print "%s contains tabs" % name
334        result = False
335    if not base in SourceProcessor.IGNORE_COPYRIGHTS:
336      if not COPYRIGHT_HEADER_PATTERN.search(contents):
337        print "%s is missing a correct copyright header." % name
338        result = False
339    if ' \n' in contents or contents.endswith(' '):
340      line = 0
341      lines = []
342      parts = contents.split(' \n')
343      if not contents.endswith(' '):
344        parts.pop()
345      for part in parts:
346        line += part.count('\n') + 1
347        lines.append(str(line))
348      linenumbers = ', '.join(lines)
349      if len(lines) > 1:
350        print "%s has trailing whitespaces in lines %s." % (name, linenumbers)
351      else:
352        print "%s has trailing whitespaces in line %s." % (name, linenumbers)
353      result = False
354    if not contents.endswith('\n') or contents.endswith('\n\n'):
355      print "%s does not end with a single new line." % name
356      result = False
357    # Check two empty lines between declarations.
358    if name.endswith(".cc"):
359      line = 0
360      lines = []
361      parts = contents.split('\n')
362      while line < len(parts) - 2:
363        if self.EndOfDeclaration(parts[line]):
364          if self.StartOfDeclaration(parts[line + 1]):
365            lines.append(str(line + 1))
366            line += 1
367          elif parts[line + 1] == "" and \
368               self.StartOfDeclaration(parts[line + 2]):
369            lines.append(str(line + 1))
370            line += 2
371        line += 1
372      if len(lines) >= 1:
373        linenumbers = ', '.join(lines)
374        if len(lines) > 1:
375          print "%s does not have two empty lines between declarations " \
376                "in lines %s." % (name, linenumbers)
377        else:
378          print "%s does not have two empty lines between declarations " \
379                "in line %s." % (name, linenumbers)
380        result = False
381    # Sanitize flags for fuzzer.
382    if "mjsunit" in name:
383      match = FLAGS_LINE.search(contents)
384      if match:
385        print "%s Flags should use '-' (not '_')" % name
386        result = False
387    return result
388
389  def ProcessFiles(self, files, path):
390    success = True
391    violations = 0
392    for file in files:
393      try:
394        handle = open(file)
395        contents = handle.read()
396        if not self.ProcessContents(file, contents):
397          success = False
398          violations += 1
399      finally:
400        handle.close()
401    print "Total violating files: %s" % violations
402    return success
403
404
405def CheckExternalReferenceRegistration(workspace):
406  code = subprocess.call(
407      [sys.executable, join(workspace, "tools", "external-reference-check.py")])
408  return code == 0
409
410
411def _CheckStatusFileForDuplicateKeys(filepath):
412  comma_space_bracket = re.compile(", *]")
413  lines = []
414  with open(filepath) as f:
415    for line in f.readlines():
416      # Skip all-comment lines.
417      if line.lstrip().startswith("#"): continue
418      # Strip away comments at the end of the line.
419      comment_start = line.find("#")
420      if comment_start != -1:
421        line = line[:comment_start]
422      line = line.strip()
423      # Strip away trailing commas within the line.
424      line = comma_space_bracket.sub("]", line)
425      if len(line) > 0:
426        lines.append(line)
427
428  # Strip away trailing commas at line ends. Ugh.
429  for i in range(len(lines) - 1):
430    if (lines[i].endswith(",") and len(lines[i + 1]) > 0 and
431        lines[i + 1][0] in ("}", "]")):
432      lines[i] = lines[i][:-1]
433
434  contents = "\n".join(lines)
435  # JSON wants double-quotes.
436  contents = contents.replace("'", '"')
437  # Fill in keywords (like PASS, SKIP).
438  for key in statusfile.KEYWORDS:
439    contents = re.sub(r"\b%s\b" % key, "\"%s\"" % key, contents)
440
441  status = {"success": True}
442  def check_pairs(pairs):
443    keys = {}
444    for key, value in pairs:
445      if key in keys:
446        print("%s: Error: duplicate key %s" % (filepath, key))
447        status["success"] = False
448      keys[key] = True
449
450  json.loads(contents, object_pairs_hook=check_pairs)
451  return status["success"]
452
453def CheckStatusFiles(workspace):
454  success = True
455  suite_paths = utils.GetSuitePaths(join(workspace, "test"))
456  for root in suite_paths:
457    suite_path = join(workspace, "test", root)
458    status_file_path = join(suite_path, root + ".status")
459    suite = testsuite.TestSuite.LoadTestSuite(suite_path)
460    if suite and exists(status_file_path):
461      success &= statusfile.PresubmitCheck(status_file_path)
462      success &= _CheckStatusFileForDuplicateKeys(status_file_path)
463  return success
464
465def CheckAuthorizedAuthor(input_api, output_api):
466  """For non-googler/chromites committers, verify the author's email address is
467  in AUTHORS.
468  """
469  # TODO(maruel): Add it to input_api?
470  import fnmatch
471
472  author = input_api.change.author_email
473  if not author:
474    input_api.logging.info('No author, skipping AUTHOR check')
475    return []
476  authors_path = input_api.os_path.join(
477      input_api.PresubmitLocalPath(), 'AUTHORS')
478  valid_authors = (
479      input_api.re.match(r'[^#]+\s+\<(.+?)\>\s*$', line)
480      for line in open(authors_path))
481  valid_authors = [item.group(1).lower() for item in valid_authors if item]
482  if not any(fnmatch.fnmatch(author.lower(), valid) for valid in valid_authors):
483    input_api.logging.info('Valid authors are %s', ', '.join(valid_authors))
484    return [output_api.PresubmitPromptWarning(
485        ('%s is not in AUTHORS file. If you are a new contributor, please visit'
486        '\n'
487        'http://www.chromium.org/developers/contributing-code and read the '
488        '"Legal" section\n'
489        'If you are a chromite, verify the contributor signed the CLA.') %
490        author)]
491  return []
492
493def GetOptions():
494  result = optparse.OptionParser()
495  result.add_option('--no-lint', help="Do not run cpplint", default=False,
496                    action="store_true")
497  return result
498
499
500def Main():
501  workspace = abspath(join(dirname(sys.argv[0]), '..'))
502  parser = GetOptions()
503  (options, args) = parser.parse_args()
504  success = True
505  print "Running C++ lint check..."
506  if not options.no_lint:
507    success &= CppLintProcessor().Run(workspace)
508  print "Running copyright header, trailing whitespaces and " \
509        "two empty lines between declarations check..."
510  success &= SourceProcessor().Run(workspace)
511  success &= CheckExternalReferenceRegistration(workspace)
512  success &= CheckStatusFiles(workspace)
513  if success:
514    return 0
515  else:
516    return 1
517
518
519if __name__ == '__main__':
520  sys.exit(Main())
521