1# -*- coding:utf-8 -*-
2# Copyright 2016 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15
16"""Functions that implement the actual checks."""
17
18from __future__ import print_function
19
20import json
21import os
22import platform
23import re
24import sys
25
26_path = os.path.realpath(__file__ + '/../..')
27if sys.path[0] != _path:
28    sys.path.insert(0, _path)
29del _path
30
31# pylint: disable=wrong-import-position
32import rh.results
33import rh.git
34import rh.utils
35
36
37class Placeholders(object):
38    """Holder class for replacing ${vars} in arg lists.
39
40    To add a new variable to replace in config files, just add it as a @property
41    to this class using the form.  So to add support for BIRD:
42      @property
43      def var_BIRD(self):
44        return <whatever this is>
45
46    You can return either a string or an iterable (e.g. a list or tuple).
47    """
48
49    def __init__(self, diff=()):
50        """Initialize.
51
52        Args:
53          diff: The list of files that changed.
54        """
55        self.diff = diff
56
57    def expand_vars(self, args):
58        """Perform place holder expansion on all of |args|.
59
60        Args:
61          args: The args to perform expansion on.
62
63        Returns:
64          The updated |args| list.
65        """
66        all_vars = set(self.vars())
67        replacements = dict((var, self.get(var)) for var in all_vars)
68
69        ret = []
70        for arg in args:
71            # First scan for exact matches
72            for key, val in replacements.items():
73                var = '${%s}' % (key,)
74                if arg == var:
75                    if isinstance(val, str):
76                        ret.append(val)
77                    else:
78                        ret.extend(val)
79                    # We break on first hit to avoid double expansion.
80                    break
81            else:
82                # If no exact matches, do an inline replacement.
83                def replace(m):
84                    val = self.get(m.group(1))
85                    if isinstance(val, str):
86                        return val
87                    else:
88                        return ' '.join(val)
89                ret.append(re.sub(r'\$\{(%s)\}' % ('|'.join(all_vars),),
90                                  replace, arg))
91
92        return ret
93
94    @classmethod
95    def vars(cls):
96        """Yield all replacement variable names."""
97        for key in dir(cls):
98            if key.startswith('var_'):
99                yield key[4:]
100
101    def get(self, var):
102        """Helper function to get the replacement |var| value."""
103        return getattr(self, 'var_%s' % (var,))
104
105    @property
106    def var_PREUPLOAD_COMMIT_MESSAGE(self):
107        """The git commit message."""
108        return os.environ.get('PREUPLOAD_COMMIT_MESSAGE', '')
109
110    @property
111    def var_PREUPLOAD_COMMIT(self):
112        """The git commit sha1."""
113        return os.environ.get('PREUPLOAD_COMMIT', '')
114
115    @property
116    def var_PREUPLOAD_FILES(self):
117        """List of files modified in this git commit."""
118        return [x.file for x in self.diff if x.status != 'D']
119
120    @property
121    def var_REPO_ROOT(self):
122        """The root of the repo checkout."""
123        return rh.git.find_repo_root()
124
125    @property
126    def var_BUILD_OS(self):
127        """The build OS (see _get_build_os_name for details)."""
128        return _get_build_os_name()
129
130
131class HookOptions(object):
132    """Holder class for hook options."""
133
134    def __init__(self, name, args, tool_paths):
135        """Initialize.
136
137        Args:
138          name: The name of the hook.
139          args: The override commandline arguments for the hook.
140          tool_paths: A dictionary with tool names to paths.
141        """
142        self.name = name
143        self._args = args
144        self._tool_paths = tool_paths
145
146    @staticmethod
147    def expand_vars(args, diff=()):
148        """Perform place holder expansion on all of |args|."""
149        replacer = Placeholders(diff=diff)
150        return replacer.expand_vars(args)
151
152    def args(self, default_args=(), diff=()):
153        """Gets the hook arguments, after performing place holder expansion.
154
155        Args:
156          default_args: The list to return if |self._args| is empty.
157          diff: The list of files that changed in the current commit.
158
159        Returns:
160          A list with arguments.
161        """
162        args = self._args
163        if not args:
164            args = default_args
165
166        return self.expand_vars(args, diff=diff)
167
168    def tool_path(self, tool_name):
169        """Gets the path in which the |tool_name| executable can be found.
170
171        This function performs expansion for some place holders.  If the tool
172        does not exist in the overridden |self._tool_paths| dictionary, the tool
173        name will be returned and will be run from the user's $PATH.
174
175        Args:
176          tool_name: The name of the executable.
177
178        Returns:
179          The path of the tool with all optional place holders expanded.
180        """
181        assert tool_name in TOOL_PATHS
182        if tool_name not in self._tool_paths:
183            return TOOL_PATHS[tool_name]
184
185        tool_path = os.path.normpath(self._tool_paths[tool_name])
186        return self.expand_vars([tool_path])[0]
187
188
189def _run_command(cmd, **kwargs):
190    """Helper command for checks that tend to gather output."""
191    kwargs.setdefault('redirect_stderr', True)
192    kwargs.setdefault('combine_stdout_stderr', True)
193    kwargs.setdefault('capture_output', True)
194    kwargs.setdefault('error_code_ok', True)
195    return rh.utils.run_command(cmd, **kwargs)
196
197
198def _match_regex_list(subject, expressions):
199    """Try to match a list of regular expressions to a string.
200
201    Args:
202      subject: The string to match regexes on.
203      expressions: An iterable of regular expressions to check for matches with.
204
205    Returns:
206      Whether the passed in subject matches any of the passed in regexes.
207    """
208    for expr in expressions:
209        if re.search(expr, subject):
210            return True
211    return False
212
213
214def _filter_diff(diff, include_list, exclude_list=()):
215    """Filter out files based on the conditions passed in.
216
217    Args:
218      diff: list of diff objects to filter.
219      include_list: list of regex that when matched with a file path will cause
220          it to be added to the output list unless the file is also matched with
221          a regex in the exclude_list.
222      exclude_list: list of regex that when matched with a file will prevent it
223          from being added to the output list, even if it is also matched with a
224          regex in the include_list.
225
226    Returns:
227      A list of filepaths that contain files matched in the include_list and not
228      in the exclude_list.
229    """
230    filtered = []
231    for d in diff:
232        if (d.status != 'D' and
233                _match_regex_list(d.file, include_list) and
234                not _match_regex_list(d.file, exclude_list)):
235            # We've got a match!
236            filtered.append(d)
237    return filtered
238
239
240def _get_build_os_name():
241    """Gets the build OS name.
242
243    Returns:
244      A string in a format usable to get prebuilt tool paths.
245    """
246    system = platform.system()
247    if 'Darwin' in system or 'Macintosh' in system:
248        return 'darwin-x86'
249    else:
250        # TODO: Add more values if needed.
251        return 'linux-x86'
252
253
254def _fixup_func_caller(cmd, **kwargs):
255    """Wraps |cmd| around a callable automated fixup.
256
257    For hooks that support automatically fixing errors after running (e.g. code
258    formatters), this function provides a way to run |cmd| as the |fixup_func|
259    parameter in HookCommandResult.
260    """
261    def wrapper():
262        result = _run_command(cmd, **kwargs)
263        if result.returncode not in (None, 0):
264            return result.output
265        return None
266    return wrapper
267
268
269def _check_cmd(hook_name, project, commit, cmd, fixup_func=None, **kwargs):
270    """Runs |cmd| and returns its result as a HookCommandResult."""
271    return [rh.results.HookCommandResult(hook_name, project, commit,
272                                         _run_command(cmd, **kwargs),
273                                         fixup_func=fixup_func)]
274
275
276# Where helper programs exist.
277TOOLS_DIR = os.path.realpath(__file__ + '/../../tools')
278
279def get_helper_path(tool):
280    """Return the full path to the helper |tool|."""
281    return os.path.join(TOOLS_DIR, tool)
282
283
284def check_custom(project, commit, _desc, diff, options=None, **kwargs):
285    """Run a custom hook."""
286    return _check_cmd(options.name, project, commit, options.args((), diff),
287                      **kwargs)
288
289
290def check_checkpatch(project, commit, _desc, diff, options=None):
291    """Run |diff| through the kernel's checkpatch.pl tool."""
292    tool = get_helper_path('checkpatch.pl')
293    cmd = ([tool, '-', '--root', project.dir] +
294           options.args(('--ignore=GERRIT_CHANGE_ID',), diff))
295    return _check_cmd('checkpatch.pl', project, commit, cmd,
296                      input=rh.git.get_patch(commit))
297
298
299def check_clang_format(project, commit, _desc, diff, options=None):
300    """Run git clang-format on the commit."""
301    tool = get_helper_path('clang-format.py')
302    clang_format = options.tool_path('clang-format')
303    git_clang_format = options.tool_path('git-clang-format')
304    tool_args = (['--clang-format', clang_format, '--git-clang-format',
305                  git_clang_format] +
306                 options.args(('--style', 'file', '--commit', commit), diff))
307    cmd = [tool] + tool_args
308    fixup_func = _fixup_func_caller([tool, '--fix'] + tool_args)
309    return _check_cmd('clang-format', project, commit, cmd,
310                      fixup_func=fixup_func)
311
312
313def check_google_java_format(project, commit, _desc, _diff, options=None):
314    """Run google-java-format on the commit."""
315
316    tool = get_helper_path('google-java-format.py')
317    google_java_format = options.tool_path('google-java-format')
318    google_java_format_diff = options.tool_path('google-java-format-diff')
319    tool_args = ['--google-java-format', google_java_format,
320                 '--google-java-format-diff', google_java_format_diff,
321                 '--commit', commit] + options.args()
322    cmd = [tool] + tool_args
323    fixup_func = _fixup_func_caller([tool, '--fix'] + tool_args)
324    return _check_cmd('google-java-format', project, commit, cmd,
325                      fixup_func=fixup_func)
326
327
328def check_commit_msg_bug_field(project, commit, desc, _diff, options=None):
329    """Check the commit message for a 'Bug:' line."""
330    field = 'Bug'
331    regex = r'^%s: (None|[0-9]+(, [0-9]+)*)$' % (field,)
332    check_re = re.compile(regex)
333
334    if options.args():
335        raise ValueError('commit msg %s check takes no options' % (field,))
336
337    found = []
338    for line in desc.splitlines():
339        if check_re.match(line):
340            found.append(line)
341
342    if not found:
343        error = ('Commit message is missing a "%s:" line.  It must match:\n'
344                 '%s') % (field, regex)
345    else:
346        return
347
348    return [rh.results.HookResult('commit msg: "%s:" check' % (field,),
349                                  project, commit, error=error)]
350
351
352def check_commit_msg_changeid_field(project, commit, desc, _diff, options=None):
353    """Check the commit message for a 'Change-Id:' line."""
354    field = 'Change-Id'
355    regex = r'^%s: I[a-f0-9]+$' % (field,)
356    check_re = re.compile(regex)
357
358    if options.args():
359        raise ValueError('commit msg %s check takes no options' % (field,))
360
361    found = []
362    for line in desc.splitlines():
363        if check_re.match(line):
364            found.append(line)
365
366    if len(found) == 0:
367        error = ('Commit message is missing a "%s:" line.  It must match:\n'
368                 '%s') % (field, regex)
369    elif len(found) > 1:
370        error = ('Commit message has too many "%s:" lines.  There can be only '
371                 'one.') % (field,)
372    else:
373        return
374
375    return [rh.results.HookResult('commit msg: "%s:" check' % (field,),
376                                  project, commit, error=error)]
377
378
379TEST_MSG = """Commit message is missing a "Test:" line.  It must match:
380%s
381
382The Test: stanza is free-form and should describe how you tested your change.
383As a CL author, you'll have a consistent place to describe the testing strategy
384you use for your work. As a CL reviewer, you'll be reminded to discuss testing
385as part of your code review, and you'll more easily replicate testing when you
386patch in CLs locally.
387
388Some examples below:
389
390Test: make WITH_TIDY=1 mmma art
391Test: make test-art
392Test: manual - took a photo
393Test: refactoring CL. Existing unit tests still pass.
394
395Check the git history for more examples. It's a free-form field, so we urge
396you to develop conventions that make sense for your project. Note that many
397projects use exact test commands, which are perfectly fine.
398
399Adding good automated tests with new code is critical to our goals of keeping
400the system stable and constantly improving quality. Please use Test: to
401highlight this area of your development. And reviewers, please insist on
402high-quality Test: descriptions.
403"""
404
405
406def check_commit_msg_test_field(project, commit, desc, _diff, options=None):
407    """Check the commit message for a 'Test:' line."""
408    field = 'Test'
409    regex = r'^%s: .*$' % (field,)
410    check_re = re.compile(regex)
411
412    if options.args():
413        raise ValueError('commit msg %s check takes no options' % (field,))
414
415    found = []
416    for line in desc.splitlines():
417        if check_re.match(line):
418            found.append(line)
419
420    if not found:
421        error = TEST_MSG % (regex)
422    else:
423        return
424
425    return [rh.results.HookResult('commit msg: "%s:" check' % (field,),
426                                  project, commit, error=error)]
427
428
429def check_cpplint(project, commit, _desc, diff, options=None):
430    """Run cpplint."""
431    # This list matches what cpplint expects.  We could run on more (like .cxx),
432    # but cpplint would just ignore them.
433    filtered = _filter_diff(diff, [r'\.(cc|h|cpp|cu|cuh)$'])
434    if not filtered:
435        return
436
437    cpplint = options.tool_path('cpplint')
438    cmd = [cpplint] + options.args(('${PREUPLOAD_FILES}',), filtered)
439    return _check_cmd('cpplint', project, commit, cmd)
440
441
442def check_gofmt(project, commit, _desc, diff, options=None):
443    """Checks that Go files are formatted with gofmt."""
444    filtered = _filter_diff(diff, [r'\.go$'])
445    if not filtered:
446        return
447
448    gofmt = options.tool_path('gofmt')
449    cmd = [gofmt, '-l'] + options.args((), filtered)
450    ret = []
451    for d in filtered:
452        data = rh.git.get_file_content(commit, d.file)
453        result = _run_command(cmd, input=data)
454        if result.output:
455            ret.append(rh.results.HookResult(
456                'gofmt', project, commit, error=result.output,
457                files=(d.file,)))
458    return ret
459
460
461def check_json(project, commit, _desc, diff, options=None):
462    """Verify json files are valid."""
463    if options.args():
464        raise ValueError('json check takes no options')
465
466    filtered = _filter_diff(diff, [r'\.json$'])
467    if not filtered:
468        return
469
470    ret = []
471    for d in filtered:
472        data = rh.git.get_file_content(commit, d.file)
473        try:
474            json.loads(data)
475        except ValueError as e:
476            ret.append(rh.results.HookResult(
477                'json', project, commit, error=str(e),
478                files=(d.file,)))
479    return ret
480
481
482def check_pylint(project, commit, _desc, diff, options=None):
483    """Run pylint."""
484    filtered = _filter_diff(diff, [r'\.py$'])
485    if not filtered:
486        return
487
488    pylint = options.tool_path('pylint')
489    cmd = [
490        get_helper_path('pylint.py'),
491        '--executable-path', pylint,
492    ] + options.args(('${PREUPLOAD_FILES}',), filtered)
493    return _check_cmd('pylint', project, commit, cmd)
494
495
496def check_xmllint(project, commit, _desc, diff, options=None):
497    """Run xmllint."""
498    # XXX: Should we drop most of these and probe for <?xml> tags?
499    extensions = frozenset((
500        'dbus-xml',  # Generated DBUS interface.
501        'dia',       # File format for Dia.
502        'dtd',       # Document Type Definition.
503        'fml',       # Fuzzy markup language.
504        'form',      # Forms created by IntelliJ GUI Designer.
505        'fxml',      # JavaFX user interfaces.
506        'glade',     # Glade user interface design.
507        'grd',       # GRIT translation files.
508        'iml',       # Android build modules?
509        'kml',       # Keyhole Markup Language.
510        'mxml',      # Macromedia user interface markup language.
511        'nib',       # OS X Cocoa Interface Builder.
512        'plist',     # Property list (for OS X).
513        'pom',       # Project Object Model (for Apache Maven).
514        'rng',       # RELAX NG schemas.
515        'sgml',      # Standard Generalized Markup Language.
516        'svg',       # Scalable Vector Graphics.
517        'uml',       # Unified Modeling Language.
518        'vcproj',    # Microsoft Visual Studio project.
519        'vcxproj',   # Microsoft Visual Studio project.
520        'wxs',       # WiX Transform File.
521        'xhtml',     # XML HTML.
522        'xib',       # OS X Cocoa Interface Builder.
523        'xlb',       # Android locale bundle.
524        'xml',       # Extensible Markup Language.
525        'xsd',       # XML Schema Definition.
526        'xsl',       # Extensible Stylesheet Language.
527    ))
528
529    filtered = _filter_diff(diff, [r'\.(%s)$' % '|'.join(extensions)])
530    if not filtered:
531        return
532
533    # TODO: Figure out how to integrate schema validation.
534    # XXX: Should we use python's XML libs instead?
535    cmd = ['xmllint'] + options.args(('${PREUPLOAD_FILES}',), filtered)
536
537    return _check_cmd('xmllint', project, commit, cmd)
538
539
540# Hooks that projects can opt into.
541# Note: Make sure to keep the top level README.md up to date when adding more!
542BUILTIN_HOOKS = {
543    'checkpatch': check_checkpatch,
544    'clang_format': check_clang_format,
545    'commit_msg_bug_field': check_commit_msg_bug_field,
546    'commit_msg_changeid_field': check_commit_msg_changeid_field,
547    'commit_msg_test_field': check_commit_msg_test_field,
548    'cpplint': check_cpplint,
549    'gofmt': check_gofmt,
550    'google_java_format': check_google_java_format,
551    'jsonlint': check_json,
552    'pylint': check_pylint,
553    'xmllint': check_xmllint,
554}
555
556# Additional tools that the hooks can call with their default values.
557# Note: Make sure to keep the top level README.md up to date when adding more!
558TOOL_PATHS = {
559    'clang-format': 'clang-format',
560    'cpplint': os.path.join(TOOLS_DIR, 'cpplint.py'),
561    'git-clang-format': 'git-clang-format',
562    'gofmt': 'gofmt',
563    'google-java-format': 'google-java-format',
564    'google-java-format-diff': 'google-java-format-diff.py',
565    'pylint': 'pylint',
566}
567