1# Copyright 2020 The Pigweed Authors
2#
3# Licensed under the Apache License, Version 2.0 (the "License"); you may not
4# use this file except in compliance with the License. You may obtain a copy of
5# the License at
6#
7#     https://www.apache.org/licenses/LICENSE-2.0
8#
9# Unless required by applicable law or agreed to in writing, software
10# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
11# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
12# License for the specific language governing permissions and limitations under
13# the License.
14"""Preconfigured checks for Python code.
15
16These checks assume that they are running in a preconfigured Python environment.
17"""
18
19import logging
20import os
21from pathlib import Path
22import sys
23from typing import Callable, Iterable, List, Set, Tuple
24
25try:
26    import pw_presubmit
27except ImportError:
28    # Append the pw_presubmit package path to the module search path to allow
29    # running this module without installing the pw_presubmit package.
30    sys.path.append(os.path.dirname(os.path.dirname(
31        os.path.abspath(__file__))))
32    import pw_presubmit
33
34from pw_presubmit import call, filter_paths
35from pw_presubmit.git_repo import python_packages_containing, list_files
36from pw_presubmit.git_repo import PythonPackage
37
38_LOG = logging.getLogger(__name__)
39
40
41def run_module(*args, **kwargs):
42    return call('python', '-m', *args, **kwargs)
43
44
45TEST_PATTERNS = ('*_test.py', )
46
47
48@filter_paths(endswith='.py')
49def test_python_packages(ctx: pw_presubmit.PresubmitContext,
50                         patterns: Iterable[str] = TEST_PATTERNS) -> None:
51    """Finds and runs test files in Python package directories.
52
53    Finds the Python packages containing the affected paths, then searches
54    within that package for test files. All files matching the provided patterns
55    are executed with Python.
56    """
57    packages: List[PythonPackage] = []
58    for repo in ctx.repos:
59        packages += python_packages_containing(ctx.paths, repo=repo)[0]
60
61    if not packages:
62        _LOG.info('No Python packages were found.')
63        return
64
65    for package in packages:
66        for test in list_files(pathspecs=tuple(patterns),
67                               repo_path=package.root):
68            call('python', test)
69
70
71@filter_paths(endswith='.py')
72def pylint(ctx: pw_presubmit.PresubmitContext) -> None:
73    disable_checkers = [
74        # BUG(pwbug/22): Hanging indent check conflicts with YAPF 0.29. For
75        # now, use YAPF's version even if Pylint is doing the correct thing
76        # just to keep operations simpler. When YAPF upstream fixes the issue,
77        # delete this code.
78        #
79        # See also: https://github.com/google/yapf/issues/781
80        'bad-continuation',
81    ]
82    run_module(
83        'pylint',
84        '--jobs=0',
85        f'--disable={",".join(disable_checkers)}',
86        *ctx.paths,
87        cwd=ctx.root,
88    )
89
90
91@filter_paths(endswith='.py')
92def mypy(ctx: pw_presubmit.PresubmitContext) -> None:
93    """Runs mypy on all paths and their packages."""
94    packages: List[PythonPackage] = []
95    other_files: List[Path] = []
96
97    for repo, paths in ctx.paths_by_repo().items():
98        new_packages, files = python_packages_containing(paths, repo=repo)
99        packages += new_packages
100        other_files += files
101
102        for package in new_packages:
103            other_files += package.other_files
104
105    # Under some circumstances, mypy cannot check multiple Python files with the
106    # same module name. Group filenames so that no duplicates occur in the same
107    # mypy invocation. Also, omit setup.py from mypy checks.
108    filename_sets: List[Set[str]] = [set()]
109    path_sets: List[List[Path]] = [list(p.package for p in packages)]
110
111    for path in (p for p in other_files if p.name != 'setup.py'):
112        for filenames, paths in zip(filename_sets, path_sets):
113            if path.name not in filenames:
114                paths.append(path)
115                filenames.add(path.name)
116                break
117        else:
118            path_sets.append([path])
119            filename_sets.append({path.name})
120
121    env = os.environ.copy()
122    # Use this environment variable to force mypy to colorize output.
123    # See https://github.com/python/mypy/issues/7771
124    env['MYPY_FORCE_COLOR'] = '1'
125
126    for paths in path_sets:
127        run_module(
128            'mypy',
129            *paths,
130            '--pretty',
131            '--color-output',
132            '--show-error-codes',
133            # TODO(pwbug/146): Some imports from installed packages fail. These
134            # imports should be fixed and this option removed. See
135            # https://mypy.readthedocs.io/en/stable/installed_packages.html
136            '--ignore-missing-imports',
137            env=env)
138
139
140_ALL_CHECKS = (
141    test_python_packages,
142    pylint,
143    mypy,
144)
145
146
147def all_checks(endswith: str = '.py',
148               **filter_paths_args) -> Tuple[Callable, ...]:
149    return tuple(
150        filter_paths(endswith=endswith, **filter_paths_args)(function)
151        for function in _ALL_CHECKS)
152