1#!/usr/bin/env python3
2#
3# Copyright (C) 2017 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17
18import argparse
19import glob
20import logging
21import os
22import shutil
23import subprocess
24import tempfile
25import xml.etree.ElementTree as xml_tree
26
27import utils
28
29
30class GPLChecker(object):
31    """Checks that all GPL projects in a VNDK snapshot have released sources.
32
33    Makes sure that the current source tree have the sources for all GPL
34    prebuilt libraries in a specified VNDK snapshot version.
35    """
36    MANIFEST_XML = utils.MANIFEST_FILE_NAME
37    MODULE_PATHS_TXT = utils.MODULE_PATHS_FILE_NAME
38
39    def __init__(self, install_dir, android_build_top, gpl_projects,
40                 temp_artifact_dir, remote_git):
41        """GPLChecker constructor.
42
43        Args:
44          install_dir: string, absolute path to the prebuilts/vndk/v{version}
45            directory where the build files will be generated.
46          android_build_top: string, absolute path to ANDROID_BUILD_TOP
47          gpl_projects: list of strings, names of libraries under GPL
48          temp_artifact_dir: string, temp directory to hold build artifacts
49            fetched from Android Build server.
50          remote_git: string, remote name to fetch and check if the revision of
51            VNDK snapshot is included in the source if it is not in the current
52            git repository.
53        """
54        self._android_build_top = android_build_top
55        self._install_dir = install_dir
56        self._remote_git = remote_git
57        self._gpl_projects = gpl_projects
58        self._manifest_file = os.path.join(temp_artifact_dir,
59                                           self.MANIFEST_XML)
60
61        if not os.path.isfile(self._manifest_file):
62            raise RuntimeError(
63                '{manifest} not found at {manifest_file}'.format(
64                    manifest=self.MANIFEST_XML,
65                    manifest_file=self._manifest_file))
66
67    def _parse_module_paths(self):
68        """Parses the module_paths.txt files into a dictionary,
69
70        Returns:
71          module_paths: dict, e.g. {libfoo.so: some/path/here}
72        """
73        module_paths = dict()
74        for file in utils.find(self._install_dir, [self.MODULE_PATHS_TXT]):
75            file_path = os.path.join(self._install_dir, file)
76            with open(file_path, 'r') as f:
77                for line in f.read().strip().split('\n'):
78                    paths = line.split(' ')
79                    if len(paths) > 1:
80                        if paths[0] not in module_paths:
81                            module_paths[paths[0]] = paths[1]
82        return module_paths
83
84    def _parse_manifest(self):
85        """Parses manifest.xml file and returns list of 'project' tags."""
86
87        root = xml_tree.parse(self._manifest_file).getroot()
88        return root.findall('project')
89
90    def _get_revision(self, module_path, manifest_projects):
91        """Returns revision value recorded in manifest.xml for given project.
92
93        Args:
94          module_path: string, project path relative to ANDROID_BUILD_TOP
95          manifest_projects: list of xml_tree.Element, list of 'project' tags
96        """
97        revision = None
98        for project in manifest_projects:
99            path = project.get('path')
100            if module_path.startswith(path):
101                revision = project.get('revision')
102                break
103        return revision
104
105    def _check_revision_exists(self, revision, git_project_path):
106        """Checks whether a revision is found in a git project of current tree.
107
108        Args:
109          revision: string, revision value recorded in manifest.xml
110          git_project_path: string, path relative to ANDROID_BUILD_TOP
111        """
112        path = utils.join_realpath(self._android_build_top, git_project_path)
113
114        def _check_rev_list(revision):
115            """Checks whether revision is reachable from HEAD of git project."""
116
117            logging.info('Checking if revision {rev} exists in {proj}'.format(
118                rev=revision, proj=git_project_path))
119            try:
120                cmd = [
121                    'git', '-C', path, 'rev-list', 'HEAD..{}'.format(revision)
122                ]
123                output = utils.check_output(cmd).strip()
124            except subprocess.CalledProcessError as error:
125                logging.error('Error: {}'.format(error))
126                return False
127            else:
128                if output:
129                    logging.debug(
130                        '{proj} does not have the following revisions: {rev}'.
131                        format(proj=git_project_path, rev=output))
132                    return False
133                else:
134                    logging.info(
135                        'Found revision {rev} in project {proj}'.format(
136                            rev=revision, proj=git_project_path))
137            return True
138
139        def _get_2nd_parent_if_merge_commit(revision):
140            """Checks if the commit is merge commit.
141
142            Returns:
143              revision: string, the 2nd parent which is the merged commit.
144              If the commit is not a merge commit, returns None.
145            """
146            logging.info(
147                'Checking if the parent of revision {rev} exists in {proj}'.
148                format(rev=revision, proj=git_project_path))
149            try:
150                cmd = [
151                    'git', '-C', path, 'rev-parse', '--verify',
152                    '{}^2'.format(revision)]
153                parent_revision = utils.check_output(cmd).strip()
154            except subprocess.CalledProcessError as error:
155                logging.error(
156                    'Failed to get parent of revision {rev} from "{remote}": '
157                    '{err}'.format(
158                        rev=revision, remote=self._remote_git, err=error))
159                logging.error('{} is not a merge commit and must be included '
160                    'in the current branch'.format(revision))
161                return None
162            else:
163                return parent_revision
164
165        if _check_rev_list(revision):
166            return True
167
168        # VNDK snapshots built from a *-release branch will have merge
169        # CLs in the manifest because the *-dev branch is merged to the
170        # *-release branch periodically. In order to extract the
171        # revision relevant to the source of the git_project_path,
172        # we find the parent of the merge commit.
173        try:
174            cmd = ['git', '-C', path, 'fetch', self._remote_git, revision]
175            utils.check_call(cmd)
176        except subprocess.CalledProcessError as error:
177            logging.error(
178                'Failed to fetch revision {rev} from "{remote}": '
179                '{err}'.format(
180                    rev=revision, remote=self._remote_git, err=error))
181            logging.error('Try --remote to manually set remote name')
182            raise
183
184        parent_revision = _get_2nd_parent_if_merge_commit(revision)
185        while True:
186            if not parent_revision:
187                return False
188            if _check_rev_list(parent_revision):
189                return True
190            parent_revision = _get_2nd_parent_if_merge_commit(parent_revision)
191
192    def check_gpl_projects(self):
193        """Checks that all GPL projects have released sources.
194
195        Raises:
196          ValueError: There are GPL projects with unreleased sources.
197        """
198        logging.info('Starting license check for GPL projects...')
199
200        if not self._gpl_projects:
201            logging.info('No GPL projects found.')
202            return
203
204        logging.info('GPL projects found: {}'.format(', '.join(self._gpl_projects)))
205
206        module_paths = self._parse_module_paths()
207        manifest_projects = self._parse_manifest()
208        released_projects = []
209        unreleased_projects = []
210
211        for name in self._gpl_projects:
212            lib = name if name.endswith('.so') else name + '.so'
213            if lib not in module_paths:
214                raise RuntimeError(
215                    'No module path was found for {lib} in {module_paths}'.
216                    format(lib=lib, module_paths=self.MODULE_PATHS_TXT))
217
218            module_path = module_paths[lib]
219            revision = self._get_revision(module_path, manifest_projects)
220            if not revision:
221                raise RuntimeError(
222                    'No project found for {path} in {manifest}'.format(
223                        path=module_path, manifest=self.MANIFEST_XML))
224            revision_exists = self._check_revision_exists(
225                revision, module_path)
226            if not revision_exists:
227                unreleased_projects.append((lib, module_path))
228            else:
229                released_projects.append((lib, module_path))
230
231        if released_projects:
232            logging.info('Released GPL projects: {}'.format(released_projects))
233
234        if unreleased_projects:
235            raise ValueError(
236                ('FAIL: The following GPL projects have NOT been released in '
237                 'current tree: {}'.format(unreleased_projects)))
238
239        logging.info('PASS: All GPL projects have source in current tree.')
240
241
242def get_args():
243    parser = argparse.ArgumentParser()
244    parser.add_argument(
245        'vndk_version',
246        type=utils.vndk_version_int,
247        help='VNDK snapshot version to check, e.g. "{}".'.format(
248            utils.MINIMUM_VNDK_VERSION))
249    parser.add_argument('-b', '--branch', help='Branch to pull manifest from.')
250    parser.add_argument('--build', help='Build number to pull manifest from.')
251    parser.add_argument(
252        '--remote',
253        default='aosp',
254        help=('Remote name to fetch and check if the revision of VNDK snapshot '
255              'is included in the source to conform GPL license. default=aosp'))
256    parser.add_argument('-m', '--modules', help='list of modules to check',
257                        nargs='+')
258    parser.add_argument(
259        '-v',
260        '--verbose',
261        action='count',
262        default=0,
263        help='Increase output verbosity, e.g. "-v", "-vv".')
264    return parser.parse_args()
265
266
267def main():
268    """For local testing purposes.
269
270    Note: VNDK snapshot must be already installed under
271      prebuilts/vndk/v{version}.
272    """
273    ANDROID_BUILD_TOP = utils.get_android_build_top()
274    PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP,
275                                             'prebuilts/vndk')
276
277    args = get_args()
278    vndk_version = args.vndk_version
279    install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version))
280    remote = args.remote
281    if not os.path.isdir(install_dir):
282        raise ValueError(
283            'Please provide valid VNDK version. {} does not exist.'
284            .format(install_dir))
285    utils.set_logging_config(args.verbose)
286
287    temp_artifact_dir = tempfile.mkdtemp()
288    os.chdir(temp_artifact_dir)
289    manifest_pattern = 'manifest_{}.xml'.format(args.build)
290    manifest_dest = os.path.join(temp_artifact_dir, utils.MANIFEST_FILE_NAME)
291    logging.info('Fetching {file} from {branch} (bid: {build})'.format(
292        file=manifest_pattern, branch=args.branch, build=args.build))
293    utils.fetch_artifact(args.branch, args.build, manifest_pattern,
294                         manifest_dest)
295
296    license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP, args.modules,
297                                 temp_artifact_dir, remote)
298    try:
299        license_checker.check_gpl_projects()
300    except ValueError as error:
301        logging.error('Error: {}'.format(error))
302        raise
303    finally:
304        logging.info(
305            'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir))
306        shutil.rmtree(temp_artifact_dir)
307
308    logging.info('Done.')
309
310
311if __name__ == '__main__':
312    main()
313