1#!/usr/bin/env python3
2#
3# Copyright (C) 2017 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17"""Installs VNDK snapshot under prebuilts/vndk/v{version}."""
18
19import argparse
20import glob
21import logging
22import os
23import re
24import shutil
25import subprocess
26import sys
27import tempfile
28import textwrap
29
30import utils
31
32from check_gpl_license import GPLChecker
33from gen_buildfiles import GenBuildFile
34
35ANDROID_BUILD_TOP = utils.get_android_build_top()
36PREBUILTS_VNDK_DIR = utils.join_realpath(ANDROID_BUILD_TOP, 'prebuilts/vndk')
37
38
39def start_branch(build):
40    branch_name = 'update-' + (build or 'local')
41    logging.info('Creating branch {branch} in {dir}'.format(
42        branch=branch_name, dir=os.getcwd()))
43    utils.check_call(['repo', 'start', branch_name, '.'])
44
45
46def remove_old_snapshot(install_dir):
47    logging.info('Removing any old files in {}'.format(install_dir))
48    for file in glob.glob('{}/*'.format(install_dir)):
49        try:
50            if os.path.isfile(file):
51                os.unlink(file)
52            elif os.path.isdir(file):
53                shutil.rmtree(file)
54        except Exception as error:
55            logging.error('Error: {}'.format(error))
56            sys.exit(1)
57
58
59def install_snapshot(branch, build, local_dir, install_dir, temp_artifact_dir):
60    """Installs VNDK snapshot build artifacts to prebuilts/vndk/v{version}.
61
62    1) Fetch build artifacts from Android Build server or from local_dir
63    2) Unzip build artifacts
64
65    Args:
66      branch: string or None, branch name of build artifacts
67      build: string or None, build number of build artifacts
68      local_dir: string or None, local dir to pull artifacts from
69      install_dir: string, directory to install VNDK snapshot
70      temp_artifact_dir: string, temp directory to hold build artifacts fetched
71        from Android Build server. For 'local' option, is set to None.
72    """
73    artifact_pattern = 'android-vndk-*.zip'
74
75    if branch and build:
76        artifact_dir = temp_artifact_dir
77        os.chdir(temp_artifact_dir)
78        logging.info('Fetching {pattern} from {branch} (bid: {build})'.format(
79            pattern=artifact_pattern, branch=branch, build=build))
80        utils.fetch_artifact(branch, build, artifact_pattern)
81
82        manifest_pattern = 'manifest_{}.xml'.format(build)
83        logging.info('Fetching {file} from {branch} (bid: {build})'.format(
84            file=manifest_pattern, branch=branch, build=build))
85        utils.fetch_artifact(branch, build, manifest_pattern,
86                             utils.MANIFEST_FILE_NAME)
87
88        os.chdir(install_dir)
89    elif local_dir:
90        logging.info('Fetching local VNDK snapshot from {}'.format(local_dir))
91        artifact_dir = local_dir
92
93    artifacts = glob.glob(os.path.join(artifact_dir, artifact_pattern))
94    for artifact in artifacts:
95        logging.info('Unzipping VNDK snapshot: {}'.format(artifact))
96        utils.check_call(['unzip', '-qn', artifact, '-d', install_dir])
97
98        # rename {install_dir}/{arch}/include/out/soong/.intermediates
99        for soong_intermediates_dir in glob.glob(install_dir + '/*/include/' + utils.SOONG_INTERMEDIATES_DIR):
100            generated_headers_dir = soong_intermediates_dir.replace(
101                utils.SOONG_INTERMEDIATES_DIR,
102                utils.GENERATED_HEADERS_DIR
103            )
104            os.rename(soong_intermediates_dir, generated_headers_dir)
105
106def gather_notice_files(install_dir):
107    """Gathers all NOTICE files to a common NOTICE_FILES directory."""
108
109    common_notices_dir = utils.NOTICE_FILES_DIR_PATH
110    logging.info('Creating {} directory to gather all NOTICE files...'.format(
111        common_notices_dir))
112    os.makedirs(common_notices_dir)
113    for arch in utils.get_snapshot_archs(install_dir):
114        notices_dir_per_arch = os.path.join(arch, utils.NOTICE_FILES_DIR_NAME)
115        if os.path.isdir(notices_dir_per_arch):
116            for notice_file in glob.glob(
117                    '{}/**'.format(notices_dir_per_arch), recursive=True):
118                if os.path.isfile(notice_file):
119                    rel_path = os.path.relpath(notice_file, notices_dir_per_arch)
120                    target_path = os.path.join(common_notices_dir, rel_path)
121                    if not os.path.isfile(target_path):
122                        os.makedirs(os.path.dirname(target_path), exist_ok=True)
123                        shutil.copy(notice_file, target_path)
124            shutil.rmtree(notices_dir_per_arch)
125
126
127def post_processe_files_if_needed(vndk_version):
128    """Renames vndkcore.libraries.txt and vndksp.libraries.txt
129    files to have version suffix.
130    Create empty vndkproduct.libraries.txt file if not exist.
131
132    Args:
133      vndk_version: int, version of VNDK snapshot
134    """
135    def add_version_suffix(file_name):
136        logging.info('Rename {} to have version suffix'.format(file_name))
137        target_files = glob.glob(
138            os.path.join(utils.CONFIG_DIR_PATH_PATTERN, file_name))
139        for target_file in target_files:
140            name, ext = os.path.splitext(target_file)
141            os.rename(target_file, name + '.' + str(vndk_version) + ext)
142    def create_empty_file_if_not_exist(file_name):
143        target_dirs = glob.glob(utils.CONFIG_DIR_PATH_PATTERN)
144        for dir in target_dirs:
145            path = os.path.join(dir, file_name)
146            if os.path.isfile(path):
147                continue
148            logging.info('Creating empty file: {}'.format(path))
149            open(path, 'a').close()
150
151    files_to_add_version_suffix = ('vndkcore.libraries.txt',
152                                   'vndkprivate.libraries.txt')
153    files_to_create_if_not_exist = ('vndkproduct.libraries.txt',)
154    for file_to_rename in files_to_add_version_suffix:
155        add_version_suffix(file_to_rename)
156    for file_to_create in files_to_create_if_not_exist:
157        create_empty_file_if_not_exist(file_to_create)
158
159
160def update_buildfiles(buildfile_generator):
161    # To parse json information, read and generate arch android.bp using
162    # generate_android_bp() first.
163    logging.info('Generating Android.bp files...')
164    buildfile_generator.generate_android_bp()
165
166    logging.info('Generating root Android.bp file...')
167    buildfile_generator.generate_root_android_bp()
168
169    logging.info('Generating common/Android.bp file...')
170    buildfile_generator.generate_common_android_bp()
171
172def copy_owners(root_dir, install_dir):
173    path = os.path.dirname(__file__)
174    shutil.copy(os.path.join(root_dir, path, 'OWNERS'), install_dir)
175
176def check_gpl_license(license_checker):
177    try:
178        license_checker.check_gpl_projects()
179    except ValueError as error:
180        logging.error('***CANNOT INSTALL VNDK SNAPSHOT***: {}'.format(error))
181        raise
182
183
184def commit(branch, build, version):
185    logging.info('Making commit...')
186    utils.check_call(['git', 'add', '.'])
187    message = textwrap.dedent("""\
188        Update VNDK snapshot v{version} to build {build}.
189
190        Taken from branch {branch}.""").format(
191        version=version, branch=branch, build=build)
192    utils.check_call(['git', 'commit', '-m', message])
193
194
195def run(vndk_version, branch, build_id, local, use_current_branch, remote,
196        verbose):
197    ''' Fetch and updtate the VNDK snapshots
198
199    Args:
200      vndk_version: int, VNDK snapshot version to install.
201      branch: string, Branch to pull build from.
202      build: string, Build number to pull.
203      local: string, Fetch local VNDK snapshot artifacts from specified local
204             directory instead of Android Build server.
205      use-current-branch: boolean, Perform the update in the current branch.
206                          Do not repo start.
207      remote: string, Remote name to fetch and check if the revision of VNDK
208              snapshot is included in the source to conform GPL license.
209      verbose: int, Increase log output verbosity.
210    '''
211    local_path = None
212    if local:
213        local_path = os.path.abspath(os.path.expanduser(local))
214
215    if local_path:
216        if build_id or branch:
217            raise ValueError(
218                'When --local option is set, --branch or --build cannot be '
219                'specified.')
220        elif not os.path.isdir(local_path):
221            raise RuntimeError(
222                'The specified local directory, {}, does not exist.'.format(
223                    local_path))
224    else:
225        if not (build_id and branch):
226            raise ValueError(
227                'Please provide both --branch and --build or set --local '
228                'option.')
229
230    install_dir = os.path.join(PREBUILTS_VNDK_DIR, 'v{}'.format(vndk_version))
231    if not os.path.isdir(install_dir):
232        raise RuntimeError(
233            'The directory for VNDK snapshot version {ver} does not exist.\n'
234            'Please request a new git project for prebuilts/vndk/v{ver} '
235            'before installing new snapshot.'.format(ver=vndk_version))
236
237    utils.set_logging_config(verbose)
238    root_dir = os.getcwd()
239    os.chdir(install_dir)
240
241    if not use_current_branch:
242        start_branch(build_id)
243
244    remove_old_snapshot(install_dir)
245    os.makedirs(utils.COMMON_DIR_PATH)
246
247    temp_artifact_dir = None
248    if not local_path:
249        temp_artifact_dir = tempfile.mkdtemp()
250
251    try:
252        install_snapshot(branch, build_id, local_path, install_dir,
253                         temp_artifact_dir)
254        gather_notice_files(install_dir)
255        post_processe_files_if_needed(vndk_version)
256
257        buildfile_generator = GenBuildFile(install_dir, vndk_version)
258        update_buildfiles(buildfile_generator)
259
260        copy_owners(root_dir, install_dir)
261
262        if not local_path and not branch.startswith('android'):
263            license_checker = GPLChecker(install_dir, ANDROID_BUILD_TOP,
264                                         buildfile_generator.modules_with_restricted_lic,
265                                         temp_artifact_dir, remote)
266            check_gpl_license(license_checker)
267            logging.info(
268                'Successfully updated VNDK snapshot v{}'.format(vndk_version))
269    except Exception as error:
270        logging.error('FAILED TO INSTALL SNAPSHOT: {}'.format(error))
271        raise
272    finally:
273        if temp_artifact_dir:
274            logging.info(
275                'Deleting temp_artifact_dir: {}'.format(temp_artifact_dir))
276            shutil.rmtree(temp_artifact_dir)
277
278    if not local_path:
279        commit(branch, build_id, vndk_version)
280        logging.info(
281            'Successfully created commit for VNDK snapshot v{}'.format(
282                vndk_version))
283
284    logging.info('Done.')
285
286
287def get_args():
288    parser = argparse.ArgumentParser()
289    parser.add_argument(
290        'vndk_version',
291        type=utils.vndk_version_int,
292        help='VNDK snapshot version to install, e.g. "{}".'.format(
293            utils.MINIMUM_VNDK_VERSION))
294    parser.add_argument('-b', '--branch', help='Branch to pull build from.')
295    parser.add_argument('--build', help='Build number to pull.')
296    parser.add_argument(
297        '--local',
298        help=('Fetch local VNDK snapshot artifacts from specified local '
299              'directory instead of Android Build server. '
300              'Example: --local=/path/to/local/dir'))
301    parser.add_argument(
302        '--use-current-branch',
303        action='store_true',
304        help='Perform the update in the current branch. Do not repo start.')
305    parser.add_argument(
306        '--remote',
307        default='aosp',
308        help=('Remote name to fetch and check if the revision of VNDK snapshot '
309              'is included in the source to conform GPL license. default=aosp'))
310    parser.add_argument(
311        '-v',
312        '--verbose',
313        action='count',
314        default=0,
315        help='Increase output verbosity, e.g. "-v", "-vv".')
316    return parser.parse_args()
317
318
319def main():
320    """Program entry point."""
321    args = get_args()
322    run(args.vndk_version, args.branch, args.build, args.local,
323        args.use_current_branch, args.remote, args.verbose)
324
325
326if __name__ == '__main__':
327    main()
328