1#
2# Copyright (C) 2018 The Android Open Source Project
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""A commandline tool to check and update packages in external/
16
17Example usage:
18updater.sh checkall
19updater.sh update kotlinc
20updater.sh update --refresh --keep_date rust/crates/libc
21"""
22
23import argparse
24import enum
25import json
26import os
27import sys
28import subprocess
29import time
30from typing import Dict, Iterator, List, Union, Tuple, Type
31from pathlib import Path
32
33from base_updater import Updater
34from crates_updater import CratesUpdater
35from git_updater import GitUpdater
36from github_archive_updater import GithubArchiveUpdater
37import fileutils
38import git_utils
39# pylint: disable=import-error
40import metadata_pb2  # type: ignore
41import updater_utils
42
43UPDATERS: List[Type[Updater]] = [
44    CratesUpdater,
45    GithubArchiveUpdater,
46    GitUpdater,
47]
48
49TMP_BRANCH_NAME = 'tmp_auto_upgrade'
50USE_COLOR = sys.stdout.isatty()
51
52
53@enum.unique
54class Color(enum.Enum):
55    """Colors for output to console."""
56    FRESH = '\x1b[32m'
57    STALE = '\x1b[31;1m'
58    ERROR = '\x1b[31m'
59
60
61END_COLOR = '\033[0m'
62
63
64def color_string(string: str, color: Color) -> str:
65    """Changes the color of a string when print to terminal."""
66    if not USE_COLOR:
67        return string
68    return color.value + string + END_COLOR
69
70
71def build_updater(proj_path: Path) -> Tuple[Updater, metadata_pb2.MetaData]:
72    """Build updater for a project specified by proj_path.
73
74    Reads and parses METADATA file. And builds updater based on the information.
75
76    Args:
77      proj_path: Absolute or relative path to the project.
78
79    Returns:
80      The updater object built. None if there's any error.
81    """
82
83    proj_path = fileutils.get_absolute_project_path(proj_path)
84    metadata = fileutils.read_metadata(proj_path)
85    updater = updater_utils.create_updater(metadata, proj_path, UPDATERS)
86    return (updater, metadata)
87
88
89def _do_update(args: argparse.Namespace, updater: Updater,
90               metadata: metadata_pb2.MetaData) -> None:
91    full_path = updater.project_path
92
93    if args.branch_and_commit:
94        git_utils.checkout(full_path, args.remote_name + '/master')
95        git_utils.start_branch(full_path, TMP_BRANCH_NAME)
96
97    updater.update()
98
99    updated_metadata = metadata_pb2.MetaData()
100    updated_metadata.CopyFrom(metadata)
101    updated_metadata.third_party.version = updater.latest_version
102    for metadata_url in updated_metadata.third_party.url:
103        if metadata_url == updater.current_url:
104            metadata_url.CopyFrom(updater.latest_url)
105    # For Rust crates, replace GIT url with ARCHIVE url
106    if isinstance(updater, CratesUpdater):
107        updater.update_metadata(updated_metadata, full_path)
108    fileutils.write_metadata(full_path, updated_metadata, args.keep_date)
109    git_utils.add_file(full_path, 'METADATA')
110
111    if args.branch_and_commit:
112        msg = 'Upgrade {} to {}\n\nTest: make\n'.format(
113            args.path, updater.latest_version)
114        git_utils.add_file(full_path, '*')
115        git_utils.commit(full_path, msg)
116
117    if args.push_change:
118        git_utils.push(full_path, args.remote_name, updater.has_errors)
119
120    if args.branch_and_commit:
121        git_utils.checkout(full_path, args.remote_name + '/master')
122
123
124def check_and_update(args: argparse.Namespace,
125                     proj_path: Path,
126                     update_lib=False) -> Union[Updater, str]:
127    """Checks updates for a project. Prints result on console.
128
129    Args:
130      args: commandline arguments
131      proj_path: Absolute or relative path to the project.
132      update: If false, will only check for new version, but not update.
133    """
134
135    try:
136        rel_proj_path = fileutils.get_relative_project_path(proj_path)
137        print(f'Checking {rel_proj_path}. ', end='')
138        updater, metadata = build_updater(proj_path)
139        updater.check()
140
141        current_ver = updater.current_version
142        latest_ver = updater.latest_version
143        print('Current version: {}. Latest version: {}'.format(
144            current_ver, latest_ver),
145              end='')
146
147        has_new_version = current_ver != latest_ver
148        if has_new_version:
149            print(color_string(' Out of date!', Color.STALE))
150        else:
151            print(color_string(' Up to date.', Color.FRESH))
152
153        if update_lib and args.refresh:
154            print('Refreshing the current version')
155            updater.use_current_as_latest()
156        if update_lib and (has_new_version or args.force or args.refresh):
157            _do_update(args, updater, metadata)
158        return updater
159    # pylint: disable=broad-except
160    except Exception as err:
161        print('{} {}.'.format(color_string('Failed.', Color.ERROR), err))
162        return str(err)
163
164
165def _check_path(args: argparse.Namespace, paths: Iterator[str],
166                delay: int) -> Dict[str, Dict[str, str]]:
167    results = {}
168    for path in paths:
169        res = {}
170        updater = check_and_update(args, Path(path))
171        if isinstance(updater, str):
172            res['error'] = updater
173        else:
174            res['current'] = updater.current_version
175            res['latest'] = updater.latest_version
176        relative_path = fileutils.get_relative_project_path(Path(path))
177        results[str(relative_path)] = res
178        time.sleep(delay)
179    return results
180
181
182def _list_all_metadata() -> Iterator[str]:
183    for path, dirs, files in os.walk(fileutils.EXTERNAL_PATH):
184        if fileutils.METADATA_FILENAME in files:
185            # Skip sub directories.
186            dirs[:] = []
187            yield path
188        dirs.sort(key=lambda d: d.lower())
189
190
191def check(args: argparse.Namespace) -> None:
192    """Handler for check command."""
193    paths = _list_all_metadata() if args.all else args.paths
194    results = _check_path(args, paths, args.delay)
195
196    if args.json_output is not None:
197        with Path(args.json_output).open('w') as res_file:
198            json.dump(results, res_file, sort_keys=True, indent=4)
199
200
201def update(args: argparse.Namespace) -> None:
202    """Handler for update command."""
203    check_and_update(args, args.path, update_lib=True)
204
205
206def parse_args() -> argparse.Namespace:
207    """Parses commandline arguments."""
208
209    parser = argparse.ArgumentParser(
210        description='Check updates for third party projects in external/.')
211    subparsers = parser.add_subparsers(dest='cmd')
212    subparsers.required = True
213
214    # Creates parser for check command.
215    check_parser = subparsers.add_parser('check',
216                                         help='Check update for one project.')
217    check_parser.add_argument(
218        'paths',
219        nargs='*',
220        help='Paths of the project. '
221        'Relative paths will be resolved from external/.')
222    check_parser.add_argument('--json_output',
223                              help='Path of a json file to write result to.')
224    check_parser.add_argument(
225        '--all',
226        action='store_true',
227        help='If set, check updates for all supported projects.')
228    check_parser.add_argument(
229        '--delay',
230        default=0,
231        type=int,
232        help='Time in seconds to wait between checking two projects.')
233    check_parser.set_defaults(func=check)
234
235    # Creates parser for update command.
236    update_parser = subparsers.add_parser('update', help='Update one project.')
237    update_parser.add_argument(
238        'path',
239        help='Path of the project. '
240        'Relative paths will be resolved from external/.')
241    update_parser.add_argument(
242        '--force',
243        help='Run update even if there\'s no new version.',
244        action='store_true')
245    update_parser.add_argument(
246        '--refresh',
247        help='Run update and refresh to the current version.',
248        action='store_true')
249    update_parser.add_argument(
250        '--keep_date',
251        help='Run update and do not change date in METADATA.',
252        action='store_true')
253    update_parser.add_argument('--branch_and_commit',
254                               action='store_true',
255                               help='Starts a new branch and commit changes.')
256    update_parser.add_argument('--push_change',
257                               action='store_true',
258                               help='Pushes change to Gerrit.')
259    update_parser.add_argument('--remote_name',
260                               default='aosp',
261                               required=False,
262                               help='Upstream remote name.')
263    update_parser.set_defaults(func=update)
264
265    return parser.parse_args()
266
267
268def main() -> None:
269    """The main entry."""
270
271    args = parse_args()
272    args.func(args)
273
274
275if __name__ == '__main__':
276    main()
277