1#!/usr/bin/python
2#
3# Copyright (C) 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17
18"""
19Merges upstream files to ojluni. This is done by using git to perform a 3-way
20merge between the current (base) upstream version, ojluni and the new (target)
21upstream version. The 3-way merge is needed because ojluni sometimes contains
22some Android-specific changes from the upstream version.
23
24This tool is for libcore maintenance; if you're not maintaining libcore,
25you won't need it (and might not have access to some of the instructions
26below).
27
28The naming of the repositories (expected, ojluni, 7u40, 8u121-b13,
299b113+, 9+181) is based on the directory name where corresponding
30snapshots are stored when following the instructions at
31http://go/libcore-o-verify
32
33This script tries to preserve Android changes to upstream code when moving to a
34newer version.
35
36All the work is made in a new directory which is initialized as a git
37repository. An example of the repository structure, where an update is made
38from version 9b113+ to 11+28, would be:
39
40    *   5593705 (HEAD -> main) Merge branch 'ojluni'
41    |\
42    | * 2effe03 (ojluni) Ojluni commit
43    * | 1bef5f3 Target commit (11+28)
44    |/
45    * 9ae2fbf Base commit (9b113+)
46
47The conflicts during the merge get resolved by git whenever possible. However,
48sometimes there are conflicts that need to be resolved manually. If that is the
49case, the script will terminate to allow for the resolving. Once the user has
50resolved the conflicts, they should rerun the script with the --continue
51option.
52
53Once the merge is complete, the script will copy the merged version back to
54ojluni within the $ANDROID_BUILD_TOP location.
55
56For the script to run correctly, it needs the following environment variables
57defined:
58    - OJLUNI_UPSTREAMS
59    - ANDROID_BUILD_TOP
60
61Possible uses:
62
63To merge in changes from a newer version of the upstream using a default
64working dir created in /tmp:
65    merge-from-upstream -f expected -t 11+28 java/util/concurrent
66
67To merge in changes from a newer version of the upstream using a custom
68working dir:
69    merge-from-upstream -f expected -t 11+28 \
70            -d $HOME/tmp/ojluni-merge java/util/concurrent
71
72To merge in changes for a single file:
73    merge-from-upstream -f 9b113+ -t 11+28 \
74        java/util/concurrent/atomic/AtomicInteger.java
75
76To merge in changes, using a custom folder, that require conflict resolution:
77    merge-from-upstream -f expected -t 11+28 \
78        -d $HOME/tmp/ojluni-merge \
79        java/util/concurrent
80    <manually resolve conflicts and add them to git staging>
81    merge-from-upstream --continue \
82        -d $HOME/tmp/ojluni-merge java/util/concurrent
83"""
84
85import argparse
86import os
87import os.path
88import subprocess
89import sys
90import shutil
91
92
93def printerr(msg):
94    sys.stderr.write(msg + "\r\n")
95
96
97def user_check(msg):
98    choice = str(input(msg + " [y/N] ")).strip().lower()
99    if choice[:1] == 'y':
100        return True
101    return False
102
103
104def check_env_vars():
105    keys = [
106        'OJLUNI_UPSTREAMS',
107        'ANDROID_BUILD_TOP',
108    ]
109    result = True
110    for key in keys:
111        if key not in os.environ:
112            printerr("Unable to run, you must have {} defined".format(key))
113            result = False
114    return result
115
116
117def get_upstream_path(version, rel_path):
118    upstreams = os.environ['OJLUNI_UPSTREAMS']
119    return '{}/{}/{}'.format(upstreams, version, rel_path)
120
121
122def get_ojluni_path(rel_path):
123    android_build_top = os.environ['ANDROID_BUILD_TOP']
124    return '{}/libcore/ojluni/src/main/java/{}'.format(
125        android_build_top, rel_path)
126
127
128def make_copy(src, dst):
129    print("Copy " + src + " -> " + dst)
130    if os.path.isfile(src):
131        if os.path.exists(dst) and os.path.isfile(dst):
132            os.remove(dst)
133        shutil.copy(src, dst)
134    else:
135        shutil.copytree(src, dst, dirs_exist_ok=True)
136
137
138class Repo:
139    def __init__(self, dir):
140        self.dir = dir
141
142    def init(self):
143        if 0 != subprocess.call(['git', 'init', '-b', 'main', self.dir]):
144            raise RuntimeError(
145                "Unable to initialize working git repository.")
146        subprocess.call(['git', '-C', self.dir,
147                         'config', 'rerere.enabled', 'true'])
148
149    def commit_all(self, id, msg):
150        if 0 != subprocess.call(['git', '-C', self.dir, 'add', '*']):
151            raise RuntimeError("Unable to add the {} files.".format(id))
152        if 0 != subprocess.call(['git', '-C', self.dir, 'commit',
153                                '-m', msg]):
154            raise RuntimeError("Unable to commit the {} files.".format(id))
155
156    def checkout_branch(self, branch, is_new=False):
157        cmd = ['git', '-C', self.dir, 'checkout']
158        if is_new:
159            cmd.append('-b')
160        cmd.append(branch)
161        if 0 != subprocess.call(cmd):
162            raise RuntimeError("Unable to checkout the {} branch."
163                               .format(branch))
164
165    def merge(self, branch):
166        """
167        Tries to merge in a branch and returns True if the merge commit has
168        been created. If there are conflicts to be resolved, this returns
169        False.
170        """
171        if 0 == subprocess.call(['git', '-C', self.dir,
172                                'merge', branch, '--no-edit']):
173            return True
174        if not self.is_merging():
175            raise RuntimeError("Unable to run merge for the {} branch."
176                               .format(branch))
177        subprocess.call(['git', '-C', self.dir, 'rerere'])
178        return False
179
180    def check_resolved_from_cache(self):
181        """
182        Checks if some conflicts have been resolved by the git rerere tool. The
183        tool only applies the previous resolution, but does not mark the file
184        as resolved afterwards. Therefore this function will go through the
185        unresolved files and see if there are outstanding conflicts. If all
186        conflicts have been resolved, the file gets stages.
187
188        Returns True if all conflicts are resolved, False otherwise.
189        """
190        # git diff --check will exit with error if there are conflicts to be
191        # resolved, therefore we need to use check=False option to avoid an
192        # exception to be raised
193        conflict_markers = subprocess.run(['git', '-C', self.dir,
194                                           'diff', '--check'],
195                                          stdout=subprocess.PIPE,
196                                          check=False).stdout
197        conflicts = subprocess.check_output(['git', '-C', self.dir, 'diff',
198                                             '--name-only', '--diff-filter=U'])
199
200        for filename in conflicts.splitlines():
201            if conflict_markers.find(filename) != -1:
202                print("{} still has conflicts, please resolve manually".
203                      format(filename))
204            else:
205                print("{} has been resolved, staging it".format(filename))
206                subprocess.call(['git', '-C', self.dir, 'add', filename])
207
208        return not self.has_conflicts()
209
210    def has_changes(self):
211        result = subprocess.check_output(['git', '-C', self.dir, 'status',
212                                          '--porcelain'])
213        return len(result) != 0
214
215    def has_conflicts(self):
216        conflicts = subprocess.check_output(['git', '-C', self.dir, 'diff',
217                                             '--name-only', '--diff-filter=U'])
218        return len(conflicts) != 0
219
220    def is_merging(self):
221        return 0 == subprocess.call(['git', '-C', self.dir, 'rev-parse',
222                                     '-q', '--verify', 'MERGE_HEAD'],
223                                    stdout=subprocess.DEVNULL)
224
225    def complete_merge(self):
226        print("Completing merge in {}".format(self.dir))
227        subprocess.call(['git', '-C', self.dir, 'rerere'])
228        if 0 != subprocess.call(['git', '-C', self.dir,
229                                 'commit', '--no-edit']):
230            raise RuntimeError("Unable to complete the merge in {}."
231                               .format(self.dir))
232        if self.is_merging():
233            raise RuntimeError(
234                "Merging in {} is not complete".format(self.dir))
235
236    def load_resolve_files(self, resolve_dir):
237        print("Loading resolve files from {}".format(resolve_dir))
238        if not os.path.lexists(resolve_dir):
239            print("Resolve dir {} not found, no resolutions will be used"
240                  .format(resolve_dir))
241            return
242        make_copy(resolve_dir, self.dir + "/.git/rr-cache")
243
244    def save_resolve_files(self, resolve_dir):
245        print("Saving resolve files to {}".format(resolve_dir))
246        if not os.path.lexists(resolve_dir):
247            os.makedirs(resolve_dir)
248        make_copy(self.dir + "/.git/rr-cache", resolve_dir)
249
250
251class Merger:
252    def __init__(self, repo_dir, rel_path, resolve_dir):
253        self.repo = Repo(repo_dir)
254        # Have all the source files copied inside a src dir, so we don't have
255        # any issue with copying back the .git dir
256        self.working_dir = repo_dir + "/src"
257        self.rel_path = rel_path
258        self.resolve_dir = resolve_dir
259
260    def create_working_dir(self):
261        if os.path.lexists(self.repo.dir):
262            if not user_check(
263                    '{} already exists. Can it be removed?'
264                    .format(self.repo.dir)):
265                raise RuntimeError(
266                    'Will not remove {}. Consider using another '
267                    'working dir'.format(self.repo.dir))
268            try:
269                shutil.rmtree(self.repo.dir)
270            except OSError:
271                printerr("Unable to delete {}.".format(self.repo.dir))
272                raise
273        os.makedirs(self.working_dir)
274        self.repo.init()
275        if self.resolve_dir is not None:
276            self.repo.load_resolve_files(self.resolve_dir)
277
278    def copy_upstream_files(self, version, msg):
279        full_path = get_upstream_path(version, self.rel_path)
280        make_copy(full_path, self.working_dir)
281        self.repo.commit_all(version, msg)
282
283    def copy_base_files(self, base_version):
284        self.copy_upstream_files(base_version,
285                                 'Base commit ({})'.format(base_version))
286
287    def copy_target_files(self, target_version):
288        self.copy_upstream_files(target_version,
289                                 'Target commit ({})'.format(target_version))
290
291    def copy_ojluni_files(self):
292        full_path = get_ojluni_path(self.rel_path)
293        make_copy(full_path, self.working_dir)
294        if self.repo.has_changes():
295            self.repo.commit_all('ojluni', 'Ojluni commit')
296            return True
297        else:
298            return False
299
300    def run_ojluni_merge(self):
301        if self.repo.merge('ojluni'):
302            return
303        if self.repo.check_resolved_from_cache():
304            self.repo.complete_merge()
305            return
306        raise RuntimeError('\r\nThere are conflicts to be resolved.'
307                           '\r\nManually merge the changes and rerun '
308                           'this script with --continue')
309
310    def copy_back_to_ojluni(self):
311        # Save any resolutions that were made for future reuse
312        if self.resolve_dir is not None:
313            self.repo.save_resolve_files(self.resolve_dir)
314
315        src_path = self.working_dir
316        dst_path = get_ojluni_path(self.rel_path)
317        if os.path.isfile(dst_path):
318            src_path += '/' + os.path.basename(self.rel_path)
319        make_copy(src_path, dst_path)
320
321    def run(self, base_version, target_version):
322        print("Merging {} from {} into ojluni (based on {}). "
323              "Using {} as working dir."
324              .format(self.rel_path, target_version,
325                      base_version, self.repo.dir))
326        self.create_working_dir()
327        self.copy_base_files(base_version)
328        # The ojluni code should be added in its own branch. This is to make
329        # Git perform the 3-way merge once a commit is added with the latest
330        # upstream code.
331        self.repo.checkout_branch('ojluni', is_new=True)
332        merge_needed = self.copy_ojluni_files()
333        self.repo.checkout_branch('main')
334        self.copy_target_files(target_version)
335        if merge_needed:
336            # Runs the merge in the working directory, if some conflicts need
337            # to be resolved manually, then an exception is raised which will
338            # terminate the script, informing the user that manual intervention
339            # is needed.
340            self.run_ojluni_merge()
341        else:
342            print("No merging needed as there were no "
343                  "Android-specific changes, forwarding to new version ({})"
344                  .format(target_version))
345        self.copy_back_to_ojluni()
346
347    def complete_existing_run(self):
348        if self.repo.is_merging():
349            self.repo.complete_merge()
350        self.copy_back_to_ojluni()
351
352
353def main():
354    if not check_env_vars():
355        return
356
357    upstreams = os.environ['OJLUNI_UPSTREAMS']
358    repositories = sorted(
359        [d for d in os.listdir(upstreams)
360         if os.path.isdir(os.path.join(upstreams, d))]
361    )
362
363    parser = argparse.ArgumentParser(
364        description='''
365        Merge upstream files from ${OJLUNI_UPSTREAMS} to libcore/ojluni.
366        Needs the base (from) repository as well as the target (to) repository.
367        Repositories can be chosen from:
368        ''' + ' '.join(repositories) + '.',
369        # include default values in help
370        formatter_class=argparse.ArgumentDefaultsHelpFormatter,
371    )
372    parser.add_argument('-f', '--from', default='expected',
373                        choices=repositories,
374                        dest='base',
375                        help='Repository on which the requested ojluni '
376                        'files are based.')
377    parser.add_argument('-t', '--to',
378                        choices=repositories,
379                        dest='target',
380                        help='Repository to which the requested ojluni '
381                        'files will be updated.')
382    parser.add_argument('-d', '--work-dir', default='/tmp/ojluni-merge',
383                        help='Path where the merge will be performed. '
384                        'Any existing files in the path will be removed')
385    parser.add_argument('-r', '--resolve-dir', default=None,
386                        dest='resolve_dir',
387                        help='Path where the git resolutions are cached. '
388                        'By default, no cache is used.')
389    parser.add_argument('--continue', action='store_true', dest='proceed',
390                        help='Flag to specify after merge conflicts '
391                        'are resolved')
392    parser.add_argument('rel_path', nargs=1, metavar='<relative_path>',
393                        help='File to merge: a relative path below '
394                        'libcore/ojluni/ which could point to '
395                        'a file or folder.')
396    args = parser.parse_args()
397    try:
398        merger = Merger(args.work_dir, args.rel_path[0], args.resolve_dir)
399        if args.proceed:
400            merger.complete_existing_run()
401        else:
402            if args.target is None:
403                raise RuntimeError('Please specify the target upstream '
404                                   'version using the -t/--to argument')
405            merger.run(args.base, args.target)
406    except Exception as e:
407        printerr(str(e))
408
409
410if __name__ == "__main__":
411    main()
412