1#!/usr/bin/python
2#
3# Copyright (C) 2021 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17"""
18Reports on merge status of Java files in a package based on four
19repositories:
20
21baseline - upstream baseline used for previous Android release
22release  - files in previous Android release
23current  - target for merge
24upstream - new upstream being merged
25
26Example output:
27$ tools/upstream/pkg-status java.security.spec
28AlgorithmParameterSpec.java: Unchanged, Done
29DSAGenParameterSpec.java: Added, TO DO
30DSAParameterSpec.java: Unchanged, Done
31DSAPrivateKeySpec.java: Unchanged, Done
32DSAPublicKeySpec.java: Unchanged, Done
33ECField.java: Unchanged, Done
34ECFieldF2m.java: Unchanged, Done
35ECFieldFp.java: Unchanged, Done
36ECGenParameterSpec.java: Updated, TO DO
37[...]
38"""
39
40import argparse
41import hashlib
42import os
43import os.path
44import sys
45from enum import Enum
46from pathlib import Path
47
48RED = '\u001b[31m'
49GREEN = "\u001b[32m"
50YELLOW = "\u001b[33m"
51RESET = "\u001b[0m"
52
53
54def colourise(colour, string):
55    """Wrap a string with an ANSI colour code"""
56    return "%s%s%s" % (colour, string, RESET)
57
58
59def red(string):
60    """Wrap a string with a red ANSI colour code"""
61    return colourise(RED, string)
62
63
64def green(string):
65    """Wrap a string with a green ANSI colour code"""
66    return colourise(GREEN, string)
67
68
69def yellow(string):
70    """Wrap a string with a yellow ANSI colour code"""
71    return colourise(YELLOW, string)
72
73
74class WorkStatus(Enum):
75    """Enum for a file's work completion status"""
76    UNKNOWN = ('Unknown', red)
77    TODO = ('TO DO', yellow)
78    DONE = ('Done', green)
79    PROBABLY_DONE = ('Probably done', green)
80    ERROR = ('Error', red)
81
82    def colourise(self, string):
83        """Colourise a string using the method for this enum value"""
84        return self.colourfunc(string)
85
86    def __init__(self, description, colourfunc):
87        self.description = description
88        self.colourfunc = colourfunc
89
90
91class MergeStatus(Enum):
92    """Enum for a file's merge status"""
93    UNKNOWN = 'Unknown!'
94    MISSING = 'Missing'
95    ADDED = 'Added'
96    DELETED = 'Deleted or moved'
97    UNCHANGED = 'Unchanged'
98    UPDATED = 'Updated'
99
100    def __init__(self, description):
101        self.description = description
102
103
104class MergeConfig:
105    """
106    Configuration for an upstream merge.
107
108    Encapsulates the paths to each of the required code repositories.
109    """
110    def __init__(self, baseline, release, current, upstream) -> None:
111        self.baseline = baseline
112        self.release = release
113        self.current = current
114        self.upstream = upstream
115        try:
116            # Root of checked-out Android sources, set by the "lunch" command.
117            self.android_build_top = os.environ['ANDROID_BUILD_TOP']
118            # Root of repository snapshots.
119            self.ojluni_upstreams = os.environ['OJLUNI_UPSTREAMS']
120        except KeyError:
121            sys.exit('`lunch` and set OJLUNI_UPSTREAMS first.')
122
123
124    def java_dir(self, repo, pkg):
125        relpath = pkg.replace('.', '/')
126        if repo == self.current:
127            return '%s/libcore/%s/src/main/java/%s' % (
128                    self.android_build_top, self.current, relpath)
129        else:
130            return '%s/%s/%s' % (self.ojluni_upstreams, repo, relpath)
131
132    def baseline_dir(self, pkg):
133        return self.java_dir(self.baseline, pkg)
134
135    def release_dir(self, pkg):
136        return self.java_dir(self.release, pkg)
137
138    def current_dir(self, pkg):
139        return self.java_dir(self.current, pkg)
140
141    def upstream_dir(self, pkg):
142        return self.java_dir(self.upstream, pkg)
143
144
145class JavaPackage:
146    """
147    Encapsulates information about a single Java package, notably paths
148    to it within each repository.
149    """
150    def __init__(self, config, name) -> None:
151        self.name = name
152        self.baseline_dir = config.baseline_dir(name)
153        self.release_dir = config.release_dir(name)
154        self.current_dir = config.current_dir(name)
155        self.upstream_dir = config.upstream_dir(name)
156
157    @staticmethod
158    def list_candidate_files(path):
159        """Returns a list of all the Java filenames in a directory."""
160        return list(filter(
161                lambda f: f.endswith('.java') and f != 'package-info.java',
162                os.listdir(path)))
163
164    def all_files(self):
165        """Returns the union of all the Java filenames in all repositories."""
166        files = set(self.list_candidate_files(self.baseline_dir))
167        files.update(self.list_candidate_files(self.release_dir))
168        files.update(self.list_candidate_files(self.upstream_dir))
169        files.update(self.list_candidate_files(self.current_dir))
170        return sorted(list(files))
171
172    def java_files(self):
173        """Returns a list of JavaFiles corresponding to all filenames."""
174        return map(lambda f: JavaFile(self, f), self.all_files())
175
176    def baseline_path(self, filename):
177        return Path(self.baseline_dir + '/' + filename)
178
179    def release_path(self, filename):
180        return Path(self.release_dir + '/' + filename)
181
182    def current_path(self, filename):
183        return Path(self.current_dir + '/' + filename)
184
185    def upstream_path(self, filename):
186        return Path(self.upstream_dir + '/' + filename)
187
188    def report_merge_status(self):
189        """Report on the mergse status of this package."""
190        for file in self.java_files():
191            merge_status, work_status = file.status()
192            text = '%s: %s, %s' % \
193                   (
194                           file.name, merge_status.description,
195                           work_status.description)
196            print(work_status.colourise(text))
197            if work_status == WorkStatus.ERROR:
198                print(file.baseline_sum, file.baseline)
199                print(file.release_sum, file.release)
200                print(file.current_sum, file.current)
201                print(file.upstream_sum, file.upstream)
202
203
204class JavaFile:
205    """
206    Encapsulates information about a single Java file in a package across
207    all of the repositories involved in a merge.
208    """
209    def __init__(self, package, name):
210        self.package = package
211        self.name = name
212        # Paths for this file in each repository
213        self.baseline = package.baseline_path(name)
214        self.release = package.release_path(name)
215        self.upstream = package.upstream_path(name)
216        self.current = package.current_path(name)
217        # Checksums for this file in each repository, or None if absent
218        self.baseline_sum = self.checksum(self.baseline)
219        self.release_sum = self.checksum(self.release)
220        self.upstream_sum = self.checksum(self.upstream)
221        self.current_sum = self.checksum(self.current)
222        # List of methods for determining file's merge status.
223        # Order matters - see merge_status() for details
224        self.merge_status_methods = [
225                (self.check_for_missing, MergeStatus.MISSING),
226                (self.check_for_unchanged, MergeStatus.UNCHANGED),
227                (self.check_for_added_upstream, MergeStatus.ADDED),
228                (self.check_for_removed_upstream, MergeStatus.DELETED),
229                (self.check_for_changed_upstream, MergeStatus.UPDATED),
230        ]
231        # Map of methods from merge status to determine work status
232        self.work_status_methods = {
233                MergeStatus.MISSING: self.calculate_missing_work_status,
234                MergeStatus.UNCHANGED: self.calculate_unchanged_work_status,
235                MergeStatus.ADDED: self.calculate_added_work_status,
236                MergeStatus.DELETED: self.calculate_deleted_work_status,
237                MergeStatus.UPDATED: self.calculate_updated_work_status,
238        }
239
240    def is_android_changed(self):
241        """
242        Returns true if the file was changed between the baseline and Android
243        release.
244        """
245        return self.is_in_release() and self.baseline_sum != self.release_sum
246
247    def is_android_unchanged(self):
248        """
249        Returns true if the file is in the Android release and is unchanged.
250        """
251        return self.is_in_release() and self.baseline_sum == self.release_sum
252
253    def check_for_changed_upstream(self):
254        """Returns true if the file is changed upstream since the baseline."""
255        return self.baseline_sum != self.upstream_sum
256
257    def is_in_baseline(self):
258        return self.baseline_sum is not None
259
260    def is_in_release(self):
261        """Returns true if the file is present in the baseline and release."""
262        return self.is_in_baseline() and self.release_sum is not None
263
264    def is_in_current(self):
265        """Returns true if the file is in current, release and baseline."""
266        return self.is_in_release() and self.current_sum is not None
267
268    def is_in_upstream(self):
269        return self.upstream_sum is not None
270
271    def check_for_missing(self):
272        """
273        Returns true if the file is expected to be in current, but isn't.
274        """
275        return self.is_in_release() and self.is_in_upstream() \
276               and not self.is_in_current()
277
278    def removed_in_release(self):
279        """Returns true if the file was removed by Android in the release."""
280        return self.is_in_baseline() and not self.is_in_release()
281
282    def check_for_removed_upstream(self):
283        """Returns true if the file was removed upstream since the baseline."""
284        return self.is_in_baseline() and not self.is_in_upstream()
285
286    def check_for_added_upstream(self):
287        """Returns true if the file was added upstream since the baseline."""
288        return self.is_in_upstream() and not self.is_in_baseline()
289
290    def check_for_unchanged(self):
291        """Returns true if the file is unchanged upstream since the baseline."""
292        return not self.check_for_changed_upstream()
293
294    def merge_status(self):
295        """
296        Returns the merge status for this file, or UNKNOWN.
297        Tries each merge_status_method in turn, and if one returns true
298        then this method returns the associated merge status.
299        """
300        for (method, status) in self.merge_status_methods:
301            if method():
302                return status
303        return MergeStatus.UNKNOWN
304
305    def work_status(self):
306        """
307        Returns the work status for this file.
308        Looks up a status method based on the merge statis and uses that to
309        determine the work status.
310        """
311        status = self.merge_status()
312        if status in self.work_status_methods:
313            return self.work_status_methods[status]()
314        return WorkStatus.ERROR
315
316    @staticmethod
317    def calculate_missing_work_status():
318        """Missing files are always an error."""
319        return WorkStatus.ERROR
320
321    def calculate_unchanged_work_status(self):
322        """
323        File is unchanged upstream, so should be unchanged between release and
324        current.
325        """
326        if self.current_sum == self.release_sum:
327            return WorkStatus.DONE
328        return WorkStatus.UNKNOWN
329
330    def calculate_added_work_status(self):
331        """File was added upstream so needs to be added to current."""
332        if self.current_sum is None:
333            return WorkStatus.TODO
334        if self.current_sum == self.upstream_sum:
335            return WorkStatus.DONE
336        #     XXX check for change markers if android changed
337        return WorkStatus.UNKNOWN
338
339    def calculate_deleted_work_status(self):
340        """File was removed upstream so needs to be removed from current."""
341        if self.is_in_current():
342            return WorkStatus.TODO
343        return WorkStatus.DONE
344
345    def calculate_updated_work_status(self):
346        """File was updated upstream."""
347        if self.current_sum == self.upstream_sum:
348            if self.is_android_unchanged():
349                return WorkStatus.DONE
350            # Looks like Android changes are missing in current
351            return WorkStatus.ERROR
352        if self.is_android_unchanged():
353            return WorkStatus.TODO
354        # If we get here there are upstream and Android changes that need
355        # to be merged,  If possible use the file copyright date as a
356        # heuristic to determine if upstream has been merged into current
357        release_copyright = self.get_copyright(self.release)
358        current_copyright = self.get_copyright(self.current)
359        upstream_copyright = self.get_copyright(self.upstream)
360        if release_copyright == upstream_copyright:
361            # Upstream copyright same as last release, so can't infer anything
362            return WorkStatus.UNKNOWN
363        if current_copyright == upstream_copyright:
364            return WorkStatus.PROBABLY_DONE
365        if current_copyright == release_copyright:
366            return WorkStatus.TODO
367        # Give up
368        return WorkStatus.UNKNOWN
369
370    def status(self):
371        return self.merge_status(), self.work_status()
372
373    @staticmethod
374    def checksum(path):
375        """Returns a checksum string for a file, SHA256 as a hex string."""
376        try:
377            with open(path, 'rb') as file:
378                bytes = file.read()
379                return hashlib.sha256(bytes).hexdigest()
380        except:
381            return None
382
383    @staticmethod
384    def get_copyright(file):
385        """Returns the upstream copyright line for a file."""
386        try:
387            with open(file, 'r') as file:
388                for count in range(5):
389                    line = file.readline()
390                    if line.startswith(
391                            ' * Copyright') and 'Android' not in line:
392                        return line
393                return None
394        except:
395            return None
396
397
398def main():
399    parser = argparse.ArgumentParser(
400            description='Report on merge status of Java packages',
401            formatter_class=argparse.ArgumentDefaultsHelpFormatter)
402
403    # TODO(prb): Add help for available repositories
404    parser.add_argument('-b', '--baseline', default='expected',
405                        help='Baseline repo')
406    parser.add_argument('-r', '--release', default='sc-release',
407                        help='Last released repo')
408    parser.add_argument('-u', '--upstream', default='11+28',
409                        help='Upstream repo.')
410    parser.add_argument('-c', '--current', default='ojluni',
411                        help='Current repo.')
412    parser.add_argument('pkgs', nargs="+",
413                        help='Packages to report on')
414
415    args = parser.parse_args()
416    config = MergeConfig(args.baseline, args.release, args.current,
417                         args.upstream)
418
419    for pkg_name in args.pkgs:
420        try:
421            package = JavaPackage(config, pkg_name)
422            package.report_merge_status()
423        except Exception as e:
424            print(red("ERROR: Unable to process package " + pkg_name + e))
425
426
427if __name__ == "__main__":
428    main()
429