1#!/usr/bin/env python
2#
3# Copyright (C) 2018 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16"""Generate API lists for non-SDK API enforcement."""
17import argparse
18from collections import defaultdict, namedtuple
19import functools
20import os
21import re
22import sys
23
24# Names of flags recognized by the `hiddenapi` tool.
25FLAG_SDK = 'sdk'
26FLAG_UNSUPPORTED = 'unsupported'
27FLAG_BLOCKED = 'blocked'
28FLAG_MAX_TARGET_O = 'max-target-o'
29FLAG_MAX_TARGET_P = 'max-target-p'
30FLAG_MAX_TARGET_Q = 'max-target-q'
31FLAG_MAX_TARGET_R = 'max-target-r'
32FLAG_CORE_PLATFORM_API = 'core-platform-api'
33FLAG_PUBLIC_API = 'public-api'
34FLAG_SYSTEM_API = 'system-api'
35FLAG_TEST_API = 'test-api'
36
37# List of all known flags.
38FLAGS_API_LIST = [
39    FLAG_SDK,
40    FLAG_UNSUPPORTED,
41    FLAG_BLOCKED,
42    FLAG_MAX_TARGET_O,
43    FLAG_MAX_TARGET_P,
44    FLAG_MAX_TARGET_Q,
45    FLAG_MAX_TARGET_R,
46]
47ALL_FLAGS = FLAGS_API_LIST + [
48    FLAG_CORE_PLATFORM_API,
49    FLAG_PUBLIC_API,
50    FLAG_SYSTEM_API,
51    FLAG_TEST_API,
52]
53
54FLAGS_API_LIST_SET = set(FLAGS_API_LIST)
55ALL_FLAGS_SET = set(ALL_FLAGS)
56
57# Option specified after one of FLAGS_API_LIST to indicate that
58# only known and otherwise unassigned entries should be assign the
59# given flag.
60# For example, the max-target-P list is checked in as it was in P,
61# but signatures have changes since then. The flag instructs this
62# script to skip any entries which do not exist any more.
63FLAG_IGNORE_CONFLICTS = "ignore-conflicts"
64
65# Option specified after one of FLAGS_API_LIST to express that all
66# apis within a given set of packages should be assign the given flag.
67FLAG_PACKAGES = "packages"
68
69# Option specified after one of FLAGS_API_LIST to indicate an extra
70# tag that should be added to the matching APIs.
71FLAG_TAG = "tag"
72
73# Regex patterns of fields/methods used in serialization. These are
74# considered public API despite being hidden.
75SERIALIZATION_PATTERNS = [
76    r'readObject\(Ljava/io/ObjectInputStream;\)V',
77    r'readObjectNoData\(\)V',
78    r'readResolve\(\)Ljava/lang/Object;',
79    r'serialVersionUID:J',
80    r'serialPersistentFields:\[Ljava/io/ObjectStreamField;',
81    r'writeObject\(Ljava/io/ObjectOutputStream;\)V',
82    r'writeReplace\(\)Ljava/lang/Object;',
83]
84
85# Single regex used to match serialization API. It combines all the
86# SERIALIZATION_PATTERNS into a single regular expression.
87SERIALIZATION_REGEX = re.compile(r'.*->(' + '|'.join(SERIALIZATION_PATTERNS) + r')$')
88
89# Predicates to be used with filter_apis.
90HAS_NO_API_LIST_ASSIGNED = lambda api, flags: not FLAGS_API_LIST_SET.intersection(flags)
91IS_SERIALIZATION = lambda api, flags: SERIALIZATION_REGEX.match(api)
92
93
94class StoreOrderedOptions(argparse.Action):
95    """An argparse action that stores a number of option arguments in the order that
96    they were specified.
97    """
98    def __call__(self, parser, args, values, option_string = None):
99        items = getattr(args, self.dest, None)
100        if items is None:
101            items = []
102        items.append([option_string.lstrip('-'), values])
103        setattr(args, self.dest, items)
104
105def get_args():
106    """Parses command line arguments.
107
108    Returns:
109        Namespace: dictionary of parsed arguments
110    """
111    parser = argparse.ArgumentParser()
112    parser.add_argument('--output', required=True)
113    parser.add_argument('--csv', nargs='*', default=[], metavar='CSV_FILE',
114        help='CSV files to be merged into output')
115
116    for flag in ALL_FLAGS:
117        parser.add_argument('--' + flag, dest='ordered_flags', metavar='TXT_FILE',
118            action=StoreOrderedOptions, help='lists of entries with flag "' + flag + '"')
119    parser.add_argument('--' + FLAG_IGNORE_CONFLICTS, dest='ordered_flags', nargs=0,
120        action=StoreOrderedOptions, help='Indicates that only known and otherwise unassigned '
121        'entries should be assign the given flag. Must follow a list of entries and applies '
122        'to the preceding such list.')
123    parser.add_argument('--' + FLAG_PACKAGES, dest='ordered_flags', nargs=0,
124        action=StoreOrderedOptions, help='Indicates that the previous list of entries '
125        'is a list of packages. All members in those packages will be given the flag. '
126        'Must follow a list of entries and applies to the preceding such list.')
127    parser.add_argument('--' + FLAG_TAG, dest='ordered_flags', nargs=1,
128        action=StoreOrderedOptions, help='Adds an extra tag to the previous list of entries. '
129        'Must follow a list of entries and applies to the preceding such list.')
130
131    return parser.parse_args()
132
133
134def read_lines(filename):
135    """Reads entire file and return it as a list of lines.
136
137    Lines which begin with a hash are ignored.
138
139    Args:
140        filename (string): Path to the file to read from.
141
142    Returns:
143        Lines of the file as a list of string.
144    """
145    with open(filename, 'r') as f:
146        lines = f.readlines();
147    lines = filter(lambda line: not line.startswith('#'), lines)
148    lines = map(lambda line: line.strip(), lines)
149    return set(lines)
150
151
152def write_lines(filename, lines):
153    """Writes list of lines into a file, overwriting the file if it exists.
154
155    Args:
156        filename (string): Path to the file to be writing into.
157        lines (list): List of strings to write into the file.
158    """
159    lines = map(lambda line: line + '\n', lines)
160    with open(filename, 'w') as f:
161        f.writelines(lines)
162
163
164def extract_package(signature):
165    """Extracts the package from a signature.
166
167    Args:
168        signature (string): JNI signature of a method or field.
169
170    Returns:
171        The package name of the class containing the field/method.
172    """
173    full_class_name = signature.split(";->")[0]
174    # Example: Landroid/hardware/radio/V1_2/IRadio$Proxy
175    if (full_class_name[0] != "L"):
176        raise ValueError("Expected to start with 'L': %s" % full_class_name)
177    full_class_name = full_class_name[1:]
178    # If full_class_name doesn't contain '/', then package_name will be ''.
179    package_name = full_class_name.rpartition("/")[0]
180    return package_name.replace('/', '.')
181
182
183class FlagsDict:
184    def __init__(self):
185        self._dict_keyset = set()
186        self._dict = defaultdict(set)
187
188    def _check_entries_set(self, keys_subset, source):
189        assert isinstance(keys_subset, set)
190        assert keys_subset.issubset(self._dict_keyset), (
191            "Error: {} specifies signatures not present in code:\n"
192            "{}"
193            "Please visit go/hiddenapi for more information.").format(
194                source, "".join(map(lambda x: "  " + str(x) + "\n", keys_subset - self._dict_keyset)))
195
196    def _check_flags_set(self, flags_subset, source):
197        assert isinstance(flags_subset, set)
198        assert flags_subset.issubset(ALL_FLAGS_SET), (
199            "Error processing: {}\n"
200            "The following flags were not recognized: \n"
201            "{}\n"
202            "Please visit go/hiddenapi for more information.").format(
203                source, "\n".join(flags_subset - ALL_FLAGS_SET))
204
205    def filter_apis(self, filter_fn):
206        """Returns APIs which match a given predicate.
207
208        This is a helper function which allows to filter on both signatures (keys) and
209        flags (values). The built-in filter() invokes the lambda only with dict's keys.
210
211        Args:
212            filter_fn : Function which takes two arguments (signature/flags) and returns a boolean.
213
214        Returns:
215            A set of APIs which match the predicate.
216        """
217        return set(filter(lambda x: filter_fn(x, self._dict[x]), self._dict_keyset))
218
219    def get_valid_subset_of_unassigned_apis(self, api_subset):
220        """Sanitizes a key set input to only include keys which exist in the dictionary
221        and have not been assigned any API list flags.
222
223        Args:
224            entries_subset (set/list): Key set to be sanitized.
225
226        Returns:
227            Sanitized key set.
228        """
229        assert isinstance(api_subset, set)
230        return api_subset.intersection(self.filter_apis(HAS_NO_API_LIST_ASSIGNED))
231
232    def generate_csv(self):
233        """Constructs CSV entries from a dictionary.
234
235        Old versions of flags are used to generate the file.
236
237        Returns:
238            List of lines comprising a CSV file. See "parse_and_merge_csv" for format description.
239        """
240        lines = []
241        for api in self._dict:
242          flags = sorted(self._dict[api])
243          lines.append(",".join([api] + flags))
244        return sorted(lines)
245
246    def parse_and_merge_csv(self, csv_lines, source = "<unknown>"):
247        """Parses CSV entries and merges them into a given dictionary.
248
249        The expected CSV format is:
250            <api signature>,<flag1>,<flag2>,...,<flagN>
251
252        Args:
253            csv_lines (list of strings): Lines read from a CSV file.
254            source (string): Origin of `csv_lines`. Will be printed in error messages.
255
256        Throws:
257            AssertionError if parsed flags are invalid.
258        """
259        # Split CSV lines into arrays of values.
260        csv_values = [ line.split(',') for line in csv_lines ]
261
262        # Update the full set of API signatures.
263        self._dict_keyset.update([ csv[0] for csv in csv_values ])
264
265        # Check that all flags are known.
266        csv_flags = set()
267        for csv in csv_values:
268          csv_flags.update(csv[1:])
269        self._check_flags_set(csv_flags, source)
270
271        # Iterate over all CSV lines, find entry in dict and append flags to it.
272        for csv in csv_values:
273            flags = csv[1:]
274            if (FLAG_PUBLIC_API in flags) or (FLAG_SYSTEM_API in flags):
275                flags.append(FLAG_SDK)
276            self._dict[csv[0]].update(flags)
277
278    def assign_flag(self, flag, apis, source="<unknown>", tag = None):
279        """Assigns a flag to given subset of entries.
280
281        Args:
282            flag (string): One of ALL_FLAGS.
283            apis (set): Subset of APIs to receive the flag.
284            source (string): Origin of `entries_subset`. Will be printed in error messages.
285
286        Throws:
287            AssertionError if parsed API signatures of flags are invalid.
288        """
289        # Check that all APIs exist in the dict.
290        self._check_entries_set(apis, source)
291
292        # Check that the flag is known.
293        self._check_flags_set(set([ flag ]), source)
294
295        # Iterate over the API subset, find each entry in dict and assign the flag to it.
296        for api in apis:
297            self._dict[api].add(flag)
298            if tag:
299                self._dict[api].add(tag)
300
301
302FlagFile = namedtuple('FlagFile', ('flag', 'file', 'ignore_conflicts', 'packages', 'tag'))
303
304def parse_ordered_flags(ordered_flags):
305    r = []
306    currentflag, file, ignore_conflicts, packages, tag = None, None, False, False, None
307    for flag_value in ordered_flags:
308        flag, value = flag_value[0], flag_value[1]
309        if flag in ALL_FLAGS_SET:
310            if currentflag:
311                r.append(FlagFile(currentflag, file, ignore_conflicts, packages, tag))
312                ignore_conflicts, packages, tag = False, False, None
313            currentflag = flag
314            file = value
315        else:
316            if currentflag is None:
317                raise argparse.ArgumentError('--%s is only allowed after one of %s' % (
318                    flag, ' '.join(['--%s' % f for f in ALL_FLAGS_SET])))
319            if flag == FLAG_IGNORE_CONFLICTS:
320                ignore_conflicts = True
321            elif flag == FLAG_PACKAGES:
322                packages = True
323            elif flag == FLAG_TAG:
324                tag = value[0]
325
326
327    if currentflag:
328        r.append(FlagFile(currentflag, file, ignore_conflicts, packages, tag))
329    return r
330
331
332def main(argv):
333    # Parse arguments.
334    args = vars(get_args())
335    flagfiles = parse_ordered_flags(args['ordered_flags'] or [])
336
337    # Initialize API->flags dictionary.
338    flags = FlagsDict()
339
340    # Merge input CSV files into the dictionary.
341    # Do this first because CSV files produced by parsing API stubs will
342    # contain the full set of APIs. Subsequent additions from text files
343    # will be able to detect invalid entries, and/or filter all as-yet
344    # unassigned entries.
345    for filename in args["csv"]:
346        flags.parse_and_merge_csv(read_lines(filename), filename)
347
348    # Combine inputs which do not require any particular order.
349    # (1) Assign serialization API to SDK.
350    flags.assign_flag(FLAG_SDK, flags.filter_apis(IS_SERIALIZATION))
351
352    # (2) Merge text files with a known flag into the dictionary.
353    for info in flagfiles:
354        if (not info.ignore_conflicts) and (not info.packages):
355            flags.assign_flag(info.flag, read_lines(info.file), info.file, info.tag)
356
357    # Merge text files where conflicts should be ignored.
358    # This will only assign the given flag if:
359    # (a) the entry exists, and
360    # (b) it has not been assigned any other flag.
361    # Because of (b), this must run after all strict assignments have been performed.
362    for info in flagfiles:
363        if info.ignore_conflicts:
364            valid_entries = flags.get_valid_subset_of_unassigned_apis(read_lines(info.file))
365            flags.assign_flag(info.flag, valid_entries, filename, info.tag)
366
367    # All members in the specified packages will be assigned the appropriate flag.
368    for info in flagfiles:
369        if info.packages:
370            packages_needing_list = set(read_lines(info.file))
371            should_add_signature_to_list = lambda sig,lists: extract_package(
372                sig) in packages_needing_list and not lists
373            valid_entries = flags.filter_apis(should_add_signature_to_list)
374            flags.assign_flag(info.flag, valid_entries, info.file, info.tag)
375
376    # Mark all remaining entries as blocked.
377    flags.assign_flag(FLAG_BLOCKED, flags.filter_apis(HAS_NO_API_LIST_ASSIGNED))
378
379    # Write output.
380    write_lines(args["output"], flags.generate_csv())
381
382if __name__ == "__main__":
383    main(sys.argv)
384