1#!/usr/bin/env python
2#
3# Copyright (C) 2018 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#            http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17"""Generate ICU stable C API wrapper source.
18
19
20This script parses all the header files specified by the ICU module names. For
21each function in the allowlist, it generates the NDK headers, and shim functions
22to shim.cpp, which in turn calls the real implementation at runtime.
23The tool relies on libclang to parse header files.
24
25Reference to ICU4C stable C APIs:
26http://icu-project.org/apiref/icu4c/files.html
27"""
28from __future__ import absolute_import
29from __future__ import print_function
30
31import logging
32import os
33import re
34import shutil
35import subprocess
36
37from genutil import (
38    android_path,
39    generate_shim,
40    generate_symbol_txt,
41    get_allowlisted_apis,
42    AllowlistedDeclarationFilter,
43    DeclaredFunctionsParser,
44    StableDeclarationFilter,
45)
46
47# No suffix for ndk shim
48SYMBOL_SUFFIX = ''
49
50SECRET_PROCESSING_TOKEN = "@@@SECRET@@@"
51
52DOC_BLOCK_COMMENT = r"\/\*\*(?:\*(?!\/)|[^*])*\*\/[ ]*\n"
53TILL_CLOSE_PARENTHESIS = r"[^)^;]*\)"
54STABLE_MACRO = r"(?:U_STABLE|U_CAPI)"
55STABLE_FUNCTION_DECLARATION = r"^(" + DOC_BLOCK_COMMENT + STABLE_MACRO \
56                              + TILL_CLOSE_PARENTHESIS + ");$"
57NONSTABLE_FUNCTION_DECLARATION = r"^(" + DOC_BLOCK_COMMENT + r"(U_INTERNAL|U_DEPRECATED|U_DRAFT)" \
58                                 + TILL_CLOSE_PARENTHESIS + ");$"
59
60REGEX_STABLE_FUNCTION_DECLARATION = re.compile(STABLE_FUNCTION_DECLARATION, re.MULTILINE)
61REGEX_NONSTABLE_FUNCTION_DECLARATION = re.compile(NONSTABLE_FUNCTION_DECLARATION, re.MULTILINE)
62
63def get_allowlisted_regex_string(decl_names):
64    """Return a regex in string to capture the C function declarations in the decl_names list"""
65    tag = "|".join(decl_names)
66    return r"(" + DOC_BLOCK_COMMENT + STABLE_MACRO + r"[^(]*(?=" + tag + r")(" + tag + ")" \
67           + r"\("+ TILL_CLOSE_PARENTHESIS +");$"
68
69def get_replacement_adding_api_level_macro(api_level):
70    """Return the replacement string adding the NDK C macro
71    guarding C function declaration by the api_level"""
72    return r"\1 __INTRODUCED_IN({0});\n\n".format(api_level)
73
74def modify_func_declarations(src_path, dst_path, decl_names):
75    """Process the source file,
76    remove the C function declarations not in the decl_names,
77    add guard the functions listed in decl_names by the API level,
78    and output to the dst_path """
79    allowlist_regex_string = get_allowlisted_regex_string(decl_names)
80    allowlist_decl_regex = re.compile('^' + allowlist_regex_string, re.MULTILINE)
81    secret_allowlist_decl_regex = re.compile('^' + SECRET_PROCESSING_TOKEN
82                                             + allowlist_regex_string, re.MULTILINE)
83    with open(src_path, "r") as file:
84        src = file.read()
85
86    # Remove all non-stable function declarations
87    modified = REGEX_NONSTABLE_FUNCTION_DECLARATION.sub('', src)
88
89    # Insert intermediate token to all functions in the allowlist
90    if decl_names:
91        modified = allowlist_decl_regex.sub(SECRET_PROCESSING_TOKEN + r"\1;", modified)
92    # Remove all other stable declarations not in the allowlist
93    modified = REGEX_STABLE_FUNCTION_DECLARATION.sub('', modified)
94    # Insert C macro and annotation to indicate the API level to each functions in the allowlist
95    modified = secret_allowlist_decl_regex.sub(
96        get_replacement_adding_api_level_macro(31), modified)
97
98    with open(dst_path, "w") as out:
99        out.write(modified)
100def remove_ignored_includes(file_path, include_list):
101    """
102    Remove the included header, i.e. #include lines, listed in include_list from the file_path
103    header.
104    """
105
106    # Do nothing if the list is empty
107    if not include_list:
108        return
109
110    tag = "|".join(include_list)
111
112    with open(file_path, "r") as file:
113        content = file.read()
114
115    regex = re.compile(r"^#include \"unicode\/(" + tag + ")\"\n", re.MULTILINE)
116    content = regex.sub('', content)
117
118    with open(file_path, "w") as out:
119        out.write(content)
120
121def copy_header_only_files():
122    """Copy required header only files"""
123    base_src_path = android_path('external/icu/icu4c/source/')
124    base_dest_path = android_path('external/icu/libicu/ndk_headers/unicode/')
125    with open(android_path('external/icu/tools/icu4c_srcgen/libicu_required_header_only_files.txt'),
126              'r') as in_file:
127        header_only_files = [
128            base_src_path + line.strip() for line in in_file.readlines() if not line.startswith('#')
129        ]
130
131    for src_path in header_only_files:
132        dest_path = base_dest_path + os.path.basename(src_path)
133        cmd = ['sed',
134               "s/U_SHOW_CPLUSPLUS_API/LIBICU_U_SHOW_CPLUSPLUS_API/g",
135               src_path
136               ]
137
138        with open(dest_path, "w") as destfile:
139            subprocess.check_call(cmd, stdout=destfile)
140
141def copy_cts_headers():
142    """Copy headers from common/ and i18n/ to cts_headers/ for compiling cintltst as CTS."""
143    dst_folder = android_path('external/icu/libicu/cts_headers')
144    if os.path.exists(dst_folder):
145        shutil.rmtree(dst_folder)
146    os.mkdir(dst_folder)
147    os.mkdir(os.path.join(dst_folder, 'unicode'))
148
149    shutil.copyfile(android_path('external/icu/android_icu4c/include/uconfig_local.h'),
150                    android_path('external/icu/libicu/cts_headers/uconfig_local.h'))
151
152    header_subfolders = (
153        'common',
154        'common/unicode',
155        'i18n',
156        'i18n/unicode',
157    )
158    for subfolder in header_subfolders:
159        path = android_path('external/icu/icu4c/source', subfolder)
160        files = [os.path.join(path, f) for f in os.listdir(path) if f.endswith('.h')]
161
162        for src_path in files:
163            base_header_name = os.path.basename(src_path)
164            dst_path = dst_folder
165            if subfolder.endswith('unicode'):
166                dst_path = os.path.join(dst_path, 'unicode')
167            dst_path = os.path.join(dst_path, base_header_name)
168
169            shutil.copyfile(src_path, dst_path)
170
171def get_rename_macro_regex(decl_names):
172    """Return a regex in string to capture the C macro defining the name in the decl_names list"""
173    tag = "|".join(decl_names)
174    return re.compile(r"^(#define (?:" + tag + r") .*)$", re.MULTILINE)
175
176def generate_cts_headers(decl_names):
177    """Generate headers for compiling cintltst as CTS."""
178    copy_cts_headers()
179
180    # Disable all C macro renaming the NDK functions in order to test the functions in the CTS
181    urename_path = android_path('external/icu/libicu/cts_headers/unicode/urename.h')
182    with open(urename_path, "r") as file:
183        src = file.read()
184
185    regex = get_rename_macro_regex(decl_names)
186    modified = regex.sub(r"// \1", src)
187
188    with open(urename_path, "w") as out:
189        out.write(modified)
190
191IGNORED_INCLUDE_DEPENDENCY = {
192    "ubrk.h": ["parseerr.h", ],
193    "ulocdata.h": ["ures.h", "uset.h", ],
194    "unorm2.h": ["uset.h", ],
195    "ustring.h": ["uiter.h", ],
196}
197
198def main():
199    """Parse the ICU4C headers and generate the shim libicu."""
200    logging.basicConfig(level=logging.DEBUG)
201
202    allowlisted_apis = get_allowlisted_apis('libicu_export.txt')
203    decl_filters = [StableDeclarationFilter()]
204    decl_filters.append(AllowlistedDeclarationFilter(allowlisted_apis))
205    parser = DeclaredFunctionsParser(decl_filters, [])
206    parser.set_ignored_include_dependency(IGNORED_INCLUDE_DEPENDENCY)
207
208    parser.parse()
209
210    includes = parser.header_includes
211    functions = parser.declared_functions
212    header_to_function_names = parser.header_to_function_names
213
214    # The shim has the allowlisted functions only
215    functions = [f for f in functions if f.name in allowlisted_apis]
216
217    headers_folder = android_path('external/icu/libicu/ndk_headers/unicode')
218    if os.path.exists(headers_folder):
219        shutil.rmtree(headers_folder)
220    os.mkdir(headers_folder)
221
222    with open(android_path('external/icu/libicu/src/shim.cpp'),
223              'w') as out_file:
224        out_file.write(generate_shim(functions, includes, SYMBOL_SUFFIX, 'libicu_shim.cpp.j2')
225                       .encode('utf8'))
226
227    with open(android_path('external/icu/libicu/libicu.map.txt'), 'w') as out_file:
228        out_file.write(generate_symbol_txt(functions, [], 'libicu.map.txt.j2')
229                       .encode('utf8'))
230
231    # Process the C headers and put them into the ndk folder.
232    for src_path in parser.header_paths_to_copy:
233        basename = os.path.basename(src_path)
234        dst_path = os.path.join(headers_folder, basename)
235        modify_func_declarations(src_path, dst_path, header_to_function_names[basename])
236        # Remove #include lines from the header files.
237        if basename in IGNORED_INCLUDE_DEPENDENCY:
238            remove_ignored_includes(dst_path, IGNORED_INCLUDE_DEPENDENCY[basename])
239
240    copy_header_only_files()
241
242    generate_cts_headers(allowlisted_apis)
243
244    # Apply documentation patches by the following shell script
245    subprocess.check_call(
246        [android_path('external/icu/tools/icu4c_srcgen/doc_patches/apply_patches.sh')])
247
248if __name__ == '__main__':
249    main()
250