1#!/usr/bin/env python3
2#
3# Copyright (C) 2019 The Android Open Source Project
4# All rights reserved.
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions
8# are met:
9#  * Redistributions of source code must retain the above copyright
10#    notice, this list of conditions and the following disclaimer.
11#  * Redistributions in binary form must reproduce the above copyright
12#    notice, this list of conditions and the following disclaimer in
13#    the documentation and/or other materials provided with the
14#    distribution.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS
19# FOR A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE
20# COPYRIGHT OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT,
21# INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING,
22# BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS
23# OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED
24# AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
25# OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT
26# OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
27# SUCH DAMAGE.
28
29# Generate a benchmark using a JSON dump of ELF file symbols and relocations.
30
31import argparse
32import codecs
33import json
34import math
35import os
36import re
37import shlex
38import shutil
39import subprocess
40import sys
41import tempfile
42import textwrap
43import typing
44from enum import Enum
45from typing import Dict, List, Optional, Set
46from subprocess import PIPE, DEVNULL
47from pathlib import Path
48
49from common_types import LoadedLibrary, SymbolRef, SymBind, SymKind, bfs_walk, json_to_elf_tree
50
51
52g_obfuscate = True
53g_benchmark_name = 'linker_reloc_bench'
54
55
56kBionicSonames: Set[str] = set([
57    'libc.so',
58    'libdl.so',
59    'libdl_android.so',
60    'libm.so',
61    'ld-android.so',
62])
63
64# Skip these symbols so the benchmark runs on multiple C libraries (glibc, Bionic, musl).
65kBionicIgnoredSymbols: Set[str] = set([
66    '__FD_ISSET_chk',
67    '__FD_SET_chk',
68    '__assert',
69    '__assert2',
70    '__b64_ntop',
71    '__cmsg_nxthdr',
72    '__cxa_thread_atexit_impl',
73    '__errno',
74    '__gnu_basename',
75    '__gnu_strerror_r',
76    '__memcpy_chk',
77    '__memmove_chk',
78    '__memset_chk',
79    '__open_2',
80    '__openat_2',
81    '__pread64_chk',
82    '__pread_chk',
83    '__read_chk',
84    '__readlink_chk',
85    '__register_atfork',
86    '__sF',
87    '__strcat_chk',
88    '__strchr_chk',
89    '__strcpy_chk',
90    '__strlcat_chk',
91    '__strlcpy_chk',
92    '__strlen_chk',
93    '__strncat_chk',
94    '__strncpy_chk',
95    '__strncpy_chk2',
96    '__strrchr_chk',
97    '__system_property_area_serial',
98    '__system_property_find',
99    '__system_property_foreach',
100    '__system_property_get',
101    '__system_property_read',
102    '__system_property_serial',
103    '__system_property_set',
104    '__umask_chk',
105    '__vsnprintf_chk',
106    '__vsprintf_chk',
107    'android_dlopen_ext',
108    'android_set_abort_message',
109    'arc4random_buf',
110    'dl_unwind_find_exidx',
111    'fts_close',
112    'fts_open',
113    'fts_read',
114    'fts_set',
115    'getprogname',
116    'gettid',
117    'isnanf',
118    'lseek64',
119    'lstat64',
120    'mallinfo',
121    'malloc_info',
122    'pread64',
123    'pthread_gettid_np',
124    'pwrite64',
125    'res_mkquery',
126    'strlcpy',
127    'strtoll_l',
128    'strtoull_l',
129    'tgkill',
130])
131
132
133Definitions = Dict[str, LoadedLibrary]
134
135def build_symbol_index(lib: LoadedLibrary) -> Definitions:
136    defs: Dict[str, LoadedLibrary] = {}
137    for lib in bfs_walk(lib):
138        for sym in lib.syms.values():
139            if not sym.defined: continue
140            defs.setdefault(sym.name, lib)
141    return defs
142
143
144def check_rels(root: LoadedLibrary, defs: Definitions) -> None:
145    # Find every symbol for every relocation in the load group.
146    has_missing = False
147    for lib in bfs_walk(root):
148        rels = lib.rels
149        for sym in rels.got + rels.jump_slots + [sym for off, sym in rels.symbolic]:
150            if sym.name not in defs:
151                if sym.is_weak:
152                    pass # print('info: weak undefined', lib.soname, r)
153                else:
154                    print(f'error: {lib.soname}: unresolved relocation to {sym.name}')
155                    has_missing = True
156    if has_missing: sys.exit('error: had unresolved relocations')
157
158
159# Obscure names to avoid polluting Android code search.
160def rot13(text: str) -> str:
161    if g_obfuscate:
162        result = codecs.getencoder("rot-13")(text)[0]
163        assert isinstance(result, str)
164        return result
165    else:
166        return text
167
168
169def make_asm_file(lib: LoadedLibrary, is_main: bool, out_filename: Path, map_out_filename: Path,
170                  defs: Definitions) -> bool:
171
172    def trans_sym(name: str, ver: Optional[str]) -> Optional[str]:
173        nonlocal defs
174        d = defs.get(name)
175        if d is not None and d.soname in kBionicSonames:
176            if name in kBionicIgnoredSymbols: return None
177            # Discard relocations to newer Bionic symbols, because there aren't many of them, and
178            # they would limit where the benchmark can run.
179            if ver == 'LIBC': return name
180            return None
181        return 'b_' + rot13(name)
182
183    versions: Dict[Optional[str], List[str]] = {}
184
185    with open(out_filename, 'w') as out:
186        out.write(f'// AUTO-GENERATED BY {os.path.basename(__file__)} -- do not edit manually\n')
187        out.write(f'#include "{g_benchmark_name}_asm.h"\n')
188        out.write('.data\n')
189        out.write('.p2align 4\n')
190
191        if is_main:
192            out.write('.text\n' 'MAIN\n')
193
194        for d in lib.syms.values():
195            if not d.defined: continue
196            sym = trans_sym(d.name, None)
197            if sym is None: continue
198            binding = 'weak' if d.bind == SymBind.Weak else 'globl'
199            if d.kind == SymKind.Func:
200                out.write('.text\n'
201                          f'.{binding} {sym}\n'
202                          f'.type {sym},%function\n'
203                          f'{sym}:\n'
204                          'nop\n')
205            else: # SymKind.Var
206                out.write('.data\n'
207                          f'.{binding} {sym}\n'
208                          f'.type {sym},%object\n'
209                          f'{sym}:\n'
210                          f'.space __SIZEOF_POINTER__\n')
211            versions.setdefault(d.ver_name, []).append(sym)
212
213        out.write('.text\n')
214        for r in lib.rels.jump_slots:
215            sym = trans_sym(r.name, r.ver)
216            if sym is None: continue
217            if r.is_weak: out.write(f'.weak {sym}\n')
218            out.write(f'CALL({sym})\n')
219        out.write('.text\n')
220        for r in lib.rels.got:
221            sym = trans_sym(r.name, r.ver)
222            if sym is None: continue
223            if r.is_weak: out.write(f'.weak {sym}\n')
224            out.write(f'GOT_RELOC({sym})\n')
225
226        out.write('.data\n')
227        out.write('local_label:\n')
228
229        image = []
230        for off in lib.rels.relative:
231            image.append((off, f'DATA_WORD(local_label)\n'))
232        for off, r in lib.rels.symbolic:
233            sym = trans_sym(r.name, r.ver)
234            if sym is None: continue
235            text = f'DATA_WORD({sym})\n'
236            if r.is_weak: text += f'.weak {sym}\n'
237            image.append((off, text))
238        image.sort()
239
240        cur_off = 0
241        for off, text in image:
242            if cur_off < off:
243                out.write(f'.space (__SIZEOF_POINTER__ * {off - cur_off})\n')
244                cur_off = off
245            out.write(text)
246            cur_off += 1
247
248    has_map_file = False
249    if len(versions) > 0 and list(versions.keys()) != [None]:
250        has_map_file = True
251        with open(map_out_filename, 'w') as out:
252            if None in versions:
253                print(f'error: {out_filename} has both unversioned and versioned symbols')
254                print(versions.keys())
255                sys.exit(1)
256            for ver in sorted(versions.keys()):
257                assert ver is not None
258                out.write(f'{rot13(ver)} {{\n')
259                if len(versions[ver]) > 0:
260                    out.write('  global:\n')
261                    out.write(''.join(f'    {x};\n' for x in versions[ver]))
262                out.write(f'}};\n')
263
264    return has_map_file
265
266
267class LibNames:
268    def __init__(self, root: LoadedLibrary):
269        self._root = root
270        self._names: Dict[LoadedLibrary, str] = {}
271        all_libs = [x for x in bfs_walk(root) if x is not root and x.soname not in kBionicSonames]
272        num_digits = math.ceil(math.log10(len(all_libs) + 1))
273        if g_obfuscate:
274            self._names = {x : f'{i:0{num_digits}}' for i, x in enumerate(all_libs)}
275        else:
276            self._names = {x : re.sub(r'\.so$', '', x.soname) for x in all_libs}
277
278    def name(self, lib: LoadedLibrary) -> str:
279        if lib is self._root:
280            return f'{g_benchmark_name}_main'
281        else:
282            return f'lib{g_benchmark_name}_{self._names[lib]}'
283
284
285# Generate a ninja file directly that builds the benchmark using a C compiler driver and ninja.
286# Using a driver directly can be faster than building with Soong, and it allows testing
287# configurations that Soong can't target, like musl.
288def make_ninja_benchmark(root: LoadedLibrary, defs: Definitions, cc: str, out: Path) -> None:
289
290    lib_names = LibNames(root)
291
292    def lib_dso_name(lib: LoadedLibrary) -> str:
293        return lib_names.name(lib) + '.so'
294
295    ninja = open(out / 'build.ninja', 'w')
296    include_path = os.path.relpath(os.path.dirname(__file__) + '/../include', out)
297    common_flags = f"-Wl,-rpath-link,. -lm -I{include_path}"
298    ninja.write(textwrap.dedent(f'''\
299        rule exe
300            command = {cc} -fpie -pie $in -o $out {common_flags} $extra_args
301        rule dso
302            command = {cc} -fpic -shared $in -o $out -Wl,-soname,$out {common_flags} $extra_args
303    '''))
304
305    for lib in bfs_walk(root):
306        if lib.soname in kBionicSonames: continue
307
308        lib_base_name = lib_names.name(lib)
309        asm_name = lib_base_name + '.S'
310        map_name = lib_base_name + '.map'
311        asm_path = out / asm_name
312        map_path = out / map_name
313
314        has_map_file = make_asm_file(lib, lib is root, asm_path, map_path, defs)
315        needed = ' '.join([lib_dso_name(x) for x in lib.needed if x.soname not in kBionicSonames])
316
317        if lib is root:
318            ninja.write(f'build {lib_base_name}: exe {asm_name} {needed}\n')
319        else:
320            ninja.write(f'build {lib_dso_name(lib)}: dso {asm_name} {needed}\n')
321        if has_map_file:
322            ninja.write(f'    extra_args = -Wl,--version-script={map_name}\n')
323
324    ninja.close()
325
326    subprocess.run(['ninja', '-C', str(out), lib_names.name(root)], check=True)
327
328
329def make_soong_benchmark(root: LoadedLibrary, defs: Definitions, out: Path) -> None:
330
331    lib_names = LibNames(root)
332
333    bp = open(out / 'Android.bp', 'w')
334    bp.write(f'// AUTO-GENERATED BY {os.path.basename(__file__)} -- do not edit\n')
335
336    bp.write(f'package {{ default_applicable_licenses: ["bionic_benchmarks_license"], }}\n')
337    bp.write(f'cc_defaults {{\n')
338    bp.write(f'    name: "{g_benchmark_name}_all_libs",\n')
339    bp.write(f'    runtime_libs: [\n')
340    for lib in bfs_walk(root):
341        if lib.soname in kBionicSonames: continue
342        if lib is root: continue
343        bp.write(f'        "{lib_names.name(lib)}",\n')
344    bp.write(f'    ],\n')
345    bp.write(f'}}\n')
346
347    for lib in bfs_walk(root):
348        if lib.soname in kBionicSonames: continue
349
350        lib_base_name = lib_names.name(lib)
351        asm_name = lib_base_name + '.S'
352        map_name = lib_base_name + '.map'
353        asm_path = out / asm_name
354        map_path = out / map_name
355
356        has_map_file = make_asm_file(lib, lib is root, asm_path, map_path, defs)
357
358        if lib is root:
359            bp.write(f'cc_binary {{\n')
360            bp.write(f'    defaults: ["{g_benchmark_name}_binary"],\n')
361        else:
362            bp.write(f'cc_test_library {{\n')
363            bp.write(f'    defaults: ["{g_benchmark_name}_library"],\n')
364        bp.write(f'    name: "{lib_base_name}",\n')
365        bp.write(f'    srcs: ["{asm_name}"],\n')
366        bp.write(f'    shared_libs: [\n')
367        for need in lib.needed:
368            if need.soname in kBionicSonames: continue
369            bp.write(f'        "{lib_names.name(need)}",\n')
370        bp.write(f'    ],\n')
371        if has_map_file:
372            bp.write(f'    version_script: "{map_name}",\n')
373        bp.write('}\n')
374
375    bp.close()
376
377
378def main() -> None:
379    parser = argparse.ArgumentParser()
380    parser.add_argument('input', type=str)
381    parser.add_argument('out_dir', type=str)
382    parser.add_argument('--ninja', action='store_true',
383                        help='Generate a benchmark using a compiler and ninja rather than Soong')
384    parser.add_argument('--cc',
385                        help='For --ninja, a target-specific C clang driver and flags (e.g. "'
386                             '$NDK/toolchains/llvm/prebuilt/linux-x86_64/bin/aarch64-linux-android29-clang'
387                             ' -fuse-ld=lld")')
388
389    args = parser.parse_args()
390
391    if args.ninja:
392        if args.cc is None: sys.exit('error: --cc required with --ninja')
393
394    out = Path(args.out_dir)
395    with open(Path(args.input)) as f:
396        root = json_to_elf_tree(json.load(f))
397    defs = build_symbol_index(root)
398    check_rels(root, defs)
399
400    if out.exists(): shutil.rmtree(out)
401    os.makedirs(str(out))
402
403    if args.ninja:
404        make_ninja_benchmark(root, defs, args.cc, out)
405    else:
406        make_soong_benchmark(root, defs, out)
407
408
409if __name__ == '__main__':
410    main()
411