1#!/usr/bin/env python
2#
3# Copyright (C) 2015 The Android Open Source Project
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#      http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16#
17from __future__ import print_function
18
19import argparse
20import glob
21import multiprocessing
22import os
23import shutil
24import subprocess
25import sys
26
27import version
28
29
30THIS_DIR = os.path.realpath(os.path.dirname(__file__))
31ORIG_ENV = dict(os.environ)
32
33
34def android_path(*args):
35    return os.path.realpath(os.path.join(THIS_DIR, '../..', *args))
36
37
38def build_path(*args):
39    # Our multistage build directories will be placed under OUT_DIR if it is in
40    # the environment. By default they will be placed under
41    # $ANDROID_BUILD_TOP/out.
42    top_out = ORIG_ENV.get('OUT_DIR', android_path('out'))
43    if not os.path.isabs(top_out):
44        top_out = os.path.realpath(top_out)
45    return os.path.join(top_out, *args)
46
47
48def short_version():
49    return '.'.join([version.major, version.minor])
50
51
52def long_version():
53    return '.'.join([version.major, version.minor, version.patch])
54
55
56def install_file(src, dst):
57    print('Copying ' + src)
58    shutil.copy2(src, dst)
59
60
61def install_directory(src, dst):
62    print('Copying ' + src)
63    shutil.copytree(src, dst)
64
65
66def build(out_dir, prebuilts_path=None, prebuilts_version=None,
67          build_all_llvm_tools=None):
68    products = (
69        'aosp_arm',
70        'aosp_arm64',
71        'aosp_mips',
72        'aosp_mips64',
73        'aosp_x86',
74        'aosp_x86_64',
75    )
76    for product in products:
77        build_product(out_dir, product, prebuilts_path, prebuilts_version,
78                      build_all_llvm_tools)
79
80
81def build_product(out_dir, product, prebuilts_path, prebuilts_version,
82                  build_all_llvm_tools):
83    env = dict(ORIG_ENV)
84    env['DISABLE_LLVM_DEVICE_BUILDS'] = 'true'
85    env['DISABLE_RELOCATION_PACKER'] = 'true'
86    env['FORCE_BUILD_LLVM_COMPONENTS'] = 'true'
87    env['FORCE_BUILD_SANITIZER_SHARED_OBJECTS'] = 'true'
88    env['OUT_DIR'] = out_dir
89    env['SKIP_LLVM_TESTS'] = 'true'
90    env['SOONG_ALLOW_MISSING_DEPENDENCIES'] = 'true'
91    env['TARGET_BUILD_VARIANT'] = 'userdebug'
92    env['TARGET_PRODUCT'] = product
93
94    overrides = []
95    if prebuilts_path is not None:
96        overrides.append('LLVM_PREBUILTS_BASE={}'.format(prebuilts_path))
97    if prebuilts_version is not None:
98        overrides.append('LLVM_PREBUILTS_VERSION={}'.format(prebuilts_version))
99
100    jobs_arg = '-j{}'.format(multiprocessing.cpu_count())
101    targets = ['clang-toolchain']
102    if build_all_llvm_tools:
103        targets += ['llvm-tools']
104    subprocess.check_call(
105        ['make', jobs_arg] + overrides + targets, cwd=android_path(), env=env)
106
107
108def package_toolchain(build_dir, build_name, host, dist_dir):
109    package_name = 'clang-' + build_name
110    install_host_dir = build_path('install', host)
111    install_dir = os.path.join(install_host_dir, package_name)
112
113    # Remove any previously installed toolchain so it doesn't pollute the
114    # build.
115    if os.path.exists(install_host_dir):
116        shutil.rmtree(install_host_dir)
117
118    install_toolchain(build_dir, install_dir, host)
119
120    version_file_path = os.path.join(install_dir, 'AndroidVersion.txt')
121    with open(version_file_path, 'w') as version_file:
122        version_file.write('{}.{}.{}\n'.format(
123            version.major, version.minor, version.patch))
124
125    tarball_name = package_name + '-' + host
126    package_path = os.path.join(dist_dir, tarball_name) + '.tar.bz2'
127    print('Packaging ' + package_path)
128    args = [
129        'tar', '-cjC', install_host_dir, '-f', package_path, package_name
130    ]
131    subprocess.check_call(args)
132
133
134def install_toolchain(build_dir, install_dir, host):
135    install_built_host_files(build_dir, install_dir, host)
136    install_sanitizer_scripts(install_dir)
137    install_scan_scripts(install_dir)
138    install_analyzer_scripts(install_dir)
139    install_headers(build_dir, install_dir, host)
140    install_profile_rt(build_dir, install_dir, host)
141    install_sanitizers(build_dir, install_dir, host)
142    install_license_files(install_dir)
143    install_repo_prop(install_dir)
144
145
146def install_built_host_files(build_dir, install_dir, host):
147    is_windows = host.startswith('windows')
148    is_darwin = host.startswith('darwin-x86')
149    bin_ext = '.exe' if is_windows else ''
150
151    if is_windows:
152        lib_ext = '.dll'
153    elif is_darwin:
154        lib_ext = '.dylib'
155    else:
156        lib_ext = '.so'
157
158    built_files = [
159        'bin/clang' + bin_ext,
160        'bin/clang++' + bin_ext,
161    ]
162    if host != 'windows-x86':
163        built_files.extend([
164            'bin/FileCheck' + bin_ext,
165            'bin/llvm-as' + bin_ext,
166            'bin/llvm-dis' + bin_ext,
167            'bin/llvm-link' + bin_ext,
168            'lib64/libc++' + lib_ext,
169            'lib64/libLLVM' + lib_ext,
170            'lib64/LLVMgold' + lib_ext,
171        ])
172
173    for built_file in built_files:
174        dirname = os.path.dirname(built_file)
175        install_path = os.path.join(install_dir, dirname)
176        if not os.path.exists(install_path):
177            os.makedirs(install_path)
178
179        built_path = os.path.join(build_dir, 'host', host, built_file)
180        install_file(built_path, install_path)
181
182        file_name = os.path.basename(built_file)
183
184        # Only strip bin files (not libs) on darwin.
185        if not is_darwin or built_file.startswith('bin/'):
186            subprocess.check_call(
187                ['strip', os.path.join(install_path, file_name)])
188
189
190def install_sanitizer_scripts(install_dir):
191    script_path = android_path(
192        'external/compiler-rt/lib/asan/scripts/asan_device_setup')
193    shutil.copy2(script_path, os.path.join(install_dir, 'bin'))
194
195
196def install_analyzer_scripts(install_dir):
197    """Create and install bash scripts for invoking Clang for analysis."""
198    analyzer_text = (
199        '#!/bin/bash\n'
200        'if [ "$1" != "-cc1" ]; then\n'
201        '    `dirname $0`/../clang{clang_suffix} -target {target} "$@"\n'
202        'else\n'
203        '    # target/triple already spelled out.\n'
204        '    `dirname $0`/../clang{clang_suffix} "$@"\n'
205        'fi\n'
206    )
207
208    arch_target_pairs = (
209        ('arm64-v8a', 'aarch64-none-linux-android'),
210        ('armeabi', 'armv5te-none-linux-androideabi'),
211        ('armeabi-v7a', 'armv7-none-linux-androideabi'),
212        ('armeabi-v7a-hard', 'armv7-none-linux-androideabi'),
213        ('mips', 'mipsel-none-linux-android'),
214        ('mips64', 'mips64el-none-linux-android'),
215        ('x86', 'i686-none-linux-android'),
216        ('x86_64', 'x86_64-none-linux-android'),
217    )
218
219    for arch, target in arch_target_pairs:
220        arch_path = os.path.join(install_dir, 'bin', arch)
221        os.makedirs(arch_path)
222
223        analyzer_file_path = os.path.join(arch_path, 'analyzer')
224        print('Creating ' + analyzer_file_path)
225        with open(analyzer_file_path, 'w') as analyzer_file:
226            analyzer_file.write(
227                analyzer_text.format(clang_suffix='', target=target))
228
229        analyzerpp_file_path = os.path.join(arch_path, 'analyzer++')
230        print('Creating ' + analyzerpp_file_path)
231        with open(analyzerpp_file_path, 'w') as analyzerpp_file:
232            analyzerpp_file.write(
233                analyzer_text.format(clang_suffix='++', target=target))
234
235
236def install_scan_scripts(install_dir):
237    tools_install_dir = os.path.join(install_dir, 'tools')
238    os.makedirs(tools_install_dir)
239    tools = ('scan-build', 'scan-view')
240    tools_dir = android_path('external/clang/tools')
241    for tool in tools:
242        tool_path = os.path.join(tools_dir, tool)
243        install_path = os.path.join(install_dir, 'tools', tool)
244        install_directory(tool_path, install_path)
245
246
247def install_headers(build_dir, install_dir, host):
248    def should_copy(path):
249        if os.path.basename(path) in ('Makefile', 'CMakeLists.txt'):
250            return False
251        _, ext = os.path.splitext(path)
252        if ext == '.mk':
253            return False
254        return True
255
256    headers_src = android_path('external/clang/lib/Headers')
257    headers_dst = os.path.join(
258        install_dir, 'lib64/clang', short_version(), 'include')
259    os.makedirs(headers_dst)
260    for header in os.listdir(headers_src):
261        if not should_copy(header):
262            continue
263        install_file(os.path.join(headers_src, header), headers_dst)
264
265    install_file(android_path('bionic/libc/include/stdatomic.h'), headers_dst)
266
267    # arm_neon.h gets produced as part of external/clang/lib/Basic/Android.mk.
268    # We must bundle the resulting file as part of the official Clang headers.
269    arm_neon_h = os.path.join(
270        build_dir, 'host', host, 'obj/STATIC_LIBRARIES/'
271        'libclangBasic_intermediates/include/clang/Basic/arm_neon.h')
272    install_file(arm_neon_h, headers_dst)
273
274    os.symlink(short_version(),
275               os.path.join(install_dir, 'lib64/clang', long_version()))
276
277
278def install_profile_rt(build_dir, install_dir, host):
279    lib_dir = os.path.join(
280        install_dir, 'lib64/clang', short_version(), 'lib/linux')
281    os.makedirs(lib_dir)
282
283    install_target_profile_rt(build_dir, lib_dir)
284
285    # We only support profiling libs for Linux and Android.
286    if host == 'linux-x86':
287        install_host_profile_rt(build_dir, host, lib_dir)
288
289
290def install_target_profile_rt(build_dir, lib_dir):
291    product_to_arch = {
292        'generic': 'arm',
293        'generic_arm64': 'aarch64',
294        'generic_mips': 'mipsel',
295        'generic_mips64': 'mips64el',
296        'generic_x86': 'i686',
297        'generic_x86_64': 'x86_64',
298    }
299
300    for product, arch in product_to_arch.items():
301        product_dir = os.path.join(build_dir, 'target/product', product)
302        static_libs = os.path.join(product_dir, 'obj/STATIC_LIBRARIES')
303        built_lib = os.path.join(
304            static_libs, 'libprofile_rt_intermediates/libprofile_rt.a')
305        lib_name = 'libclang_rt.profile-{}-android.a'.format(arch)
306        install_file(built_lib, os.path.join(lib_dir, lib_name))
307
308
309def install_host_profile_rt(build_dir, host, lib_dir):
310    arch_to_obj_dir = {
311        'i686': 'obj32',
312        'x86_64': 'obj',
313    }
314
315    for arch, obj_dir in arch_to_obj_dir.items():
316        static_libs = os.path.join(
317            build_dir, 'host', host, obj_dir, 'STATIC_LIBRARIES')
318        built_lib = os.path.join(
319            static_libs, 'libprofile_rt_intermediates/libprofile_rt.a')
320        lib_name = 'libclang_rt.profile-{}.a'.format(arch)
321        install_file(built_lib, os.path.join(lib_dir, lib_name))
322
323
324def install_sanitizers(build_dir, install_dir, host):
325    headers_src = android_path('external/compiler-rt/include/sanitizer')
326    clang_lib = os.path.join(install_dir, 'lib64/clang', short_version())
327    headers_dst = os.path.join(clang_lib, 'include/sanitizer')
328    lib_dst = os.path.join(clang_lib, 'lib/linux')
329    install_directory(headers_src, headers_dst)
330
331    if host == 'linux-x86':
332        install_host_sanitizers(build_dir, host, lib_dst)
333
334    # Tuples of (product, arch, libdir)
335    product_to_arch = (
336        ('generic', 'arm', 'lib'),
337        ('generic_arm64', 'aarch64', 'lib64'),
338        ('generic_x86', 'i686', 'lib'),
339    )
340
341    for product, arch, libdir in product_to_arch:
342        product_dir = os.path.join(build_dir, 'target/product', product)
343        system_dir = os.path.join(product_dir, 'system')
344        system_lib_dir = os.path.join(system_dir, libdir)
345        lib_name = 'libclang_rt.asan-{}-android.so'.format(arch)
346        built_lib = os.path.join(system_lib_dir, lib_name)
347        install_file(built_lib, lib_dst)
348
349
350def install_host_sanitizers(build_dir, host, lib_dst):
351    # Tuples of (name, multilib).
352    libs = (
353        ('asan', True),
354        ('asan_cxx', True),
355        ('ubsan_standalone', True),
356        ('ubsan_standalone_cxx', True),
357        ('tsan', False),
358        ('tsan_cxx', False),
359    )
360
361    obj32 = os.path.join(build_dir, 'host', host, 'obj32/STATIC_LIBRARIES')
362    obj64 = os.path.join(build_dir, 'host', host, 'obj/STATIC_LIBRARIES')
363    for lib, is_multilib in libs:
364        built_lib_name = 'lib{}.a'.format(lib)
365
366        obj64_dir = os.path.join(obj64, 'lib{}_intermediates'.format(lib))
367        lib64_name = 'libclang_rt.{}-x86_64.a'.format(lib)
368        built_lib64 = os.path.join(obj64_dir, built_lib_name)
369        install_file(built_lib64, os.path.join(lib_dst, lib64_name))
370        if is_multilib:
371            obj32_dir = os.path.join(obj32, 'lib{}_intermediates'.format(lib))
372            lib32_name = 'libclang_rt.{}-i686.a'.format(lib)
373            built_lib32 = os.path.join(obj32_dir, built_lib_name)
374            install_file(built_lib32, os.path.join(lib_dst, lib32_name))
375
376
377def install_license_files(install_dir):
378    projects = (
379        'clang',
380        'compiler-rt',
381        'libcxx',
382        'libcxxabi',
383        'libunwind_llvm',
384        'llvm',
385    )
386
387    notices = []
388    for project in projects:
389        project_path = android_path('external', project)
390        license_pattern = os.path.join(project_path, 'MODULE_LICENSE_*')
391        for license_file in glob.glob(license_pattern):
392            install_file(license_file, install_dir)
393        with open(os.path.join(project_path, 'NOTICE')) as notice_file:
394            notices.append(notice_file.read())
395    with open(os.path.join(install_dir, 'NOTICE'), 'w') as notice_file:
396        notice_file.write('\n'.join(notices))
397
398
399def install_repo_prop(install_dir):
400    file_name = 'repo.prop'
401
402    dist_dir = os.environ.get('DIST_DIR')
403    if dist_dir is not None:
404        dist_repo_prop = os.path.join(dist_dir, file_name)
405        shutil.copy(dist_repo_prop, install_dir)
406    else:
407        out_file = os.path.join(install_dir, file_name)
408        with open(out_file, 'w') as prop_file:
409            cmd = [
410                'repo', 'forall', '-c',
411                'echo $REPO_PROJECT $(git rev-parse HEAD)',
412            ]
413            subprocess.check_call(cmd, stdout=prop_file)
414
415
416def parse_args():
417    parser = argparse.ArgumentParser()
418
419    parser.add_argument(
420        '--build-name', default='dev', help='Release name for the package.')
421
422    multi_stage_group = parser.add_mutually_exclusive_group()
423    multi_stage_group.add_argument(
424        '--multi-stage', action='store_true', default=True,
425        help='Perform multi-stage build (enabled by default).')
426    multi_stage_group.add_argument(
427        '--no-multi-stage', action='store_false', dest='multi_stage',
428        help='Do not perform multi-stage build.')
429
430    parser.add_argument(
431        '--build-all-llvm-tools', action='store_true', default=True,
432        help='Build all the LLVM tools/utilities.')
433
434    parser.add_argument(
435        '--no-build-all-llvm-tools', action='store_false',
436        dest='build_all_llvm_tools',
437        help='Build all the LLVM tools/utilities.')
438
439    return parser.parse_args()
440
441
442def main():
443    args = parse_args()
444
445    if sys.platform.startswith('linux'):
446        hosts = ['linux-x86', 'windows-x86']
447    elif sys.platform == 'darwin':
448        hosts = ['darwin-x86']
449    else:
450        raise RuntimeError('Unsupported host: {}'.format(sys.platform))
451
452    stage_1_out_dir = build_path('stage1')
453    build(out_dir=stage_1_out_dir)
454    final_out_dir = stage_1_out_dir
455    if args.multi_stage:
456        stage_1_install_dir = build_path('stage1-install')
457        for host in hosts:
458            package_name = 'clang-' + args.build_name
459            install_host_dir = os.path.join(stage_1_install_dir, host)
460            install_dir = os.path.join(install_host_dir, package_name)
461
462            # Remove any previously installed toolchain so it doesn't pollute
463            # the build.
464            if os.path.exists(install_host_dir):
465                shutil.rmtree(install_host_dir)
466
467            install_toolchain(stage_1_out_dir, install_dir, host)
468
469        stage_2_out_dir = build_path('stage2')
470        build(out_dir=stage_2_out_dir, prebuilts_path=stage_1_install_dir,
471              prebuilts_version=package_name,
472              build_all_llvm_tools=args.build_all_llvm_tools)
473        final_out_dir = stage_2_out_dir
474
475    dist_dir = ORIG_ENV.get('DIST_DIR', final_out_dir)
476    for host in hosts:
477        package_toolchain(final_out_dir, args.build_name, host, dist_dir)
478
479
480if __name__ == '__main__':
481    main()
482