1#!/usr/bin/env python3
2
3# Copyright 2018 The SwiftShader Authors. All Rights Reserved.
4#
5# Licensed under the Apache License, Version 2.0 (the "License");
6# you may not use this file except in compliance with the License.
7# You may obtain a copy of the License at
8#
9#    http://www.apache.org/licenses/LICENSE-2.0
10#
11# Unless required by applicable law or agreed to in writing, software
12# distributed under the License is distributed on an "AS IS" BASIS,
13# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14# See the License for the specific language governing permissions and
15# limitations under the License.
16
17import argparse
18import contextlib
19import multiprocessing
20import os
21import platform
22import re
23import shutil
24import subprocess
25import sys
26import tempfile
27from os import path
28
29# LLVM_BRANCH must match the value of the same variable in third_party/update-llvm-10.sh
30LLVM_BRANCH = "release/10.x"
31
32# LLVM_COMMIT must be set to the commit hash that we last updated to when running third_party/update-llvm-10.sh.
33# Run 'git show -s origin/llvm10-clean' and look for 'llvm-10-update: <hash>' to retrieve it.
34LLVM_COMMIT = "d32170dbd5b0d54436537b6b75beaf44324e0c28"
35
36SCRIPT_DIR = path.dirname(path.realpath(sys.argv[0]))
37LLVM_STAGING_DIR = path.abspath(path.join(tempfile.gettempdir(), "llvm-10"))
38LLVM_DIR = path.abspath(path.join(LLVM_STAGING_DIR, "llvm"))
39LLVM_OBJS = path.join(LLVM_STAGING_DIR, "build")
40LLVM_CONFIGS = path.abspath(path.join(SCRIPT_DIR, '..', 'configs'))
41
42# List of all arches SwiftShader supports
43LLVM_TARGETS = [
44    ('AArch64', ('__aarch64__',)),
45    ('ARM', ('__arm__',)),
46    ('X86', ('__i386__', '__x86_64__')),
47    ('Mips', ('__mips__',)),
48    ('PowerPC', ('__powerpc64__',)),
49]
50
51# Per-platform arches
52LLVM_TRIPLES = {
53    'android': [
54        ('__x86_64__', 'x86_64-linux-android'),
55        ('__i386__', 'i686-linux-android'),
56        ('__arm__', 'armv7-linux-androideabi'),
57        ('__aarch64__', 'aarch64-linux-android'),
58    ],
59    'linux': [
60        ('__x86_64__', 'x86_64-unknown-linux-gnu'),
61        ('__i386__', 'i686-pc-linux-gnu'),
62        ('__arm__', 'armv7-linux-gnueabihf'),
63        ('__aarch64__', 'aarch64-linux-gnu'),
64        ('__mips__', 'mipsel-linux-gnu'),
65        ('__mips64', 'mips64el-linux-gnuabi64'),
66        ('__powerpc64__', 'powerpc64le-unknown-linux-gnu'),
67    ],
68    'darwin': [
69        ('__x86_64__', 'x86_64-apple-darwin'),
70        ('__aarch64__', 'arm64-apple-darwin'),
71    ],
72    'windows': [
73        ('__x86_64__', 'x86_64-pc-win32'),
74        ('__i386__', 'i686-pc-win32'),
75        ('__arm__', 'armv7-pc-win32'),
76        ('__aarch64__', 'aarch64-pc-win32'),
77        ('__mips__', 'mipsel-pc-win32'),
78        ('__mips64', 'mips64el-pc-win32'),
79    ],
80    'fuchsia': [
81        ('__x86_64__', 'x86_64-unknown-fuchsia'),
82        ('__aarch64__', 'aarch64-unknown-fuchsia'),
83    ]
84}
85
86# Mapping of target platform to the host it must be built on
87LLVM_PLATFORM_TO_HOST_SYSTEM = {
88    'android': 'Linux',
89    'darwin': 'Darwin',
90    'linux': 'Linux',
91    'windows': 'Windows',
92    'fuchsia': 'Linux'
93}
94
95# LLVM configurations to be undefined.
96LLVM_UNDEF_MACROS = [
97    'BACKTRACE_HEADER',
98    'ENABLE_BACKTRACES',
99    'ENABLE_CRASH_OVERRIDES',
100    'HAVE_BACKTRACE',
101    'HAVE_POSIX_SPAWN',
102    'HAVE_PTHREAD_GETNAME_NP',
103    'HAVE_PTHREAD_SETNAME_NP',
104    'HAVE_TERMIOS_H',
105    'HAVE_ZLIB_H',
106    'HAVE__UNWIND_BACKTRACE',
107]
108
109# General CMake options for building LLVM
110LLVM_CMAKE_OPTIONS = [
111    '-DCMAKE_BUILD_TYPE=Release',
112    '-DLLVM_ENABLE_THREADS=ON',
113    '-DLLVM_ENABLE_TERMINFO=OFF',
114    '-DLLVM_ENABLE_LIBXML2=OFF',
115    '-DLLVM_ENABLE_LIBEDIT=OFF',
116    '-DLLVM_ENABLE_LIBPFM=OFF',
117    '-DLLVM_ENABLE_ZLIB=OFF',
118    '-DLLVM_TEMPORARILY_ALLOW_OLD_TOOLCHAIN=ON'
119]
120
121# Used when building LLVM for darwin. Affects values set in the generated config files.
122MIN_MACOS_VERSION = '10.10'
123
124@contextlib.contextmanager
125def pushd(new_dir):
126    previous_dir = os.getcwd()
127    os.chdir(new_dir)
128    try:
129        yield
130    finally:
131        os.chdir(previous_dir)
132
133
134def log(message, level=1):
135    print(' ' * level + '> ' + message)
136
137
138def run_command(command, log_level=1):
139    log(command, log_level)
140    os.system(command)
141
142
143def run_subprocess(*popenargs, log_level=1, cwd=None):
144    log([' '.join(t) for t in popenargs][0], log_level)
145    return subprocess.run(*popenargs, cwd=cwd)
146
147
148def _parse_args():
149    parser = argparse.ArgumentParser()
150    parser.add_argument('name', help='destination name',
151                        choices=['android', 'linux', 'darwin', 'windows', 'fuchsia'])
152    parser.add_argument('-j', '--jobs', help='parallel compilation', type=int)
153    return parser.parse_args()
154
155
156def validate_args(args):
157    host = platform.system()
158    if host not in LLVM_PLATFORM_TO_HOST_SYSTEM.values():
159        raise Exception(f"Host system not supported: '{host}'")
160
161    if args.name not in LLVM_PLATFORM_TO_HOST_SYSTEM.keys():
162        raise Exception(f"Unknown target platform: '{args.name}'")
163
164    expected_host = LLVM_PLATFORM_TO_HOST_SYSTEM[args.name]
165    if LLVM_PLATFORM_TO_HOST_SYSTEM[args.name] != host:
166        raise Exception(
167            f"Target platform '{args.name}' must be built on '{expected_host}', not on '{host}'")
168
169
170def get_cmake_targets_to_build(platform):
171    """Returns list of LLVM targets to build for the input platform"""
172    targets = set()
173    for arch_def, triplet in LLVM_TRIPLES[platform]:
174        for arch, defs in LLVM_TARGETS:
175            if arch_def in defs:
176                targets.add(arch)
177
178    # Maintain the sort order of LLVM_TARGETS as this affects how config
179    # headers are generated
180    return [t[0] for t in LLVM_TARGETS if t[0] in targets]
181
182
183def clone_llvm():
184    log("Cloning/Updating LLVM", 1)
185    # Clone or update staging directory
186    if not path.exists(LLVM_STAGING_DIR):
187        os.mkdir(LLVM_STAGING_DIR)
188        with pushd(LLVM_STAGING_DIR):
189            run_command('git init', 2)
190            run_command(
191                'git remote add origin https://github.com/llvm/llvm-project.git', 2)
192            run_command('git config core.sparsecheckout true', 2)
193            run_command('echo /llvm > .git/info/sparse-checkout', 2)
194
195    with pushd(LLVM_STAGING_DIR):
196        run_command('echo /llvm > .git/info/sparse-checkout', 2)
197        run_command('git fetch origin {}'.format(LLVM_BRANCH), 2)
198        run_command('git checkout {}'.format(LLVM_COMMIT), 2)
199    return
200
201
202def build_llvm(name, num_jobs):
203    """Build LLVM and get all generated files."""
204    log("Building LLVM", 1)
205    if num_jobs is None:
206        num_jobs = multiprocessing.cpu_count()
207
208    """On Windows we need to have CMake generate build files for the 64-bit
209    Visual Studio host toolchain."""
210    host = '-Thost=x64' if name == 'windows' else ''
211
212    cmake_options = LLVM_CMAKE_OPTIONS + ['-DLLVM_TARGETS_TO_BUILD=' +
213                                          ';'.join(t for t in get_cmake_targets_to_build(name))]
214
215    if name == 'darwin':
216        cmake_options.append('-DCMAKE_OSX_DEPLOYMENT_TARGET={}'.format(MIN_MACOS_VERSION))
217
218    os.makedirs(LLVM_OBJS, exist_ok=True)
219    run_subprocess(['cmake', host, LLVM_DIR] +
220                   cmake_options, log_level=2, cwd=LLVM_OBJS)
221    run_subprocess(['cmake', '--build', '.', '-j',
222                    str(num_jobs)], log_level=2, cwd=LLVM_OBJS)
223
224
225def list_files(src_base, src, dst_base, suffixes):
226    """Enumerate the files that are under `src` and end with one of the
227    `suffixes` and yield the source path and the destination path."""
228    src_base = path.abspath(src_base)
229    src = path.join(src_base, src)
230    for base_dir, dirnames, filenames in os.walk(src):
231        for filename in filenames:
232            if path.splitext(filename)[1] in suffixes:
233                relative = path.relpath(base_dir, src_base)
234                yield (path.join(base_dir, filename),
235                       path.join(dst_base, relative, filename))
236
237
238def copy_common_generated_files(dst_base):
239    """Copy platform-independent generated files."""
240    log("Copying platform-independent generated files", 1)
241    suffixes = {'.inc', '.h', '.def'}
242    subdirs = [
243        path.join('include', 'llvm', 'IR'),
244        path.join('include', 'llvm', 'Support'),
245        path.join('lib', 'IR'),
246        path.join('lib', 'Transforms', 'InstCombine'),
247    ] + [path.join('lib', 'Target', arch) for arch, defs in LLVM_TARGETS]
248    for subdir in subdirs:
249        for src, dst in list_files(LLVM_OBJS, subdir, dst_base, suffixes):
250            log('{} -> {}'.format(src, dst), 2)
251            os.makedirs(path.dirname(dst), exist_ok=True)
252            shutil.copyfile(src, dst)
253
254
255def copy_platform_file(platform, src, dst):
256    """Copy platform-dependent generated files and add platform-specific
257    modifications."""
258
259    # LLVM configuration patterns to be post-processed.
260    llvm_target_pattern = re.compile('^LLVM_[A-Z_]+\\(([A-Za-z0-9_]+)\\)$')
261    llvm_native_pattern = re.compile(
262        '^#define LLVM_NATIVE_([A-Z]+) (LLVMInitialize)?(.*)$')
263    llvm_triple_pattern = re.compile('^#define (LLVM_[A-Z_]+_TRIPLE) "(.*)"$')
264    llvm_define_pattern = re.compile('^#define ([A-Za-z0-9_]+) (.*)$')
265
266    # Build architecture-specific conditions.
267    conds = {}
268    for arch, defs in LLVM_TARGETS:
269        conds[arch] = ' || '.join('defined(' + v + ')' for v in defs)
270
271    # Get a set of platform-specific triples.
272    triples = LLVM_TRIPLES[platform]
273
274    with open(src, 'r') as src_file:
275        os.makedirs(path.dirname(dst), exist_ok=True)
276        with open(dst, 'w') as dst_file:
277            for line in src_file:
278                if line == '#define LLVM_CONFIG_H\n':
279                    print(line, file=dst_file, end='')
280                    print('', file=dst_file)
281                    print('#if !defined(__i386__) && defined(_M_IX86)',
282                          file=dst_file)
283                    print('#define __i386__ 1', file=dst_file)
284                    print('#endif', file=dst_file)
285                    print('', file=dst_file)
286                    print(
287                        '#if !defined(__x86_64__) && (defined(_M_AMD64) || defined (_M_X64))', file=dst_file)
288                    print('#define __x86_64__ 1', file=dst_file)
289                    print('#endif', file=dst_file)
290                    print('', file=dst_file)
291
292                match = llvm_target_pattern.match(line)
293                if match:
294                    arch = match.group(1)
295                    print('#if ' + conds[arch], file=dst_file)
296                    print(line, file=dst_file, end='')
297                    print('#endif', file=dst_file)
298                    continue
299
300                match = llvm_native_pattern.match(line)
301                if match:
302                    name = match.group(1)
303                    init = match.group(2) or ''
304                    arch = match.group(3)
305                    end = ''
306                    if arch.lower().endswith(name.lower()):
307                        end = arch[-len(name):]
308                    directive = '#if '
309                    for arch, defs in LLVM_TARGETS:
310                        print(directive + conds[arch], file=dst_file)
311                        print('#define LLVM_NATIVE_' + name + ' ' +
312                              init + arch + end, file=dst_file)
313                        directive = '#elif '
314                    print('#else', file=dst_file)
315                    print('#error "unknown architecture"', file=dst_file)
316                    print('#endif', file=dst_file)
317                    continue
318
319                match = llvm_triple_pattern.match(line)
320                if match:
321                    name = match.group(1)
322                    directive = '#if'
323                    for defs, triple in triples:
324                        print(directive + ' defined(' + defs + ')',
325                              file=dst_file)
326                        print('#define ' + name + ' "' + triple + '"',
327                              file=dst_file)
328                        directive = '#elif'
329                    print('#else', file=dst_file)
330                    print('#error "unknown architecture"', file=dst_file)
331                    print('#endif', file=dst_file)
332                    continue
333
334                match = llvm_define_pattern.match(line)
335                if match and match.group(1) in LLVM_UNDEF_MACROS:
336                    print('/* #undef ' + match.group(1) + ' */', file=dst_file)
337                    continue
338
339                print(line, file=dst_file, end='')
340
341
342def copy_platform_generated_files(platform, dst_base):
343    """Copy platform-specific generated files."""
344    log("Copying platform-specific generated files", 1)
345    suffixes = {'.inc', '.h', '.def'}
346    src_dir = path.join('include', 'llvm', 'Config')
347    for src, dst in list_files(LLVM_OBJS, src_dir, dst_base, suffixes):
348        log('{}, {} -> {}'.format(platform, src, dst), 2)
349        copy_platform_file(platform, src, dst)
350
351
352def main():
353    args = _parse_args()
354    validate_args(args)
355    clone_llvm()
356    build_llvm(args.name, args.jobs)
357    copy_common_generated_files(path.join(LLVM_CONFIGS, 'common'))
358    copy_platform_generated_files(
359        args.name, path.join(LLVM_CONFIGS, args.name))
360
361
362if __name__ == '__main__':
363    main()
364