1#!/bin/bash -eux
2# Copyright 2016 Google Inc.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#      http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15#
16################################################################################
17
18# See issue #4270. The compiler crashes on GCB instance with 32 vCPUs, so when
19# we compile on GCB we want 16 cores. But locally we want more (so use nproc /
20# 2).
21NPROC=$(expr $(nproc) / 2)
22
23# zlib1g-dev is needed for llvm-profdata to handle coverage data from rust compiler
24LLVM_DEP_PACKAGES="build-essential make cmake ninja-build git python3 g++-multilib binutils-dev zlib1g-dev"
25apt-get install -y $LLVM_DEP_PACKAGES --no-install-recommends
26
27# Checkout
28CHECKOUT_RETRIES=10
29function clone_with_retries {
30  REPOSITORY=$1
31  LOCAL_PATH=$2
32  CHECKOUT_RETURN_CODE=1
33
34  # Disable exit on error since we might encounter some failures while retrying.
35  set +e
36  for i in $(seq 1 $CHECKOUT_RETRIES); do
37    rm -rf $LOCAL_PATH
38    git clone $REPOSITORY $LOCAL_PATH
39    CHECKOUT_RETURN_CODE=$?
40    if [ $CHECKOUT_RETURN_CODE -eq 0 ]; then
41      break
42    fi
43  done
44
45  # Re-enable exit on error. If checkout failed, script will exit.
46  set -e
47  return $CHECKOUT_RETURN_CODE
48}
49
50function cmake_llvm {
51  extra_args="$@"
52  cmake -G "Ninja" \
53      -DLIBCXX_ENABLE_SHARED=OFF \
54      -DLIBCXX_ENABLE_STATIC_ABI_LIBRARY=ON \
55      -DLIBCXXABI_ENABLE_SHARED=OFF \
56      -DCMAKE_BUILD_TYPE=Release \
57      -DLLVM_TARGETS_TO_BUILD="$TARGET_TO_BUILD" \
58      -DLLVM_ENABLE_PROJECTS="$PROJECTS_TO_BUILD" \
59      -DLLVM_BINUTILS_INCDIR="/usr/include/" \
60      $extra_args \
61      $LLVM_SRC/llvm
62}
63
64# Use chromium's clang revision
65mkdir $SRC/chromium_tools
66cd $SRC/chromium_tools
67git clone https://chromium.googlesource.com/chromium/src/tools/clang --depth 1
68cd clang
69
70LLVM_SRC=$SRC/llvm-project
71
72# For manual bumping.
73OUR_LLVM_REVISION=llvmorg-12-init-17251-g6de48655
74
75# To allow for manual downgrades. Set to 0 to use Chrome's clang version (i.e.
76# *not* force a manual downgrade). Set to 1 to force a manual downgrade.
77FORCE_OUR_REVISION=1
78LLVM_REVISION=$(grep -Po "CLANG_REVISION = '\K([^']+)" scripts/update.py)
79
80clone_with_retries https://github.com/llvm/llvm-project.git $LLVM_SRC
81
82set +e
83git -C $LLVM_SRC merge-base --is-ancestor $OUR_LLVM_REVISION $LLVM_REVISION
84IS_OUR_REVISION_ANCESTOR_RETCODE=$?
85set -e
86
87# Use our revision if specified by FORCE_OUR_REVISION or if our revision is a
88# later revision than Chrome's (i.e. not an ancestor of Chrome's).
89if [ $IS_OUR_REVISION_ANCESTOR_RETCODE -ne 0 ] || [ $FORCE_OUR_REVISION -eq 1 ] ; then
90  LLVM_REVISION=$OUR_LLVM_REVISION
91fi
92
93git -C $LLVM_SRC checkout $LLVM_REVISION
94echo "Using LLVM revision: $LLVM_REVISION"
95
96# Build & install.
97mkdir -p $WORK/llvm-stage2 $WORK/llvm-stage1
98python3 $SRC/chromium_tools/clang/scripts/update.py --output-dir $WORK/llvm-stage1
99
100TARGET_TO_BUILD=
101case $(uname -m) in
102    x86_64)
103        TARGET_TO_BUILD=X86
104        ;;
105    aarch64)
106        TARGET_TO_BUILD=AArch64
107        ;;
108    *)
109        echo "Error: unsupported target $(uname -m)"
110        exit 1
111        ;;
112esac
113
114PROJECTS_TO_BUILD="libcxx;libcxxabi;compiler-rt;clang;lld"
115
116cd $WORK/llvm-stage2
117export CC=$WORK/llvm-stage1/bin/clang
118export CXX=$WORK/llvm-stage1/bin/clang++
119cmake_llvm
120ninja -j $NPROC
121ninja install
122rm -rf $WORK/llvm-stage1 $WORK/llvm-stage2
123
124# Use the clang we just built from now on.
125CMAKE_EXTRA_ARGS="-DCMAKE_C_COMPILER=clang -DCMAKE_CXX_COMPILER=clang++"
126
127# 32-bit libraries.
128mkdir -p $WORK/i386
129cd $WORK/i386
130cmake_llvm $CMAKE_EXTRA_ARGS \
131    -DCMAKE_INSTALL_PREFIX=/usr/i386/ \
132    -DCMAKE_C_FLAGS="-m32" \
133    -DCMAKE_CXX_FLAGS="-m32"
134
135ninja -j $NPROC cxx
136ninja install-cxx
137rm -rf $WORK/i386
138
139# MemorySanitizer instrumented libraries.
140mkdir -p $WORK/msan
141cd $WORK/msan
142
143# https://github.com/google/oss-fuzz/issues/1099
144cat <<EOF > $WORK/msan/blocklist.txt
145fun:__gxx_personality_*
146EOF
147
148cmake_llvm $CMAKE_EXTRA_ARGS \
149    -DLLVM_USE_SANITIZER=Memory \
150    -DCMAKE_INSTALL_PREFIX=/usr/msan/ \
151    -DCMAKE_CXX_FLAGS="-fsanitize-blacklist=$WORK/msan/blocklist.txt"
152
153ninja -j $NPROC cxx
154ninja install-cxx
155rm -rf $WORK/msan
156
157# DataFlowSanitizer instrumented libraries.
158mkdir -p $WORK/dfsan
159cd $WORK/dfsan
160
161cmake_llvm $CMAKE_EXTRA_ARGS \
162    -DLLVM_USE_SANITIZER=DataFlow \
163    -DCMAKE_INSTALL_PREFIX=/usr/dfsan/
164
165ninja -j $NPROC cxx cxxabi
166ninja install-cxx install-cxxabi
167rm -rf $WORK/dfsan
168
169# libFuzzer sources.
170cp -r $LLVM_SRC/compiler-rt/lib/fuzzer $SRC/libfuzzer
171
172# Cleanup
173rm -rf $LLVM_SRC
174rm -rf $SRC/chromium_tools
175apt-get remove --purge -y $LLVM_DEP_PACKAGES
176apt-get autoremove -y
177
178# Delete unneeded parts of LLVM to reduce image size.
179# See https://github.com/google/oss-fuzz/issues/5170
180LLVM_TOOLS_TMPDIR=/tmp/llvm-tools
181mkdir $LLVM_TOOLS_TMPDIR
182# Move binaries with llvm- prefix that we want into LLVM_TOOLS_TMPDIR
183mv \
184  /usr/local/bin/llvm-ar \
185  /usr/local/bin/llvm-as \
186  /usr/local/bin/llvm-config \
187  /usr/local/bin/llvm-cov \
188  /usr/local/bin/llvm-objcopy \
189  /usr/local/bin/llvm-profdata \
190  /usr/local/bin/llvm-ranlib \
191  /usr/local/bin/llvm-symbolizer \
192  /usr/local/bin/llvm-undname \
193  $LLVM_TOOLS_TMPDIR
194# Delete remaining llvm- binaries.
195rm -rf /usr/local/bin/llvm-*
196# Restore the llvm- binaries we want to keep.
197mv $LLVM_TOOLS_TMPDIR/* /usr/local/bin/
198rm -rf $LLVM_TOOLS_TMPDIR
199
200# Remove binaries from LLVM buld that we don't need.
201rm -f \
202  /usr/local/bin/bugpoint \
203  /usr/local/bin/llc \
204  /usr/local/bin/lli \
205  /usr/local/bin/clang-check \
206  /usr/local/bin/clang-refactor \
207  /usr/local/bin/clang-offload-wrapper \
208  /usr/local/bin/clang-offload-bundler \
209  /usr/local/bin/clang-check \
210  /usr/local/bin/clang-refactor \
211  /usr/local/bin/c-index-test \
212  /usr/local/bin/clang-rename \
213  /usr/local/bin/clang-scan-deps \
214  /usr/local/bin/clang-extdef-mapping \
215  /usr/local/bin/diagtool \
216  /usr/local/bin/sanstats \
217  /usr/local/bin/dsymutil \
218  /usr/local/bin/verify-uselistorder \
219  /usr/local/bin/clang-format
220
221# Remove unneeded clang libs, CMake files from LLVM build, lld libs, and the
222# libraries.
223# Note: we need fuzzer_no_main libraries for atheris. Don't delete.
224rm -rf \
225  /usr/local/lib/libclang* \
226  /usr/local/lib/liblld* \
227  /usr/local/lib/cmake/
228