1# Copyright 2018 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6from . import util 7 8 9def build_command_buffer(api, chrome_dir, skia_dir, out): 10 api.run(api.python, 'build command_buffer', 11 script=skia_dir.join('tools', 'build_command_buffer.py'), 12 args=[ 13 '--chrome-dir', chrome_dir, 14 '--output-dir', out, 15 '--extra-gn-args', 'mac_sdk_min="10.13"', 16 '--no-sync', '--no-hooks', '--make-output-dir']) 17 18 19def compile_swiftshader(api, extra_tokens, swiftshader_root, cc, cxx, out): 20 """Build SwiftShader with CMake. 21 22 Building SwiftShader works differently from any other Skia third_party lib. 23 See discussion in skia:7671 for more detail. 24 25 Args: 26 swiftshader_root: root of the SwiftShader checkout. 27 cc, cxx: compiler binaries to use 28 out: target directory for libEGL.so and libGLESv2.so 29 """ 30 swiftshader_opts = [ 31 '-DSWIFTSHADER_BUILD_TESTS=OFF', 32 '-DSWIFTSHADER_WARNINGS_AS_ERRORS=OFF', 33 '-DREACTOR_ENABLE_MEMORY_SANITIZER_INSTRUMENTATION=OFF', # Way too slow. 34 ] 35 cmake_bin = str(api.vars.workdir.join('cmake_linux', 'bin')) 36 env = { 37 'CC': cc, 38 'CXX': cxx, 39 'PATH': '%%(PATH)s:%s' % cmake_bin, 40 # We arrange our MSAN/TSAN prebuilts a little differently than 41 # SwiftShader's CMakeLists.txt expects, so we'll just keep our custom 42 # setup (everything mentioning libcxx below) and point SwiftShader's 43 # CMakeLists.txt at a harmless non-existent path. 44 'SWIFTSHADER_MSAN_INSTRUMENTED_LIBCXX_PATH': '/totally/phony/path', 45 } 46 47 # Extra flags for MSAN/TSAN, if necessary. 48 san = None 49 if 'MSAN' in extra_tokens: 50 san = ('msan','memory') 51 elif 'TSAN' in extra_tokens: 52 san = ('tsan','thread') 53 54 if san: 55 short,full = san 56 clang_linux = str(api.vars.workdir.join('clang_linux')) 57 libcxx = clang_linux + '/' + short 58 cflags = ' '.join([ 59 '-fsanitize=' + full, 60 '-stdlib=libc++', 61 '-L%s/lib' % libcxx, 62 '-lc++abi', 63 '-I%s/include' % libcxx, 64 '-I%s/include/c++/v1' % libcxx, 65 ]) 66 swiftshader_opts.extend([ 67 '-DSWIFTSHADER_{}=ON'.format(short.upper()), 68 '-DCMAKE_C_FLAGS=%s' % cflags, 69 '-DCMAKE_CXX_FLAGS=%s' % cflags, 70 ]) 71 72 # Build SwiftShader. 73 api.file.ensure_directory('makedirs swiftshader_out', out) 74 with api.context(cwd=out, env=env): 75 api.run(api.step, 'swiftshader cmake', 76 cmd=['cmake'] + swiftshader_opts + [swiftshader_root, '-GNinja']) 77 api.run(api.step, 'swiftshader ninja', 78 cmd=['ninja', '-C', out, 'libEGL.so', 'libGLESv2.so']) 79 80 81def compile_fn(api, checkout_root, out_dir): 82 skia_dir = checkout_root.join('skia') 83 compiler = api.vars.builder_cfg.get('compiler', '') 84 configuration = api.vars.builder_cfg.get('configuration', '') 85 extra_tokens = api.vars.extra_tokens 86 os = api.vars.builder_cfg.get('os', '') 87 target_arch = api.vars.builder_cfg.get('target_arch', '') 88 89 clang_linux = str(api.vars.workdir.join('clang_linux')) 90 win_toolchain = str(api.vars.workdir.join('win_toolchain')) 91 92 cc, cxx, ccache = None, None, None 93 extra_cflags = [] 94 extra_ldflags = [] 95 args = {'werror': 'true'} 96 env = {} 97 98 if os == 'Mac': 99 # XCode build is listed in parentheses after the version at 100 # https://developer.apple.com/news/releases/, or on Wikipedia here: 101 # https://en.wikipedia.org/wiki/Xcode#Version_comparison_table 102 # Use lowercase letters. 103 # https://chrome-infra-packages.appspot.com/p/infra_internal/ios/xcode 104 XCODE_BUILD_VERSION = '12c33' 105 if compiler == 'Xcode11.4.1': 106 XCODE_BUILD_VERSION = '11e503a' 107 extra_cflags.append( 108 '-DREBUILD_IF_CHANGED_xcode_build_version=%s' % XCODE_BUILD_VERSION) 109 mac_toolchain_cmd = api.vars.workdir.join( 110 'mac_toolchain', 'mac_toolchain') 111 xcode_app_path = api.vars.cache_dir.join('Xcode.app') 112 # Copied from 113 # https://chromium.googlesource.com/chromium/tools/build/+/e19b7d9390e2bb438b566515b141ed2b9ed2c7c2/scripts/slave/recipe_modules/ios/api.py#322 114 with api.step.nest('ensure xcode') as step_result: 115 step_result.presentation.step_text = ( 116 'Ensuring Xcode version %s in %s' % ( 117 XCODE_BUILD_VERSION, xcode_app_path)) 118 install_xcode_cmd = [ 119 mac_toolchain_cmd, 'install', 120 # "ios" is needed for simulator builds 121 # (Build-Mac-Clang-x64-Release-iOS). 122 '-kind', 'ios', 123 '-xcode-version', XCODE_BUILD_VERSION, 124 '-output-dir', xcode_app_path, 125 ] 126 api.step('install xcode', install_xcode_cmd) 127 api.step('select xcode', [ 128 'sudo', 'xcode-select', '-switch', xcode_app_path]) 129 if 'iOS' in extra_tokens: 130 # Can't compile for Metal before 11.0. 131 env['IPHONEOS_DEPLOYMENT_TARGET'] = '11.0' 132 else: 133 # We have some bots on 10.13. 134 env['MACOSX_DEPLOYMENT_TARGET'] = '10.13' 135 136 if 'CheckGeneratedFiles' in extra_tokens: 137 compiler = 'Clang' 138 args['skia_compile_processors'] = 'true' 139 args['skia_compile_sksl_tests'] = 'true' 140 args['skia_generate_workarounds'] = 'true' 141 142 # ccache + clang-tidy.sh chokes on the argument list. 143 if (api.vars.is_linux or os == 'Mac' or os == 'Mac10.15.5' or os == 'Mac10.15.7') and 'Tidy' not in extra_tokens: 144 if api.vars.is_linux: 145 ccache = api.vars.workdir.join('ccache_linux', 'bin', 'ccache') 146 # As of 2020-02-07, the sum of each Debian10-Clang-x86 147 # non-flutter/android/chromebook build takes less than 75G cache space. 148 env['CCACHE_MAXSIZE'] = '75G' 149 else: 150 ccache = api.vars.workdir.join('ccache_mac', 'bin', 'ccache') 151 # As of 2020-02-10, the sum of each Build-Mac-Clang- non-android build 152 # takes ~30G cache space. 153 env['CCACHE_MAXSIZE'] = '50G' 154 155 args['cc_wrapper'] = '"%s"' % ccache 156 157 env['CCACHE_DIR'] = api.vars.cache_dir.join('ccache') 158 env['CCACHE_MAXFILES'] = '0' 159 # Compilers are unpacked from cipd with bogus timestamps, only contribute 160 # compiler content to hashes. If Ninja ever uses absolute paths to changing 161 # directories we'll also need to set a CCACHE_BASEDIR. 162 env['CCACHE_COMPILERCHECK'] = 'content' 163 164 if compiler == 'Clang' and api.vars.is_linux: 165 cc = clang_linux + '/bin/clang' 166 cxx = clang_linux + '/bin/clang++' 167 extra_cflags .append('-B%s/bin' % clang_linux) 168 extra_ldflags.append('-B%s/bin' % clang_linux) 169 extra_ldflags.append('-fuse-ld=lld') 170 extra_cflags.append('-DPLACEHOLDER_clang_linux_version=%s' % 171 api.run.asset_version('clang_linux', skia_dir)) 172 if 'Static' in extra_tokens: 173 extra_ldflags.extend(['-static-libstdc++', '-static-libgcc']) 174 175 elif compiler == 'Clang': 176 cc, cxx = 'clang', 'clang++' 177 178 if 'Tidy' in extra_tokens: 179 # Swap in clang-tidy.sh for clang++, but update PATH so it can find clang++. 180 cxx = skia_dir.join("tools/clang-tidy.sh") 181 env['PATH'] = '%s:%%(PATH)s' % (clang_linux + '/bin') 182 # Increase ClangTidy code coverage by enabling features. 183 args.update({ 184 'skia_enable_fontmgr_empty': 'true', 185 'skia_enable_pdf': 'true', 186 'skia_use_expat': 'true', 187 'skia_use_freetype': 'true', 188 'skia_use_vulkan': 'true', 189 }) 190 191 if 'Coverage' in extra_tokens: 192 # See https://clang.llvm.org/docs/SourceBasedCodeCoverage.html for 193 # more info on using llvm to gather coverage information. 194 extra_cflags.append('-fprofile-instr-generate') 195 extra_cflags.append('-fcoverage-mapping') 196 extra_ldflags.append('-fprofile-instr-generate') 197 extra_ldflags.append('-fcoverage-mapping') 198 199 if compiler != 'MSVC' and configuration == 'Debug': 200 extra_cflags.append('-O1') 201 202 if 'Exceptions' in extra_tokens: 203 extra_cflags.append('/EHsc') 204 if 'Fast' in extra_tokens: 205 extra_cflags.extend(['-march=native', '-fomit-frame-pointer', '-O3', 206 '-ffp-contract=off']) 207 208 if len(extra_tokens) == 1 and extra_tokens[0].startswith('SK'): 209 extra_cflags.append('-D' + extra_tokens[0]) 210 # If we're limiting Skia at all, drop skcms to portable code. 211 if 'SK_CPU_LIMIT' in extra_tokens[0]: 212 extra_cflags.append('-DSKCMS_PORTABLE') 213 214 if 'MSAN' in extra_tokens: 215 extra_ldflags.append('-L' + clang_linux + '/msan') 216 elif 'TSAN' in extra_tokens: 217 extra_ldflags.append('-L' + clang_linux + '/tsan') 218 elif api.vars.is_linux: 219 extra_ldflags.append('-L' + clang_linux + '/lib') 220 221 if configuration != 'Debug': 222 args['is_debug'] = 'false' 223 if 'Dawn' in extra_tokens: 224 args['skia_use_dawn'] = 'true' 225 args['skia_use_gl'] = 'false' 226 # Dawn imports jinja2, which imports markupsafe. Along with DEPS, make it 227 # importable. 228 env['PYTHONPATH'] = api.path.pathsep.join([ 229 str(skia_dir.join('third_party', 'externals')), '%%(PYTHONPATH)s']) 230 if 'ANGLE' in extra_tokens: 231 args['skia_use_angle'] = 'true' 232 if 'SwiftShader' in extra_tokens: 233 swiftshader_root = skia_dir.join('third_party', 'externals', 'swiftshader') 234 swiftshader_out = out_dir.join('swiftshader_out') 235 compile_swiftshader(api, extra_tokens, swiftshader_root, cc, cxx, swiftshader_out) 236 args['skia_use_egl'] = 'true' 237 extra_cflags.extend([ 238 '-DGR_EGL_TRY_GLES3_THEN_GLES2', 239 '-I%s' % skia_dir.join( 240 'third_party', 'externals', 'egl-registry', 'api'), 241 '-I%s' % skia_dir.join( 242 'third_party', 'externals', 'opengl-registry', 'api'), 243 ]) 244 extra_ldflags.extend([ 245 '-L%s' % swiftshader_out, 246 ]) 247 if 'CommandBuffer' in extra_tokens: 248 # CommandBuffer runs against GLES version of CommandBuffer also, so 249 # include both. 250 args.update({ 251 'skia_gl_standard': '""', 252 }) 253 chrome_dir = checkout_root 254 api.run.run_once(build_command_buffer, api, chrome_dir, skia_dir, out_dir) 255 if 'MSAN' in extra_tokens: 256 args['skia_use_fontconfig'] = 'false' 257 if 'ASAN' in extra_tokens: 258 args['skia_enable_spirv_validation'] = 'false' 259 if 'NoDEPS' in extra_tokens: 260 args.update({ 261 'is_official_build': 'true', 262 'skia_enable_fontmgr_empty': 'true', 263 'skia_enable_gpu': 'true', 264 265 'skia_enable_pdf': 'false', 266 'skia_use_expat': 'false', 267 'skia_use_freetype': 'false', 268 'skia_use_harfbuzz': 'false', 269 'skia_use_libjpeg_turbo_decode': 'false', 270 'skia_use_libjpeg_turbo_encode': 'false', 271 'skia_use_libpng_decode': 'false', 272 'skia_use_libpng_encode': 'false', 273 'skia_use_libwebp_decode': 'false', 274 'skia_use_libwebp_encode': 'false', 275 'skia_use_vulkan': 'false', 276 'skia_use_zlib': 'false', 277 }) 278 if 'Shared' in extra_tokens: 279 args['is_component_build'] = 'true' 280 if 'Vulkan' in extra_tokens and not 'Android' in extra_tokens: 281 args['skia_use_vulkan'] = 'true' 282 args['skia_enable_vulkan_debug_layers'] = 'true' 283 args['skia_use_gl'] = 'false' 284 if 'Direct3D' in extra_tokens: 285 args['skia_use_direct3d'] = 'true' 286 args['skia_use_gl'] = 'false' 287 if 'Metal' in extra_tokens: 288 args['skia_use_metal'] = 'true' 289 args['skia_use_gl'] = 'false' 290 if 'OpenCL' in extra_tokens: 291 args['skia_use_opencl'] = 'true' 292 if api.vars.is_linux: 293 extra_cflags.append( 294 '-isystem%s' % api.vars.workdir.join('opencl_headers')) 295 extra_ldflags.append( 296 '-L%s' % api.vars.workdir.join('opencl_ocl_icd_linux')) 297 elif 'Win' in os: 298 extra_cflags.append( 299 '-imsvc%s' % api.vars.workdir.join('opencl_headers')) 300 extra_ldflags.append( 301 '/LIBPATH:%s' % 302 skia_dir.join('third_party', 'externals', 'opencl-lib', '3-0', 'lib', 303 'x86_64')) 304 if 'iOS' in extra_tokens: 305 # Bots use Chromium signing cert. 306 args['skia_ios_identity'] = '".*GS9WA.*"' 307 # Get mobileprovision via the CIPD package. 308 args['skia_ios_profile'] = '"%s"' % api.vars.workdir.join( 309 'provisioning_profile_ios', 310 'Upstream_Testing_Provisioning_Profile.mobileprovision') 311 if compiler == 'Clang' and 'Win' in os: 312 args['clang_win'] = '"%s"' % api.vars.workdir.join('clang_win') 313 extra_cflags.append('-DPLACEHOLDER_clang_win_version=%s' % 314 api.run.asset_version('clang_win', skia_dir)) 315 316 sanitize = '' 317 for t in extra_tokens: 318 if t.endswith('SAN'): 319 sanitize = t 320 if api.vars.is_linux and t == 'ASAN': 321 # skia:8712 and skia:8713 322 extra_cflags.append('-DSK_ENABLE_SCOPED_LSAN_SUPPRESSIONS') 323 if 'SafeStack' in extra_tokens: 324 assert sanitize == '' 325 sanitize = 'safe-stack' 326 327 if 'Wuffs' in extra_tokens: 328 args['skia_use_wuffs'] = 'true' 329 330 for (k,v) in { 331 'cc': cc, 332 'cxx': cxx, 333 'sanitize': sanitize, 334 'target_cpu': target_arch, 335 'target_os': 'ios' if 'iOS' in extra_tokens else '', 336 'win_sdk': win_toolchain + '/win_sdk' if 'Win' in os else '', 337 'win_vc': win_toolchain + '/VC' if 'Win' in os else '', 338 }.iteritems(): 339 if v: 340 args[k] = '"%s"' % v 341 if extra_cflags: 342 args['extra_cflags'] = repr(extra_cflags).replace("'", '"') 343 if extra_ldflags: 344 args['extra_ldflags'] = repr(extra_ldflags).replace("'", '"') 345 346 gn_args = ' '.join('%s=%s' % (k,v) for (k,v) in sorted(args.iteritems())) 347 gn = skia_dir.join('bin', 'gn') 348 349 with api.context(cwd=skia_dir): 350 api.run(api.python, 351 'fetch-gn', 352 script=skia_dir.join('bin', 'fetch-gn'), 353 infra_step=True) 354 if 'CheckGeneratedFiles' in extra_tokens: 355 env['PATH'] = '%s:%%(PATH)s' % skia_dir.join('bin') 356 api.run(api.python, 357 'fetch-clang-format', 358 script=skia_dir.join('bin', 'fetch-clang-format'), 359 infra_step=True) 360 361 with api.env(env): 362 if ccache: 363 api.run(api.step, 'ccache stats-start', cmd=[ccache, '-s']) 364 api.run(api.step, 'gn gen', 365 cmd=[gn, 'gen', out_dir, '--args=' + gn_args]) 366 api.run(api.step, 'ninja', cmd=['ninja', '-C', out_dir]) 367 if ccache: 368 api.run(api.step, 'ccache stats-end', cmd=[ccache, '-s']) 369 370 371def copy_build_products(api, src, dst): 372 util.copy_listed_files(api, src, dst, util.DEFAULT_BUILD_PRODUCTS) 373 extra_tokens = api.vars.extra_tokens 374 os = api.vars.builder_cfg.get('os', '') 375 376 if 'SwiftShader' in extra_tokens: 377 util.copy_listed_files(api, 378 src.join('swiftshader_out'), 379 api.vars.swarming_out_dir.join('swiftshader_out'), 380 util.DEFAULT_BUILD_PRODUCTS) 381 382 if os == 'Mac' and any('SAN' in t for t in extra_tokens): 383 # The XSAN dylibs are in 384 # Xcode.app/Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib 385 # /clang/11.0.0/lib/darwin, where 11.0.0 could change in future versions. 386 xcode_clang_ver_dirs = api.file.listdir( 387 'find XCode Clang version', 388 api.vars.cache_dir.join( 389 'Xcode.app', 'Contents', 'Developer', 'Toolchains', 390 'XcodeDefault.xctoolchain', 'usr', 'lib', 'clang'), 391 test_data=['11.0.0']) 392 assert len(xcode_clang_ver_dirs) == 1 393 dylib_dir = xcode_clang_ver_dirs[0].join('lib', 'darwin') 394 dylibs = api.file.glob_paths('find xSAN dylibs', dylib_dir, 395 'libclang_rt.*san_osx_dynamic.dylib', 396 test_data=[ 397 'libclang_rt.asan_osx_dynamic.dylib', 398 'libclang_rt.tsan_osx_dynamic.dylib', 399 'libclang_rt.ubsan_osx_dynamic.dylib', 400 ]) 401 for f in dylibs: 402 api.file.copy('copy %s' % api.path.basename(f), f, dst) 403