1#!/usr/bin/env python 2# Copyright 2016 gRPC authors. 3# 4# Licensed under the Apache License, Version 2.0 (the "License"); 5# you may not use this file except in compliance with the License. 6# You may obtain a copy of the License at 7# 8# http://www.apache.org/licenses/LICENSE-2.0 9# 10# Unless required by applicable law or agreed to in writing, software 11# distributed under the License is distributed on an "AS IS" BASIS, 12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 13# See the License for the specific language governing permissions and 14# limitations under the License. 15"""Definition of targets to build artifacts.""" 16 17import os.path 18import random 19import string 20import sys 21 22sys.path.insert(0, os.path.abspath('..')) 23import python_utils.jobset as jobset 24 25 26def create_docker_jobspec(name, 27 dockerfile_dir, 28 shell_command, 29 environ={}, 30 flake_retries=0, 31 timeout_retries=0, 32 timeout_seconds=30 * 60, 33 docker_base_image=None, 34 extra_docker_args=None, 35 verbose_success=False): 36 """Creates jobspec for a task running under docker.""" 37 environ = environ.copy() 38 environ['RUN_COMMAND'] = shell_command 39 environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name 40 41 docker_args = [] 42 for k, v in environ.items(): 43 docker_args += ['-e', '%s=%s' % (k, v)] 44 docker_env = { 45 'DOCKERFILE_DIR': dockerfile_dir, 46 'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh', 47 'OUTPUT_DIR': 'artifacts' 48 } 49 50 if docker_base_image is not None: 51 docker_env['DOCKER_BASE_IMAGE'] = docker_base_image 52 if extra_docker_args is not None: 53 docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args 54 jobspec = jobset.JobSpec( 55 cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] + 56 docker_args, 57 environ=docker_env, 58 shortname='build_artifact.%s' % (name), 59 timeout_seconds=timeout_seconds, 60 flake_retries=flake_retries, 61 timeout_retries=timeout_retries, 62 verbose_success=verbose_success) 63 return jobspec 64 65 66def create_jobspec(name, 67 cmdline, 68 environ={}, 69 shell=False, 70 flake_retries=0, 71 timeout_retries=0, 72 timeout_seconds=30 * 60, 73 use_workspace=False, 74 cpu_cost=1.0, 75 verbose_success=False): 76 """Creates jobspec.""" 77 environ = environ.copy() 78 if use_workspace: 79 environ['WORKSPACE_NAME'] = 'workspace_%s' % name 80 environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name) 81 cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh' 82 ] + cmdline 83 else: 84 environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name) 85 86 jobspec = jobset.JobSpec(cmdline=cmdline, 87 environ=environ, 88 shortname='build_artifact.%s' % (name), 89 timeout_seconds=timeout_seconds, 90 flake_retries=flake_retries, 91 timeout_retries=timeout_retries, 92 shell=shell, 93 cpu_cost=cpu_cost, 94 verbose_success=verbose_success) 95 return jobspec 96 97 98_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.10' 99 100_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'} 101 102 103class PythonArtifact: 104 """Builds Python artifacts.""" 105 106 def __init__(self, platform, arch, py_version): 107 self.name = 'python_%s_%s_%s' % (platform, arch, py_version) 108 self.platform = platform 109 self.arch = arch 110 self.labels = ['artifact', 'python', platform, arch, py_version] 111 self.py_version = py_version 112 if 'manylinux' in platform: 113 self.labels.append('linux') 114 115 def pre_build_jobspecs(self): 116 return [] 117 118 def build_jobspec(self): 119 environ = {} 120 if self.platform == 'linux_extra': 121 # Raspberry Pi build 122 environ['PYTHON'] = '/usr/local/bin/python{}'.format( 123 self.py_version) 124 environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version) 125 # https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9 126 # A QEMU bug causes submodule update to hang, so we copy directly 127 environ['RELATIVE_COPY_PATH'] = '.' 128 # Parallel builds are counterproductive in emulated environment 129 environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'] = '1' 130 extra_args = ' --entrypoint=/usr/bin/qemu-arm-static ' 131 return create_docker_jobspec( 132 self.name, 133 'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch), 134 'tools/run_tests/artifacts/build_artifact_python.sh', 135 environ=environ, 136 timeout_seconds=60 * 60 * 5, 137 docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch), 138 extra_docker_args=extra_args) 139 elif 'manylinux' in self.platform: 140 if self.arch == 'x86': 141 environ['SETARCH_CMD'] = 'linux32' 142 # Inside the manylinux container, the python installations are located in 143 # special places... 144 environ['PYTHON'] = '/opt/python/{}/bin/python'.format( 145 self.py_version) 146 environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version) 147 environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE' 148 environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE' 149 return create_docker_jobspec( 150 self.name, 151 # NOTE(rbellevi): Do *not* update this without also ensuring the 152 # base_docker_image attribute is accurate. 153 'tools/dockerfile/grpc_artifact_python_%s_%s' % 154 (self.platform, self.arch), 155 'tools/run_tests/artifacts/build_artifact_python.sh', 156 environ=environ, 157 timeout_seconds=60 * 60) 158 elif self.platform == 'windows': 159 if 'Python27' in self.py_version: 160 environ['EXT_COMPILER'] = 'mingw32' 161 else: 162 environ['EXT_COMPILER'] = 'msvc' 163 # For some reason, the batch script %random% always runs with the same 164 # seed. We create a random temp-dir here 165 dir = ''.join( 166 random.choice(string.ascii_uppercase) for _ in range(10)) 167 return create_jobspec(self.name, [ 168 'tools\\run_tests\\artifacts\\build_artifact_python.bat', 169 self.py_version, '32' if self.arch == 'x86' else '64' 170 ], 171 environ=environ, 172 timeout_seconds=45 * 60, 173 use_workspace=True) 174 else: 175 environ['PYTHON'] = self.py_version 176 environ['SKIP_PIP_INSTALL'] = 'TRUE' 177 return create_jobspec( 178 self.name, 179 ['tools/run_tests/artifacts/build_artifact_python.sh'], 180 environ=environ, 181 timeout_seconds=60 * 60 * 2, 182 use_workspace=True) 183 184 def __str__(self): 185 return self.name 186 187 188class RubyArtifact: 189 """Builds ruby native gem.""" 190 191 def __init__(self, platform, arch): 192 self.name = 'ruby_native_gem_%s_%s' % (platform, arch) 193 self.platform = platform 194 self.arch = arch 195 self.labels = ['artifact', 'ruby', platform, arch] 196 197 def pre_build_jobspecs(self): 198 return [] 199 200 def build_jobspec(self): 201 # Ruby build uses docker internally and docker cannot be nested. 202 # We are using a custom workspace instead. 203 return create_jobspec( 204 self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'], 205 use_workspace=True, 206 timeout_seconds=60 * 60) 207 208 209class CSharpExtArtifact: 210 """Builds C# native extension library""" 211 212 def __init__(self, platform, arch, arch_abi=None): 213 self.name = 'csharp_ext_%s_%s' % (platform, arch) 214 self.platform = platform 215 self.arch = arch 216 self.arch_abi = arch_abi 217 self.labels = ['artifact', 'csharp', platform, arch] 218 if arch_abi: 219 self.name += '_%s' % arch_abi 220 self.labels.append(arch_abi) 221 222 def pre_build_jobspecs(self): 223 return [] 224 225 def build_jobspec(self): 226 if self.arch == 'android': 227 return create_docker_jobspec( 228 self.name, 229 'tools/dockerfile/grpc_artifact_android_ndk', 230 'tools/run_tests/artifacts/build_artifact_csharp_android.sh', 231 environ={'ANDROID_ABI': self.arch_abi}) 232 elif self.arch == 'ios': 233 return create_jobspec( 234 self.name, 235 ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'], 236 timeout_seconds=45 * 60, 237 use_workspace=True) 238 elif self.platform == 'windows': 239 return create_jobspec(self.name, [ 240 'tools\\run_tests\\artifacts\\build_artifact_csharp.bat', 241 self.arch 242 ], 243 use_workspace=True) 244 else: 245 if self.platform == 'linux': 246 cmake_arch_option = '' # x64 is the default architecture 247 if self.arch == 'x86': 248 # TODO(jtattermusch): more work needed to enable 249 # boringssl assembly optimizations for 32-bit linux. 250 # Problem: currently we are building the artifact under 251 # 32-bit docker image, but CMAKE_SYSTEM_PROCESSOR is still 252 # set to x86_64, so the resulting boringssl binary 253 # would have undefined symbols. 254 cmake_arch_option = '-DOPENSSL_NO_ASM=ON' 255 return create_docker_jobspec( 256 self.name, 257 'tools/dockerfile/grpc_artifact_centos6_{}'.format( 258 self.arch), 259 'tools/run_tests/artifacts/build_artifact_csharp.sh', 260 environ={'CMAKE_ARCH_OPTION': cmake_arch_option}) 261 else: 262 cmake_arch_option = '' # x64 is the default architecture 263 if self.arch == 'x86': 264 cmake_arch_option = '-DCMAKE_OSX_ARCHITECTURES=i386' 265 return create_jobspec( 266 self.name, 267 ['tools/run_tests/artifacts/build_artifact_csharp.sh'], 268 environ={'CMAKE_ARCH_OPTION': cmake_arch_option}, 269 use_workspace=True) 270 271 def __str__(self): 272 return self.name 273 274 275class PHPArtifact: 276 """Builds PHP PECL package""" 277 278 def __init__(self, platform, arch): 279 self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch) 280 self.platform = platform 281 self.arch = arch 282 self.labels = ['artifact', 'php', platform, arch] 283 284 def pre_build_jobspecs(self): 285 return [] 286 287 def build_jobspec(self): 288 return create_docker_jobspec( 289 self.name, 290 'tools/dockerfile/test/php73_zts_stretch_{}'.format(self.arch), 291 'tools/run_tests/artifacts/build_artifact_php.sh') 292 293 294class ProtocArtifact: 295 """Builds protoc and protoc-plugin artifacts""" 296 297 def __init__(self, platform, arch): 298 self.name = 'protoc_%s_%s' % (platform, arch) 299 self.platform = platform 300 self.arch = arch 301 self.labels = ['artifact', 'protoc', platform, arch] 302 303 def pre_build_jobspecs(self): 304 return [] 305 306 def build_jobspec(self): 307 if self.platform != 'windows': 308 environ = {'CXXFLAGS': '', 'LDFLAGS': ''} 309 if self.platform == 'linux': 310 environ['LDFLAGS'] += ' -static-libgcc -static-libstdc++ -s' 311 return create_docker_jobspec( 312 self.name, 313 'tools/dockerfile/grpc_artifact_centos6_{}'.format( 314 self.arch), 315 'tools/run_tests/artifacts/build_artifact_protoc.sh', 316 environ=environ) 317 else: 318 environ[ 319 'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG 320 return create_jobspec( 321 self.name, 322 ['tools/run_tests/artifacts/build_artifact_protoc.sh'], 323 environ=environ, 324 timeout_seconds=60 * 60, 325 use_workspace=True) 326 else: 327 generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015' 328 return create_jobspec( 329 self.name, 330 ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'], 331 environ={'generator': generator}, 332 use_workspace=True) 333 334 def __str__(self): 335 return self.name 336 337 338def targets(): 339 """Gets list of supported targets""" 340 return [ 341 ProtocArtifact('linux', 'x64'), 342 ProtocArtifact('linux', 'x86'), 343 ProtocArtifact('macos', 'x64'), 344 ProtocArtifact('windows', 'x64'), 345 ProtocArtifact('windows', 'x86'), 346 CSharpExtArtifact('linux', 'x64'), 347 CSharpExtArtifact('macos', 'x64'), 348 CSharpExtArtifact('windows', 'x64'), 349 CSharpExtArtifact('windows', 'x86'), 350 CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'), 351 CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'), 352 CSharpExtArtifact('linux', 'android', arch_abi='x86'), 353 CSharpExtArtifact('macos', 'ios'), 354 PythonArtifact('manylinux2014', 'x64', 'cp35-cp35m'), 355 PythonArtifact('manylinux2014', 'x64', 'cp36-cp36m'), 356 PythonArtifact('manylinux2014', 'x64', 'cp37-cp37m'), 357 PythonArtifact('manylinux2014', 'x64', 'cp38-cp38'), 358 PythonArtifact('manylinux2014', 'x64', 'cp39-cp39'), 359 PythonArtifact('manylinux2014', 'x86', 'cp35-cp35m'), 360 PythonArtifact('manylinux2014', 'x86', 'cp36-cp36m'), 361 PythonArtifact('manylinux2014', 'x86', 'cp37-cp37m'), 362 PythonArtifact('manylinux2014', 'x86', 'cp38-cp38'), 363 PythonArtifact('manylinux2014', 'x86', 'cp39-cp39'), 364 PythonArtifact('manylinux2010', 'x64', 'cp27-cp27m'), 365 PythonArtifact('manylinux2010', 'x64', 'cp27-cp27mu'), 366 PythonArtifact('manylinux2010', 'x64', 'cp35-cp35m'), 367 PythonArtifact('manylinux2010', 'x64', 'cp36-cp36m'), 368 PythonArtifact('manylinux2010', 'x64', 'cp37-cp37m'), 369 PythonArtifact('manylinux2010', 'x64', 'cp38-cp38'), 370 PythonArtifact('manylinux2010', 'x64', 'cp39-cp39'), 371 PythonArtifact('manylinux2010', 'x86', 'cp27-cp27m'), 372 PythonArtifact('manylinux2010', 'x86', 'cp27-cp27mu'), 373 PythonArtifact('manylinux2010', 'x86', 'cp35-cp35m'), 374 PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'), 375 PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'), 376 PythonArtifact('manylinux2010', 'x86', 'cp38-cp38'), 377 PythonArtifact('manylinux2010', 'x86', 'cp39-cp39'), 378 PythonArtifact('linux_extra', 'armv7', '2.7'), 379 PythonArtifact('linux_extra', 'armv7', '3.5'), 380 PythonArtifact('linux_extra', 'armv7', '3.6'), 381 PythonArtifact('linux_extra', 'armv6', '2.7'), 382 PythonArtifact('linux_extra', 'armv6', '3.5'), 383 PythonArtifact('linux_extra', 'armv6', '3.6'), 384 PythonArtifact('macos', 'x64', 'python2.7'), 385 PythonArtifact('macos', 'x64', 'python3.5'), 386 PythonArtifact('macos', 'x64', 'python3.6'), 387 PythonArtifact('macos', 'x64', 'python3.7'), 388 PythonArtifact('macos', 'x64', 'python3.8'), 389 PythonArtifact('macos', 'x64', 'python3.9'), 390 PythonArtifact('windows', 'x86', 'Python27_32bit'), 391 PythonArtifact('windows', 'x86', 'Python35_32bit'), 392 PythonArtifact('windows', 'x86', 'Python36_32bit'), 393 PythonArtifact('windows', 'x86', 'Python37_32bit'), 394 PythonArtifact('windows', 'x86', 'Python38_32bit'), 395 PythonArtifact('windows', 'x86', 'Python39_32bit'), 396 PythonArtifact('windows', 'x64', 'Python27'), 397 PythonArtifact('windows', 'x64', 'Python35'), 398 PythonArtifact('windows', 'x64', 'Python36'), 399 PythonArtifact('windows', 'x64', 'Python37'), 400 PythonArtifact('windows', 'x64', 'Python38'), 401 PythonArtifact('windows', 'x64', 'Python39'), 402 RubyArtifact('linux', 'x64'), 403 RubyArtifact('macos', 'x64'), 404 PHPArtifact('linux', 'x64') 405 ] 406