1#!/usr/bin/env python
2# Copyright 2016 gRPC authors.
3#
4# Licensed under the Apache License, Version 2.0 (the "License");
5# you may not use this file except in compliance with the License.
6# You may obtain a copy of the License at
7#
8#     http://www.apache.org/licenses/LICENSE-2.0
9#
10# Unless required by applicable law or agreed to in writing, software
11# distributed under the License is distributed on an "AS IS" BASIS,
12# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13# See the License for the specific language governing permissions and
14# limitations under the License.
15"""Definition of targets to build artifacts."""
16
17import os.path
18import random
19import string
20import sys
21
22sys.path.insert(0, os.path.abspath('..'))
23import python_utils.jobset as jobset
24
25
26def create_docker_jobspec(name,
27                          dockerfile_dir,
28                          shell_command,
29                          environ={},
30                          flake_retries=0,
31                          timeout_retries=0,
32                          timeout_seconds=30 * 60,
33                          docker_base_image=None,
34                          extra_docker_args=None,
35                          verbose_success=False):
36    """Creates jobspec for a task running under docker."""
37    environ = environ.copy()
38    environ['RUN_COMMAND'] = shell_command
39    environ['ARTIFACTS_OUT'] = 'artifacts/%s' % name
40
41    docker_args = []
42    for k, v in environ.items():
43        docker_args += ['-e', '%s=%s' % (k, v)]
44    docker_env = {
45        'DOCKERFILE_DIR': dockerfile_dir,
46        'DOCKER_RUN_SCRIPT': 'tools/run_tests/dockerize/docker_run.sh',
47        'OUTPUT_DIR': 'artifacts'
48    }
49
50    if docker_base_image is not None:
51        docker_env['DOCKER_BASE_IMAGE'] = docker_base_image
52    if extra_docker_args is not None:
53        docker_env['EXTRA_DOCKER_ARGS'] = extra_docker_args
54    jobspec = jobset.JobSpec(
55        cmdline=['tools/run_tests/dockerize/build_and_run_docker.sh'] +
56        docker_args,
57        environ=docker_env,
58        shortname='build_artifact.%s' % (name),
59        timeout_seconds=timeout_seconds,
60        flake_retries=flake_retries,
61        timeout_retries=timeout_retries,
62        verbose_success=verbose_success)
63    return jobspec
64
65
66def create_jobspec(name,
67                   cmdline,
68                   environ={},
69                   shell=False,
70                   flake_retries=0,
71                   timeout_retries=0,
72                   timeout_seconds=30 * 60,
73                   use_workspace=False,
74                   cpu_cost=1.0,
75                   verbose_success=False):
76    """Creates jobspec."""
77    environ = environ.copy()
78    if use_workspace:
79        environ['WORKSPACE_NAME'] = 'workspace_%s' % name
80        environ['ARTIFACTS_OUT'] = os.path.join('..', 'artifacts', name)
81        cmdline = ['bash', 'tools/run_tests/artifacts/run_in_workspace.sh'
82                  ] + cmdline
83    else:
84        environ['ARTIFACTS_OUT'] = os.path.join('artifacts', name)
85
86    jobspec = jobset.JobSpec(cmdline=cmdline,
87                             environ=environ,
88                             shortname='build_artifact.%s' % (name),
89                             timeout_seconds=timeout_seconds,
90                             flake_retries=flake_retries,
91                             timeout_retries=timeout_retries,
92                             shell=shell,
93                             cpu_cost=cpu_cost,
94                             verbose_success=verbose_success)
95    return jobspec
96
97
98_MACOS_COMPAT_FLAG = '-mmacosx-version-min=10.10'
99
100_ARCH_FLAG_MAP = {'x86': '-m32', 'x64': '-m64'}
101
102
103class PythonArtifact:
104    """Builds Python artifacts."""
105
106    def __init__(self, platform, arch, py_version):
107        self.name = 'python_%s_%s_%s' % (platform, arch, py_version)
108        self.platform = platform
109        self.arch = arch
110        self.labels = ['artifact', 'python', platform, arch, py_version]
111        self.py_version = py_version
112        if 'manylinux' in platform:
113            self.labels.append('linux')
114
115    def pre_build_jobspecs(self):
116        return []
117
118    def build_jobspec(self):
119        environ = {}
120        if self.platform == 'linux_extra':
121            # Raspberry Pi build
122            environ['PYTHON'] = '/usr/local/bin/python{}'.format(
123                self.py_version)
124            environ['PIP'] = '/usr/local/bin/pip{}'.format(self.py_version)
125            # https://github.com/resin-io-projects/armv7hf-debian-qemu/issues/9
126            # A QEMU bug causes submodule update to hang, so we copy directly
127            environ['RELATIVE_COPY_PATH'] = '.'
128            # Parallel builds are counterproductive in emulated environment
129            environ['GRPC_PYTHON_BUILD_EXT_COMPILER_JOBS'] = '1'
130            extra_args = ' --entrypoint=/usr/bin/qemu-arm-static '
131            return create_docker_jobspec(
132                self.name,
133                'tools/dockerfile/grpc_artifact_linux_{}'.format(self.arch),
134                'tools/run_tests/artifacts/build_artifact_python.sh',
135                environ=environ,
136                timeout_seconds=60 * 60 * 5,
137                docker_base_image='quay.io/grpc/raspbian_{}'.format(self.arch),
138                extra_docker_args=extra_args)
139        elif 'manylinux' in self.platform:
140            if self.arch == 'x86':
141                environ['SETARCH_CMD'] = 'linux32'
142            # Inside the manylinux container, the python installations are located in
143            # special places...
144            environ['PYTHON'] = '/opt/python/{}/bin/python'.format(
145                self.py_version)
146            environ['PIP'] = '/opt/python/{}/bin/pip'.format(self.py_version)
147            environ['GRPC_BUILD_GRPCIO_TOOLS_DEPENDENTS'] = 'TRUE'
148            environ['GRPC_BUILD_MANYLINUX_WHEEL'] = 'TRUE'
149            return create_docker_jobspec(
150                self.name,
151                # NOTE(rbellevi): Do *not* update this without also ensuring the
152                # base_docker_image attribute is accurate.
153                'tools/dockerfile/grpc_artifact_python_%s_%s' %
154                (self.platform, self.arch),
155                'tools/run_tests/artifacts/build_artifact_python.sh',
156                environ=environ,
157                timeout_seconds=60 * 60)
158        elif self.platform == 'windows':
159            if 'Python27' in self.py_version:
160                environ['EXT_COMPILER'] = 'mingw32'
161            else:
162                environ['EXT_COMPILER'] = 'msvc'
163            # For some reason, the batch script %random% always runs with the same
164            # seed.  We create a random temp-dir here
165            dir = ''.join(
166                random.choice(string.ascii_uppercase) for _ in range(10))
167            return create_jobspec(self.name, [
168                'tools\\run_tests\\artifacts\\build_artifact_python.bat',
169                self.py_version, '32' if self.arch == 'x86' else '64'
170            ],
171                                  environ=environ,
172                                  timeout_seconds=45 * 60,
173                                  use_workspace=True)
174        else:
175            environ['PYTHON'] = self.py_version
176            environ['SKIP_PIP_INSTALL'] = 'TRUE'
177            return create_jobspec(
178                self.name,
179                ['tools/run_tests/artifacts/build_artifact_python.sh'],
180                environ=environ,
181                timeout_seconds=60 * 60 * 2,
182                use_workspace=True)
183
184    def __str__(self):
185        return self.name
186
187
188class RubyArtifact:
189    """Builds ruby native gem."""
190
191    def __init__(self, platform, arch):
192        self.name = 'ruby_native_gem_%s_%s' % (platform, arch)
193        self.platform = platform
194        self.arch = arch
195        self.labels = ['artifact', 'ruby', platform, arch]
196
197    def pre_build_jobspecs(self):
198        return []
199
200    def build_jobspec(self):
201        # Ruby build uses docker internally and docker cannot be nested.
202        # We are using a custom workspace instead.
203        return create_jobspec(
204            self.name, ['tools/run_tests/artifacts/build_artifact_ruby.sh'],
205            use_workspace=True,
206            timeout_seconds=60 * 60)
207
208
209class CSharpExtArtifact:
210    """Builds C# native extension library"""
211
212    def __init__(self, platform, arch, arch_abi=None):
213        self.name = 'csharp_ext_%s_%s' % (platform, arch)
214        self.platform = platform
215        self.arch = arch
216        self.arch_abi = arch_abi
217        self.labels = ['artifact', 'csharp', platform, arch]
218        if arch_abi:
219            self.name += '_%s' % arch_abi
220            self.labels.append(arch_abi)
221
222    def pre_build_jobspecs(self):
223        return []
224
225    def build_jobspec(self):
226        if self.arch == 'android':
227            return create_docker_jobspec(
228                self.name,
229                'tools/dockerfile/grpc_artifact_android_ndk',
230                'tools/run_tests/artifacts/build_artifact_csharp_android.sh',
231                environ={'ANDROID_ABI': self.arch_abi})
232        elif self.arch == 'ios':
233            return create_jobspec(
234                self.name,
235                ['tools/run_tests/artifacts/build_artifact_csharp_ios.sh'],
236                timeout_seconds=45 * 60,
237                use_workspace=True)
238        elif self.platform == 'windows':
239            return create_jobspec(self.name, [
240                'tools\\run_tests\\artifacts\\build_artifact_csharp.bat',
241                self.arch
242            ],
243                                  use_workspace=True)
244        else:
245            if self.platform == 'linux':
246                cmake_arch_option = ''  # x64 is the default architecture
247                if self.arch == 'x86':
248                    # TODO(jtattermusch): more work needed to enable
249                    # boringssl assembly optimizations for 32-bit linux.
250                    # Problem: currently we are building the artifact under
251                    # 32-bit docker image, but CMAKE_SYSTEM_PROCESSOR is still
252                    # set to x86_64, so the resulting boringssl binary
253                    # would have undefined symbols.
254                    cmake_arch_option = '-DOPENSSL_NO_ASM=ON'
255                return create_docker_jobspec(
256                    self.name,
257                    'tools/dockerfile/grpc_artifact_centos6_{}'.format(
258                        self.arch),
259                    'tools/run_tests/artifacts/build_artifact_csharp.sh',
260                    environ={'CMAKE_ARCH_OPTION': cmake_arch_option})
261            else:
262                cmake_arch_option = ''  # x64 is the default architecture
263                if self.arch == 'x86':
264                    cmake_arch_option = '-DCMAKE_OSX_ARCHITECTURES=i386'
265                return create_jobspec(
266                    self.name,
267                    ['tools/run_tests/artifacts/build_artifact_csharp.sh'],
268                    environ={'CMAKE_ARCH_OPTION': cmake_arch_option},
269                    use_workspace=True)
270
271    def __str__(self):
272        return self.name
273
274
275class PHPArtifact:
276    """Builds PHP PECL package"""
277
278    def __init__(self, platform, arch):
279        self.name = 'php_pecl_package_{0}_{1}'.format(platform, arch)
280        self.platform = platform
281        self.arch = arch
282        self.labels = ['artifact', 'php', platform, arch]
283
284    def pre_build_jobspecs(self):
285        return []
286
287    def build_jobspec(self):
288        return create_docker_jobspec(
289            self.name,
290            'tools/dockerfile/test/php73_zts_stretch_{}'.format(self.arch),
291            'tools/run_tests/artifacts/build_artifact_php.sh')
292
293
294class ProtocArtifact:
295    """Builds protoc and protoc-plugin artifacts"""
296
297    def __init__(self, platform, arch):
298        self.name = 'protoc_%s_%s' % (platform, arch)
299        self.platform = platform
300        self.arch = arch
301        self.labels = ['artifact', 'protoc', platform, arch]
302
303    def pre_build_jobspecs(self):
304        return []
305
306    def build_jobspec(self):
307        if self.platform != 'windows':
308            environ = {'CXXFLAGS': '', 'LDFLAGS': ''}
309            if self.platform == 'linux':
310                environ['LDFLAGS'] += ' -static-libgcc -static-libstdc++ -s'
311                return create_docker_jobspec(
312                    self.name,
313                    'tools/dockerfile/grpc_artifact_centos6_{}'.format(
314                        self.arch),
315                    'tools/run_tests/artifacts/build_artifact_protoc.sh',
316                    environ=environ)
317            else:
318                environ[
319                    'CXXFLAGS'] += ' -std=c++11 -stdlib=libc++ %s' % _MACOS_COMPAT_FLAG
320                return create_jobspec(
321                    self.name,
322                    ['tools/run_tests/artifacts/build_artifact_protoc.sh'],
323                    environ=environ,
324                    timeout_seconds=60 * 60,
325                    use_workspace=True)
326        else:
327            generator = 'Visual Studio 14 2015 Win64' if self.arch == 'x64' else 'Visual Studio 14 2015'
328            return create_jobspec(
329                self.name,
330                ['tools\\run_tests\\artifacts\\build_artifact_protoc.bat'],
331                environ={'generator': generator},
332                use_workspace=True)
333
334    def __str__(self):
335        return self.name
336
337
338def targets():
339    """Gets list of supported targets"""
340    return [
341        ProtocArtifact('linux', 'x64'),
342        ProtocArtifact('linux', 'x86'),
343        ProtocArtifact('macos', 'x64'),
344        ProtocArtifact('windows', 'x64'),
345        ProtocArtifact('windows', 'x86'),
346        CSharpExtArtifact('linux', 'x64'),
347        CSharpExtArtifact('macos', 'x64'),
348        CSharpExtArtifact('windows', 'x64'),
349        CSharpExtArtifact('windows', 'x86'),
350        CSharpExtArtifact('linux', 'android', arch_abi='arm64-v8a'),
351        CSharpExtArtifact('linux', 'android', arch_abi='armeabi-v7a'),
352        CSharpExtArtifact('linux', 'android', arch_abi='x86'),
353        CSharpExtArtifact('macos', 'ios'),
354        PythonArtifact('manylinux2014', 'x64', 'cp35-cp35m'),
355        PythonArtifact('manylinux2014', 'x64', 'cp36-cp36m'),
356        PythonArtifact('manylinux2014', 'x64', 'cp37-cp37m'),
357        PythonArtifact('manylinux2014', 'x64', 'cp38-cp38'),
358        PythonArtifact('manylinux2014', 'x64', 'cp39-cp39'),
359        PythonArtifact('manylinux2014', 'x86', 'cp35-cp35m'),
360        PythonArtifact('manylinux2014', 'x86', 'cp36-cp36m'),
361        PythonArtifact('manylinux2014', 'x86', 'cp37-cp37m'),
362        PythonArtifact('manylinux2014', 'x86', 'cp38-cp38'),
363        PythonArtifact('manylinux2014', 'x86', 'cp39-cp39'),
364        PythonArtifact('manylinux2010', 'x64', 'cp27-cp27m'),
365        PythonArtifact('manylinux2010', 'x64', 'cp27-cp27mu'),
366        PythonArtifact('manylinux2010', 'x64', 'cp35-cp35m'),
367        PythonArtifact('manylinux2010', 'x64', 'cp36-cp36m'),
368        PythonArtifact('manylinux2010', 'x64', 'cp37-cp37m'),
369        PythonArtifact('manylinux2010', 'x64', 'cp38-cp38'),
370        PythonArtifact('manylinux2010', 'x64', 'cp39-cp39'),
371        PythonArtifact('manylinux2010', 'x86', 'cp27-cp27m'),
372        PythonArtifact('manylinux2010', 'x86', 'cp27-cp27mu'),
373        PythonArtifact('manylinux2010', 'x86', 'cp35-cp35m'),
374        PythonArtifact('manylinux2010', 'x86', 'cp36-cp36m'),
375        PythonArtifact('manylinux2010', 'x86', 'cp37-cp37m'),
376        PythonArtifact('manylinux2010', 'x86', 'cp38-cp38'),
377        PythonArtifact('manylinux2010', 'x86', 'cp39-cp39'),
378        PythonArtifact('linux_extra', 'armv7', '2.7'),
379        PythonArtifact('linux_extra', 'armv7', '3.5'),
380        PythonArtifact('linux_extra', 'armv7', '3.6'),
381        PythonArtifact('linux_extra', 'armv6', '2.7'),
382        PythonArtifact('linux_extra', 'armv6', '3.5'),
383        PythonArtifact('linux_extra', 'armv6', '3.6'),
384        PythonArtifact('macos', 'x64', 'python2.7'),
385        PythonArtifact('macos', 'x64', 'python3.5'),
386        PythonArtifact('macos', 'x64', 'python3.6'),
387        PythonArtifact('macos', 'x64', 'python3.7'),
388        PythonArtifact('macos', 'x64', 'python3.8'),
389        PythonArtifact('macos', 'x64', 'python3.9'),
390        PythonArtifact('windows', 'x86', 'Python27_32bit'),
391        PythonArtifact('windows', 'x86', 'Python35_32bit'),
392        PythonArtifact('windows', 'x86', 'Python36_32bit'),
393        PythonArtifact('windows', 'x86', 'Python37_32bit'),
394        PythonArtifact('windows', 'x86', 'Python38_32bit'),
395        PythonArtifact('windows', 'x86', 'Python39_32bit'),
396        PythonArtifact('windows', 'x64', 'Python27'),
397        PythonArtifact('windows', 'x64', 'Python35'),
398        PythonArtifact('windows', 'x64', 'Python36'),
399        PythonArtifact('windows', 'x64', 'Python37'),
400        PythonArtifact('windows', 'x64', 'Python38'),
401        PythonArtifact('windows', 'x64', 'Python39'),
402        RubyArtifact('linux', 'x64'),
403        RubyArtifact('macos', 'x64'),
404        PHPArtifact('linux', 'x64')
405    ]
406