1#
2# Copyright 2015 Google Inc.
3#
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6#
7
8#!/usr/bin/env python
9
10usage = '''
11Write buildbot spec to outfile based on the bot name:
12  $ python buildbot_spec.py outfile Test-Ubuntu-GCC-GCE-CPU-AVX2-x86-Debug
13Or run self-tests:
14  $ python buildbot_spec.py test
15'''
16
17import inspect
18import json
19import os
20import sys
21
22import builder_name_schema
23import dm_flags
24import nanobench_flags
25
26
27CONFIG_DEBUG = 'Debug'
28CONFIG_RELEASE = 'Release'
29
30
31def lineno():
32  caller = inspect.stack()[1]  # Up one level to our caller.
33  return inspect.getframeinfo(caller[0]).lineno
34
35# Since we don't actually start coverage until we're in the self-test,
36# some function def lines aren't reported as covered. Add them to this
37# list so that we can ignore them.
38cov_skip = []
39
40cov_start = lineno()+1   # We care about coverage starting just past this def.
41def gyp_defines(builder_dict):
42  gyp_defs = {}
43
44  # skia_arch_type.
45  if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD:
46    arch = builder_dict['target_arch']
47  elif builder_dict['role'] == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER:
48    arch = None
49  else:
50    arch = builder_dict['arch']
51
52  arch_types = {
53    'x86':      'x86',
54    'x86_64':   'x86_64',
55    'Arm7':     'arm',
56    'Arm64':    'arm64',
57    'Mips':     'mips32',
58    'Mips64':   'mips64',
59    'MipsDSP2': 'mips32',
60  }
61  if arch in arch_types:
62    gyp_defs['skia_arch_type']  = arch_types[arch]
63
64  # housekeeper: build shared lib.
65  if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER:
66    gyp_defs['skia_shared_lib'] = '1'
67
68  # skia_gpu.
69  if builder_dict.get('cpu_or_gpu') == 'CPU':
70    gyp_defs['skia_gpu'] = '0'
71
72  # skia_warnings_as_errors.
73  werr = False
74  if builder_dict['role'] == builder_name_schema.BUILDER_ROLE_BUILD:
75    if 'Win' in builder_dict.get('os', ''):
76      if not ('GDI' in builder_dict.get('extra_config', '') or
77              'Exceptions' in builder_dict.get('extra_config', '')):
78        werr = True
79    elif ('Mac' in builder_dict.get('os', '') and
80          'Android' in builder_dict.get('extra_config', '')):
81      werr = False
82    else:
83      werr = True
84  gyp_defs['skia_warnings_as_errors'] = str(int(werr))  # True/False -> '1'/'0'
85
86  # Win debugger.
87  if 'Win' in builder_dict.get('os', ''):
88    gyp_defs['skia_win_debuggers_path'] = 'c:/DbgHelp'
89
90  # Qt SDK (Win).
91  if 'Win' in builder_dict.get('os', ''):
92    if builder_dict.get('os') == 'Win8':
93      gyp_defs['qt_sdk'] = 'C:/Qt/Qt5.1.0/5.1.0/msvc2012_64/'
94    else:
95      gyp_defs['qt_sdk'] = 'C:/Qt/4.8.5/'
96
97  # ANGLE.
98  if builder_dict.get('extra_config') == 'ANGLE':
99    gyp_defs['skia_angle'] = '1'
100    if builder_dict.get('os', '') in ('Ubuntu', 'Linux'):
101      gyp_defs['use_x11'] = '1'
102      gyp_defs['chromeos'] = '0'
103
104  # GDI.
105  if builder_dict.get('extra_config') == 'GDI':
106    gyp_defs['skia_gdi'] = '1'
107
108  # Build with Exceptions on Windows.
109  if ('Win' in builder_dict.get('os', '') and
110      builder_dict.get('extra_config') == 'Exceptions'):
111    gyp_defs['skia_win_exceptions'] = '1'
112
113  # iOS.
114  if (builder_dict.get('os') == 'iOS' or
115      builder_dict.get('extra_config') == 'iOS'):
116    gyp_defs['skia_os'] = 'ios'
117
118  # Shared library build.
119  if builder_dict.get('extra_config') == 'Shared':
120    gyp_defs['skia_shared_lib'] = '1'
121
122  # Build fastest Skia possible.
123  if builder_dict.get('extra_config') == 'Fast':
124    gyp_defs['skia_fast'] = '1'
125
126  # PDF viewer in GM.
127  if (builder_dict.get('os') == 'Mac10.8' and
128      builder_dict.get('arch') == 'x86_64' and
129      builder_dict.get('configuration') == 'Release'):
130    gyp_defs['skia_run_pdfviewer_in_gm'] = '1'
131
132  # Clang.
133  if builder_dict.get('compiler') == 'Clang':
134    gyp_defs['skia_clang_build'] = '1'
135
136  # Valgrind.
137  if 'Valgrind' in builder_dict.get('extra_config', ''):
138    gyp_defs['skia_release_optimization_level'] = '1'
139
140  # Link-time code generation just wastes time on compile-only bots.
141  if (builder_dict.get('role') == builder_name_schema.BUILDER_ROLE_BUILD and
142      builder_dict.get('compiler') == 'MSVC'):
143    gyp_defs['skia_win_ltcg'] = '0'
144
145  # Mesa.
146  if (builder_dict.get('extra_config') == 'Mesa' or
147      builder_dict.get('cpu_or_gpu_value') == 'Mesa'):
148    gyp_defs['skia_mesa'] = '1'
149
150  # VisualBench
151  if builder_dict.get('extra_config') == 'VisualBench':
152    gyp_defs['skia_use_sdl'] = '1'
153
154  # skia_use_android_framework_defines.
155  if builder_dict.get('extra_config') == 'Android_FrameworkDefs':
156    gyp_defs['skia_use_android_framework_defines'] = '1'
157
158  # Skia dump stats for perf tests and gpu
159  if (builder_dict.get('cpu_or_gpu') == 'GPU' and
160      builder_dict.get('role') == 'Perf'):
161      gyp_defs['skia_dump_stats'] = '1'
162
163  return gyp_defs
164
165
166cov_skip.extend([lineno(), lineno() + 1])
167def get_extra_env_vars(builder_dict):
168  env = {}
169  if builder_dict.get('configuration') == 'Coverage':
170    # We have to use Clang 3.6 because earlier versions do not support the
171    # compile flags we use and 3.7 and 3.8 hit asserts during compilation.
172    env['CC'] = '/usr/bin/clang-3.6'
173    env['CXX'] = '/usr/bin/clang++-3.6'
174  elif builder_dict.get('compiler') == 'Clang':
175    env['CC'] = '/usr/bin/clang'
176    env['CXX'] = '/usr/bin/clang++'
177
178  # SKNX_NO_SIMD, SK_USE_DISCARDABLE_SCALEDIMAGECACHE, etc.
179  extra_config = builder_dict.get('extra_config', '')
180  if extra_config.startswith('SK') and extra_config.isupper():
181    env['CPPFLAGS'] = '-D' + extra_config
182
183  return env
184
185
186cov_skip.extend([lineno(), lineno() + 1])
187def build_targets_from_builder_dict(builder_dict, do_test_steps, do_perf_steps):
188  """Return a list of targets to build, depending on the builder type."""
189  if builder_dict['role'] in ('Test', 'Perf') and builder_dict['os'] == 'iOS':
190    return ['iOSShell']
191  if builder_dict.get('extra_config') == 'Appurify':
192    return ['VisualBenchTest_APK']
193  t = []
194  if do_test_steps:
195    t.append('dm')
196  if do_perf_steps and builder_dict.get('extra_config') == 'VisualBench':
197      t.append('visualbench')
198  elif do_perf_steps:
199      t.append('nanobench')
200  if t:
201    return t
202  else:
203    return ['most']
204
205
206cov_skip.extend([lineno(), lineno() + 1])
207def device_cfg(builder_dict):
208  # Android.
209  if 'Android' in builder_dict.get('extra_config', ''):
210    if 'NoNeon' in builder_dict['extra_config']:
211      return 'arm_v7'
212    return {
213      'Arm64': 'arm64',
214      'x86': 'x86',
215      'x86_64': 'x86_64',
216      'Mips': 'mips',
217      'Mips64': 'mips64',
218      'MipsDSP2': 'mips_dsp2',
219    }.get(builder_dict['target_arch'], 'arm_v7_neon')
220  elif builder_dict.get('os') == 'Android':
221    return {
222      'AndroidOne':    'arm_v7_neon',
223      'GalaxyS3':      'arm_v7_neon',
224      'GalaxyS4':      'arm_v7_neon',
225      'NVIDIA_Shield': 'arm64',
226      'Nexus10':       'arm_v7_neon',
227      'Nexus5':        'arm_v7_neon',
228      'Nexus6':        'arm_v7_neon',
229      'Nexus7':        'arm_v7_neon',
230      'Nexus9':        'arm64',
231      'NexusPlayer':   'x86',
232    }[builder_dict['model']]
233
234  # ChromeOS.
235  if 'CrOS' in builder_dict.get('extra_config', ''):
236    if 'Link' in builder_dict['extra_config']:
237      return 'link'
238    if 'Daisy' in builder_dict['extra_config']:
239      return 'daisy'
240  elif builder_dict.get('os') == 'ChromeOS':
241    return {
242      'Link': 'link',
243      'Daisy': 'daisy',
244    }[builder_dict['model']]
245
246  return None
247
248
249cov_skip.extend([lineno(), lineno() + 1])
250def get_builder_spec(builder_name):
251  builder_dict = builder_name_schema.DictForBuilderName(builder_name)
252  env = get_extra_env_vars(builder_dict)
253  gyp_defs = gyp_defines(builder_dict)
254  gyp_defs_list = ['%s=%s' % (k, v) for k, v in gyp_defs.iteritems()]
255  gyp_defs_list.sort()
256  env['GYP_DEFINES'] = ' '.join(gyp_defs_list)
257  rv = {
258    'builder_cfg': builder_dict,
259    'dm_flags': dm_flags.get_args(builder_name),
260    'env': env,
261    'nanobench_flags': nanobench_flags.get_args(builder_name),
262  }
263  device = device_cfg(builder_dict)
264  if device:
265    rv['device_cfg'] = device
266
267  role = builder_dict['role']
268  if role == builder_name_schema.BUILDER_ROLE_HOUSEKEEPER:
269    configuration = CONFIG_RELEASE
270  else:
271    configuration = builder_dict.get(
272        'configuration', CONFIG_DEBUG)
273  arch = (builder_dict.get('arch') or builder_dict.get('target_arch'))
274  if ('Win' in builder_dict.get('os', '') and arch == 'x86_64'):
275    configuration += '_x64'
276  rv['configuration'] = configuration
277  rv['do_test_steps'] = role == builder_name_schema.BUILDER_ROLE_TEST
278  rv['do_perf_steps'] = (role == builder_name_schema.BUILDER_ROLE_PERF or
279                         (role == builder_name_schema.BUILDER_ROLE_TEST and
280                          configuration == CONFIG_DEBUG))
281  if 'Valgrind' in builder_name:
282    rv['do_perf_steps'] = True
283  if 'GalaxyS4' in builder_name:
284    rv['do_perf_steps'] = False
285
286  rv['build_targets'] = build_targets_from_builder_dict(
287        builder_dict, rv['do_test_steps'], rv['do_perf_steps'])
288
289  # Do we upload perf results?
290  upload_perf_results = False
291  if role == builder_name_schema.BUILDER_ROLE_PERF:
292    upload_perf_results = True
293  rv['upload_perf_results'] = upload_perf_results
294
295  # Do we upload correctness results?
296  skip_upload_bots = [
297    'ASAN',
298    'Coverage',
299    'MSAN',
300    'TSAN',
301    'UBSAN',
302    'Valgrind',
303  ]
304  upload_dm_results = True
305  for s in skip_upload_bots:
306    if s in builder_name:
307      upload_dm_results = False
308      break
309  rv['upload_dm_results'] = upload_dm_results
310
311  return rv
312
313
314cov_end = lineno()   # Don't care about code coverage past here.
315
316
317def self_test():
318  import coverage  # This way the bots don't need coverage.py to be installed.
319  args = {}
320  cases = [
321        'Build-Mac10.8-Clang-Arm7-Debug-Android',
322        'Build-Win-MSVC-x86-Debug',
323        'Build-Win-MSVC-x86-Debug-GDI',
324        'Build-Win-MSVC-x86-Debug-Exceptions',
325        'Build-Ubuntu-GCC-Arm7-Debug-Android_FrameworkDefs',
326        'Build-Ubuntu-GCC-Arm7-Debug-Android_NoNeon',
327        'Build-Ubuntu-GCC-Arm7-Debug-CrOS_Daisy',
328        'Build-Ubuntu-GCC-x86_64-Debug-CrOS_Link',
329        'Build-Ubuntu-GCC-x86_64-Release-Mesa',
330        'Build-Ubuntu-GCC-x86_64-Release-ANGLE',
331        'Housekeeper-PerCommit',
332        'Perf-Win8-MSVC-ShuttleB-GPU-HD4600-x86_64-Release-Trybot',
333        'Perf-Ubuntu-GCC-ShuttleA-GPU-GTX660-x86_64-Release-VisualBench',
334        'Test-Android-GCC-GalaxyS4-GPU-SGX544-Arm7-Debug',
335        'Perf-Android-GCC-Nexus5-GPU-Adreno330-Arm7-Release-Appurify',
336        'Test-Android-GCC-Nexus6-GPU-Adreno420-Arm7-Debug',
337        'Test-ChromeOS-GCC-Link-CPU-AVX-x86_64-Debug',
338        'Test-iOS-Clang-iPad4-GPU-SGX554-Arm7-Debug',
339        'Test-Mac10.8-Clang-MacMini4.1-GPU-GeForce320M-x86_64-Release',
340        'Test-Ubuntu-Clang-GCE-CPU-AVX2-x86_64-Coverage',
341        ('Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-'
342         'SK_USE_DISCARDABLE_SCALEDIMAGECACHE'),
343        'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-SKNX_NO_SIMD',
344        'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Fast',
345        'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-Shared',
346        'Test-Ubuntu-GCC-ShuttleA-GPU-GTX550Ti-x86_64-Release-Valgrind',
347        'Test-Win8-MSVC-ShuttleB-GPU-HD4600-x86-Release-ANGLE',
348        'Test-Win8-MSVC-ShuttleA-CPU-AVX-x86_64-Debug',
349  ]
350
351  cov = coverage.coverage()
352  cov.start()
353  for case in cases:
354    args[case] = get_builder_spec(case)
355  cov.stop()
356
357  this_file = os.path.basename(__file__)
358  _, _, not_run, _ = cov.analysis(this_file)
359  filtered = [line for line in not_run if
360              line > cov_start and line < cov_end and line not in cov_skip]
361  if filtered:
362    print 'Lines not covered by test cases: ', filtered
363    sys.exit(1)
364
365  golden = this_file.replace('.py', '.json')
366  with open(os.path.join(os.path.dirname(__file__), golden), 'w') as f:
367    json.dump(args, f, indent=2, sort_keys=True)
368
369
370if __name__ == '__main__':
371  if len(sys.argv) == 2 and sys.argv[1] == 'test':
372    self_test()
373    sys.exit(0)
374
375  if len(sys.argv) != 3:
376    print usage
377    sys.exit(1)
378
379  with open(sys.argv[1], 'w') as out:
380    json.dump(get_builder_spec(sys.argv[2]), out)
381