1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6import math 7 8 9DEPS = [ 10 'core', 11 'ct', 12 'flavor', 13 'recipe_engine/context', 14 'recipe_engine/file', 15 'recipe_engine/json', 16 'recipe_engine/path', 17 'recipe_engine/properties', 18 'recipe_engine/step', 19 'recipe_engine/time', 20 'run', 21 'skia_swarming', 22 'vars', 23] 24 25 26SKPS_VERSION_FILE = 'skps_version' 27CT_SKPS_ISOLATE = 'ct_skps.isolate' 28 29# Do not batch archive more slaves than this value. This is used to prevent 30# no output timeouts in the 'isolate tests' step. 31MAX_SLAVES_TO_BATCHARCHIVE = 100 32 33TOOL_TO_DEFAULT_SKPS_PER_SLAVE = { 34 'dm': 10000, 35 'nanobench': 1000, 36 'get_images_from_skps': 10000, 37} 38 39# The SKP repository to use. 40DEFAULT_SKPS_CHROMIUM_BUILD = '2b7e85eb251dc7-a3cf3659ed2c08' 41 42 43def RunSteps(api): 44 # Figure out which repository to use. 45 buildername = api.properties['buildername'] 46 if '1k' in buildername: 47 ct_page_type = 'All' 48 num_pages = 1000 49 elif '10k' in buildername: 50 ct_page_type = 'All' 51 num_pages = 10000 52 elif '100k' in buildername: 53 ct_page_type = 'All' 54 num_pages = 100000 55 elif '1m' in buildername: 56 ct_page_type = 'All' 57 num_pages = 1000000 58 else: 59 raise Exception('Do not recognise the buildername %s.' % buildername) 60 61 # Figure out which tool to use. 62 if 'DM' in buildername: 63 skia_tool = 'dm' 64 build_target = 'dm' 65 elif 'BENCH' in buildername: 66 skia_tool = 'nanobench' 67 build_target = 'nanobench' 68 elif 'IMG_DECODE' in buildername: 69 skia_tool = 'get_images_from_skps' 70 build_target = 'tools' 71 else: 72 raise Exception('Do not recognise the buildername %s.' % buildername) 73 74 api.core.setup() 75 api.flavor.compile(build_target) 76 77 # Required paths. 78 infrabots_dir = api.vars.skia_dir.join('infra', 'bots') 79 isolate_dir = infrabots_dir.join('ct') 80 isolate_path = isolate_dir.join(CT_SKPS_ISOLATE) 81 82 api.run.copy_build_products( 83 api.flavor.out_dir, 84 isolate_dir) 85 api.skia_swarming.setup( 86 infrabots_dir.join('tools', 'luci-go'), 87 swarming_rev='') 88 89 skps_chromium_build = api.properties.get( 90 'skps_chromium_build', DEFAULT_SKPS_CHROMIUM_BUILD) 91 92 # Set build properties to make finding SKPs convenient. 93 webpage_rankings_link = ( 94 'https://storage.cloud.google.com/%s/csv/top-1m.csv' 95 % api.ct.CT_GS_BUCKET) 96 api.step.active_result.presentation.properties['Webpage rankings'] = ( 97 webpage_rankings_link) 98 download_skps_link = ( 99 'https://pantheon.corp.google.com/storage/browser/%s/swarming/skps/%s/%s/' 100 % (api.ct.CT_GS_BUCKET, ct_page_type, skps_chromium_build)) 101 api.step.active_result.presentation.properties['Download SKPs by rank'] = ( 102 download_skps_link) 103 104 # Delete swarming_temp_dir to ensure it starts from a clean slate. 105 api.run.rmtree(api.skia_swarming.swarming_temp_dir) 106 107 num_per_slave = api.properties.get( 108 'num_per_slave', 109 min(TOOL_TO_DEFAULT_SKPS_PER_SLAVE[skia_tool], num_pages)) 110 ct_num_slaves = api.properties.get( 111 'ct_num_slaves', 112 int(math.ceil(float(num_pages) / num_per_slave))) 113 114 # Try to figure out if the SKPs we are going to isolate already exist 115 # locally by reading the SKPS_VERSION_FILE. 116 download_skps = True 117 expected_version_contents = { 118 "chromium_build": skps_chromium_build, 119 "page_type": ct_page_type, 120 "num_slaves": ct_num_slaves, 121 } 122 # Note: If this directory is changed, the corresponding places it is 123 # referenced also needs to change. As of 8/8/17 the other places are: 124 # * infra/bots/ct/ct_skps.isolate 125 # * infra/bots/ct/run_ct_skps.py 126 skps_dir = api.vars.checkout_root.join('skps', skps_chromium_build, 127 ct_page_type, str(ct_num_slaves)) 128 version_file = skps_dir.join(SKPS_VERSION_FILE) 129 if api.path.exists(version_file): # pragma: nocover 130 version_file_contents = api.file.read_text( 131 "Read %s" % version_file, 132 version_file, 133 test_data=expected_version_contents) 134 actual_version_contents = api.json.loads(version_file_contents) 135 differences = (set(expected_version_contents.items()) ^ 136 set(actual_version_contents.items())) 137 download_skps = len(differences) != 0 138 if download_skps: 139 # Delete and recreate the skps dir. 140 api.run.rmtree(skps_dir) 141 api.file.ensure_directory( 142 'makedirs %s' % api.path.basename(skps_dir), skps_dir) 143 144 # If a blacklist file exists then specify SKPs to be blacklisted. 145 blacklists_dir = api.vars.skia_dir.join('infra', 'bots', 'ct', 'blacklists') 146 blacklist_file = blacklists_dir.join( 147 '%s_%s_%s.json' % (skia_tool, ct_page_type, skps_chromium_build)) 148 blacklist_skps = [] 149 if api.path.exists(blacklist_file): # pragma: nocover 150 blacklist_file_contents = api.file.read_text( 151 "Read %s" % blacklist_file, 152 blacklist_file) 153 blacklist_skps = api.json.loads(blacklist_file_contents)['blacklisted_skps'] 154 155 for slave_num in range(1, ct_num_slaves + 1): 156 if download_skps: 157 # Download SKPs. 158 api.ct.download_swarming_skps( 159 ct_page_type, slave_num, skps_chromium_build, 160 skps_dir, 161 start_range=((slave_num-1)*num_per_slave) + 1, 162 num_skps=num_per_slave) 163 164 # Create this slave's isolated.gen.json file to use for batcharchiving. 165 extra_variables = { 166 'SLAVE_NUM': str(slave_num), 167 'TOOL_NAME': skia_tool, 168 'GIT_HASH': api.vars.got_revision, 169 'CONFIGURATION': api.vars.configuration, 170 'BUILDER': buildername, 171 'CHROMIUM_BUILD': skps_chromium_build, 172 'PAGE_TYPE': ct_page_type, 173 'NUM_SLAVES': str(ct_num_slaves), 174 } 175 api.skia_swarming.create_isolated_gen_json( 176 isolate_path, isolate_dir, 'linux', 'ct-%s-%s' % (skia_tool, slave_num), 177 extra_variables, blacklist=blacklist_skps) 178 179 if download_skps: 180 # Since we had to download SKPs create an updated version file. 181 api.file.write_text("Create %s" % version_file, 182 version_file, 183 api.json.dumps(expected_version_contents)) 184 185 # Batcharchive everything on the isolate server for efficiency. 186 max_slaves_to_batcharchive = MAX_SLAVES_TO_BATCHARCHIVE 187 if '1m' in buildername: 188 # Break up the "isolate tests" step into batches with <100k files due to 189 # https://github.com/luci/luci-go/issues/9 190 max_slaves_to_batcharchive = 5 191 tasks_to_swarm_hashes = [] 192 for slave_start_num in xrange(1, ct_num_slaves+1, max_slaves_to_batcharchive): 193 m = min(max_slaves_to_batcharchive, ct_num_slaves) 194 batcharchive_output = api.skia_swarming.batcharchive( 195 targets=['ct-' + skia_tool + '-%s' % num for num in range( 196 slave_start_num, slave_start_num + m)]) 197 tasks_to_swarm_hashes.extend(batcharchive_output) 198 # Sort the list to go through tasks in order. 199 tasks_to_swarm_hashes.sort() 200 201 # Trigger all swarming tasks. 202 dimensions={'os': 'Ubuntu-14.04'} 203 if 'GPU' in buildername: 204 dimensions['cpu'] = 'x86-64-E3-1230_v5' 205 dimensions['gpu'] = '10de:1cb3-384.90' 206 # See crbug.com/700053 207 dimensions['pool'] = 'Chrome-GPU' 208 else: 209 dimensions['cpu'] = 'x86-64-Broadwell_GCE' 210 dimensions['pool'] = 'Chrome' 211 tasks = api.skia_swarming.trigger_swarming_tasks( 212 tasks_to_swarm_hashes, dimensions=dimensions, io_timeout=40*60) 213 214 # Now collect all tasks. 215 env = {'AWS_CREDENTIAL_FILE': None, 'BOTO_CONFIG': None} 216 failed_tasks = [] 217 for task in tasks: 218 try: 219 api.skia_swarming.collect_swarming_task(task) 220 221 if skia_tool == 'nanobench': 222 output_dir = api.skia_swarming.tasks_output_dir.join( 223 task.title).join('0') 224 utc = api.time.utcnow() 225 gs_dest_dir = 'gs://skia-perf/ct/%s/%d/%02d/%02d/%02d/' % ( 226 ct_page_type, utc.year, utc.month, utc.day, utc.hour) 227 for json_output in api.file.listdir( 228 'listdir output dir', output_dir, test_data=['file 1', 'file 2']): 229 with api.context(env=env): 230 cmd = ['gsutil', 'cp', '-R', json_output, gs_dest_dir] 231 api.step('upload json output', cmd=cmd, infra_step=True) 232 233 except api.step.StepFailure as e: 234 # Add SKP links for convenience. 235 api.step.active_result.presentation.links['Webpage rankings'] = ( 236 webpage_rankings_link) 237 api.step.active_result.presentation.links['Download SKPs by rank'] = ( 238 download_skps_link) 239 failed_tasks.append(e) 240 241 if failed_tasks: 242 raise api.step.StepFailure( 243 'Failed steps: %s' % ', '.join([f.name for f in failed_tasks])) 244 245 246def GenTests(api): 247 ct_num_slaves = 5 248 num_per_slave = 10 249 skia_revision = 'abc123' 250 path_config = 'kitchen' 251 252 yield( 253 api.test('CT_DM_10k_SKPs') + 254 api.properties( 255 buildername=('Test-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Debug-All-' + 256 'CT_DM_10k_SKPs'), 257 path_config=path_config, 258 swarm_out_dir='[SWARM_OUT_DIR]', 259 ct_num_slaves=ct_num_slaves, 260 num_per_slave=num_per_slave, 261 repository='https://skia.googlesource.com/skia.git', 262 revision=skia_revision, 263 ) 264 ) 265 266 yield( 267 api.test('CT_IMG_DECODE_10k_SKPs') + 268 api.properties( 269 buildername='Test-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Debug-All-' 270 'CT_IMG_DECODE_10k_SKPs', 271 path_config=path_config, 272 swarm_out_dir='[SWARM_OUT_DIR]', 273 ct_num_slaves=ct_num_slaves, 274 num_per_slave=num_per_slave, 275 repository='https://skia.googlesource.com/skia.git', 276 revision=skia_revision, 277 ) 278 ) 279 280 yield( 281 api.test('CT_DM_100k_SKPs') + 282 api.properties( 283 buildername=('Test-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Debug-All-' + 284 'CT_DM_100k_SKPs'), 285 path_config=path_config, 286 swarm_out_dir='[SWARM_OUT_DIR]', 287 ct_num_slaves=ct_num_slaves, 288 num_per_slave=num_per_slave, 289 repository='https://skia.googlesource.com/skia.git', 290 revision=skia_revision, 291 ) 292 ) 293 294 yield( 295 api.test('CT_IMG_DECODE_100k_SKPs') + 296 api.properties( 297 buildername='Test-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Debug-All-' 298 'CT_IMG_DECODE_100k_SKPs', 299 path_config=path_config, 300 swarm_out_dir='[SWARM_OUT_DIR]', 301 ct_num_slaves=ct_num_slaves, 302 num_per_slave=num_per_slave, 303 repository='https://skia.googlesource.com/skia.git', 304 revision=skia_revision, 305 ) 306 ) 307 308 yield( 309 api.test('CT_GPU_BENCH_1k_SKPs') + 310 api.properties( 311 buildername=('Perf-Ubuntu14-GCC-Golo-GPU-QuadroP400-x86_64-Release-All-' 312 'CT_BENCH_1k_SKPs'), 313 path_config=path_config, 314 swarm_out_dir='[SWARM_OUT_DIR]', 315 ct_num_slaves=ct_num_slaves, 316 num_per_slave=num_per_slave, 317 repository='https://skia.googlesource.com/skia.git', 318 revision=skia_revision, 319 ) + 320 api.path.exists( 321 api.path['start_dir'].join('skia'), 322 api.path['start_dir'].join('src') 323 ) 324 ) 325 326 yield( 327 api.test('CT_CPU_BENCH_10k_SKPs') + 328 api.properties( 329 buildername=('Perf-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Release-All-' 330 'CT_BENCH_10k_SKPs'), 331 path_config=path_config, 332 swarm_out_dir='[SWARM_OUT_DIR]', 333 ct_num_slaves=ct_num_slaves, 334 num_per_slave=num_per_slave, 335 repository='https://skia.googlesource.com/skia.git', 336 revision=skia_revision, 337 ) + 338 api.path.exists( 339 api.path['start_dir'].join('skia'), 340 api.path['start_dir'].join('src') 341 ) 342 ) 343 344 yield( 345 api.test('CT_GPU_BENCH_10k_SKPs') + 346 api.properties( 347 buildername=('Perf-Ubuntu14-GCC-Golo-GPU-QuadroP400-x86_64-Release-All-' 348 'CT_BENCH_10k_SKPs'), 349 path_config=path_config, 350 swarm_out_dir='[SWARM_OUT_DIR]', 351 ct_num_slaves=ct_num_slaves, 352 num_per_slave=num_per_slave, 353 repository='https://skia.googlesource.com/skia.git', 354 revision=skia_revision, 355 ) + 356 api.path.exists( 357 api.path['start_dir'].join('skia'), 358 api.path['start_dir'].join('src') 359 ) 360 ) 361 362 yield( 363 api.test('CT_DM_1m_SKPs') + 364 api.properties( 365 buildername=('Test-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Debug-All-' 366 'CT_DM_1m_SKPs'), 367 path_config=path_config, 368 swarm_out_dir='[SWARM_OUT_DIR]', 369 ct_num_slaves=ct_num_slaves, 370 num_per_slave=num_per_slave, 371 repository='https://skia.googlesource.com/skia.git', 372 revision=skia_revision, 373 ) 374 ) 375 376 yield ( 377 api.test('CT_DM_SKPs_UnknownBuilder') + 378 api.properties( 379 buildername=('Test-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Debug-All-' + 380 'CT_DM_UnknownRepo_SKPs'), 381 path_config=path_config, 382 swarm_out_dir='[SWARM_OUT_DIR]', 383 ct_num_slaves=ct_num_slaves, 384 num_per_slave=num_per_slave, 385 repository='https://skia.googlesource.com/skia.git', 386 revision=skia_revision, 387 ) + 388 api.expect_exception('Exception') 389 ) 390 391 yield ( 392 api.test('CT_10k_SKPs_UnknownBuilder') + 393 api.properties( 394 buildername=('Test-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Debug-All-' + 395 'CT_UnknownTool_10k_SKPs'), 396 path_config=path_config, 397 swarm_out_dir='[SWARM_OUT_DIR]', 398 ct_num_slaves=ct_num_slaves, 399 num_per_slave=num_per_slave, 400 repository='https://skia.googlesource.com/skia.git', 401 revision=skia_revision, 402 ) + 403 api.expect_exception('Exception') 404 ) 405 406 yield( 407 api.test('CT_DM_1m_SKPs_slave3_failure') + 408 api.step_data('ct-dm-3', retcode=1) + 409 api.properties( 410 buildername=('Test-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Debug-All-' + 411 'CT_DM_1m_SKPs'), 412 path_config=path_config, 413 swarm_out_dir='[SWARM_OUT_DIR]', 414 ct_num_slaves=ct_num_slaves, 415 num_per_slave=num_per_slave, 416 repository='https://skia.googlesource.com/skia.git', 417 revision=skia_revision, 418 ) 419 ) 420 421 yield( 422 api.test('CT_DM_1m_SKPs_2slaves_failure') + 423 api.step_data('ct-dm-1', retcode=1) + 424 api.step_data('ct-dm-3', retcode=1) + 425 api.properties( 426 buildername=('Test-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Debug-All'+ 427 '-CT_DM_1m_SKPs'), 428 path_config=path_config, 429 swarm_out_dir='[SWARM_OUT_DIR]', 430 ct_num_slaves=ct_num_slaves, 431 num_per_slave=num_per_slave, 432 repository='https://skia.googlesource.com/skia.git', 433 revision=skia_revision, 434 ) 435 ) 436 437 builder = 'Test-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Debug-All-CT_DM_10k_SKPs' 438 yield( 439 api.test('CT_DM_10k_SKPs_Trybot') + 440 api.properties( 441 buildername=builder, 442 path_config=path_config, 443 swarm_out_dir='[SWARM_OUT_DIR]', 444 ct_num_slaves=ct_num_slaves, 445 num_per_slave=num_per_slave, 446 repository='https://skia.googlesource.com/skia.git', 447 patch_storage='gerrit') + 448 api.properties.tryserver( 449 buildername=builder, 450 gerrit_project='skia', 451 gerrit_url='https://skia-review.googlesource.com/', 452 ) 453 ) 454 455 builder = ('Test-Ubuntu14-GCC-GCE-CPU-AVX2-x86_64-Debug-All-' 456 'CT_IMG_DECODE_10k_SKPs') 457 yield( 458 api.test('CT_IMG_DECODE_10k_SKPs_Trybot') + 459 api.properties( 460 buildername=builder, 461 path_config=path_config, 462 swarm_out_dir='[SWARM_OUT_DIR]', 463 ct_num_slaves=ct_num_slaves, 464 num_per_slave=num_per_slave, 465 repository='https://skia.googlesource.com/skia.git', 466 revision=skia_revision, 467 patch_storage='gerrit') + 468 api.properties.tryserver( 469 buildername=builder, 470 gerrit_project='skia', 471 gerrit_url='https://skia-review.googlesource.com/', 472 ) 473 ) 474