1# Copyright 2015 The Chromium Authors. All rights reserved. 2# Use of this source code is governed by a BSD-style license that can be 3# found in the LICENSE file. 4 5 6import math 7 8 9DEPS = [ 10 'build/file', 11 'depot_tools/gsutil', 12 'recipe_engine/json', 13 'recipe_engine/path', 14 'recipe_engine/properties', 15 'recipe_engine/step', 16 'recipe_engine/time', 17 'core', 18 'ct', 19 'flavor', 20 'run', 21 'swarming', 22 'vars', 23] 24 25 26SKPS_VERSION_FILE = 'skps_version' 27CT_SKPS_ISOLATE = 'ct_skps.isolate' 28 29# Do not batch archive more slaves than this value. This is used to prevent 30# no output timeouts in the 'isolate tests' step. 31MAX_SLAVES_TO_BATCHARCHIVE = 100 32 33TOOL_TO_DEFAULT_SKPS_PER_SLAVE = { 34 'dm': 10000, 35 'nanobench': 1000, 36 'get_images_from_skps': 10000, 37} 38 39# The SKP repository to use. 40DEFAULT_SKPS_CHROMIUM_BUILD = 'fad657e-276e633' 41 42 43def RunSteps(api): 44 # Figure out which repository to use. 45 buildername = api.properties['buildername'] 46 if '1k' in buildername: 47 ct_page_type = '10k' 48 num_pages = 1000 49 elif '10k' in buildername: 50 ct_page_type = '10k' 51 num_pages = 10000 52 elif '100k' in buildername: 53 ct_page_type = '100k' 54 num_pages = 100000 55 elif '1m' in buildername: 56 ct_page_type = 'All' 57 num_pages = 1000000 58 else: 59 raise Exception('Do not recognise the buildername %s.' % buildername) 60 61 # Figure out which tool to use. 62 if 'DM' in buildername: 63 skia_tool = 'dm' 64 build_target = 'dm' 65 elif 'BENCH' in buildername: 66 skia_tool = 'nanobench' 67 build_target = 'nanobench' 68 elif 'IMG_DECODE' in buildername: 69 skia_tool = 'get_images_from_skps' 70 build_target = 'tools' 71 else: 72 raise Exception('Do not recognise the buildername %s.' % buildername) 73 74 api.core.setup() 75 api.flavor.compile(build_target) 76 77 # Required paths. 78 infrabots_dir = api.vars.skia_dir.join('infra', 'bots') 79 isolate_dir = infrabots_dir.join('ct') 80 isolate_path = isolate_dir.join(CT_SKPS_ISOLATE) 81 82 api.run.copy_build_products( 83 api.flavor.out_dir, 84 isolate_dir) 85 api.swarming.setup( 86 infrabots_dir.join('tools', 'luci-go'), 87 swarming_rev='') 88 89 skps_chromium_build = api.properties.get( 90 'skps_chromium_build', DEFAULT_SKPS_CHROMIUM_BUILD) 91 92 # Set build properties to make finding SKPs convenient. 93 webpage_rankings_link = ( 94 'https://storage.cloud.google.com/%s/csv/top-1m.csv' 95 % api.ct.CT_GS_BUCKET) 96 api.step.active_result.presentation.properties['Webpage rankings'] = ( 97 webpage_rankings_link) 98 download_skps_link = ( 99 'https://pantheon.corp.google.com/storage/browser/%s/swarming/skps/%s/%s/' 100 % (api.ct.CT_GS_BUCKET, ct_page_type, skps_chromium_build)) 101 api.step.active_result.presentation.properties['Download SKPs by rank'] = ( 102 download_skps_link) 103 104 # Delete swarming_temp_dir to ensure it starts from a clean slate. 105 api.run.rmtree(api.swarming.swarming_temp_dir) 106 107 num_per_slave = api.properties.get( 108 'num_per_slave', 109 min(TOOL_TO_DEFAULT_SKPS_PER_SLAVE[skia_tool], num_pages)) 110 ct_num_slaves = api.properties.get( 111 'ct_num_slaves', 112 int(math.ceil(float(num_pages) / num_per_slave))) 113 114 # Try to figure out if the SKPs we are going to isolate already exist 115 # locally by reading the SKPS_VERSION_FILE. 116 download_skps = True 117 expected_version_contents = { 118 "chromium_build": skps_chromium_build, 119 "page_type": ct_page_type, 120 "num_slaves": ct_num_slaves, 121 } 122 skps_dir = api.vars.checkout_root.join('skps', buildername) 123 version_file = skps_dir.join(SKPS_VERSION_FILE) 124 if api.path.exists(version_file): # pragma: nocover 125 version_file_contents = api.file.read( 126 "Read %s" % version_file, 127 version_file, 128 test_data=expected_version_contents, 129 infra_step=True) 130 actual_version_contents = api.json.loads(version_file_contents) 131 differences = (set(expected_version_contents.items()) ^ 132 set(actual_version_contents.items())) 133 download_skps = len(differences) != 0 134 if download_skps: 135 # Delete and recreate the skps dir. 136 api.run.rmtree(skps_dir) 137 api.file.makedirs(api.path.basename(skps_dir), skps_dir) 138 139 # If a blacklist file exists then specify SKPs to be blacklisted. 140 blacklists_dir = api.vars.skia_dir.join('infra', 'bots', 'ct', 'blacklists') 141 blacklist_file = blacklists_dir.join( 142 '%s_%s_%s.json' % (skia_tool, ct_page_type, skps_chromium_build)) 143 blacklist_skps = [] 144 if api.path.exists(blacklist_file): # pragma: nocover 145 blacklist_file_contents = api.file.read( 146 "Read %s" % blacklist_file, 147 blacklist_file, 148 infra_step=True) 149 blacklist_skps = api.json.loads(blacklist_file_contents)['blacklisted_skps'] 150 151 for slave_num in range(1, ct_num_slaves + 1): 152 if download_skps: 153 # Download SKPs. 154 api.ct.download_swarming_skps( 155 ct_page_type, slave_num, skps_chromium_build, 156 skps_dir, 157 start_range=((slave_num-1)*num_per_slave) + 1, 158 num_skps=num_per_slave) 159 160 # Create this slave's isolated.gen.json file to use for batcharchiving. 161 extra_variables = { 162 'SLAVE_NUM': str(slave_num), 163 'TOOL_NAME': skia_tool, 164 'GIT_HASH': api.vars.got_revision, 165 'CONFIGURATION': api.vars.configuration, 166 'BUILDER': buildername, 167 } 168 api.swarming.create_isolated_gen_json( 169 isolate_path, isolate_dir, 'linux', 'ct-%s-%s' % (skia_tool, slave_num), 170 extra_variables, blacklist=blacklist_skps) 171 172 if download_skps: 173 # Since we had to download SKPs create an updated version file. 174 api.file.write("Create %s" % version_file, 175 version_file, 176 api.json.dumps(expected_version_contents), 177 infra_step=True) 178 179 # Batcharchive everything on the isolate server for efficiency. 180 max_slaves_to_batcharchive = MAX_SLAVES_TO_BATCHARCHIVE 181 if '1m' in buildername: 182 # Break up the "isolate tests" step into batches with <100k files due to 183 # https://github.com/luci/luci-go/issues/9 184 max_slaves_to_batcharchive = 5 185 tasks_to_swarm_hashes = [] 186 for slave_start_num in xrange(1, ct_num_slaves+1, max_slaves_to_batcharchive): 187 m = min(max_slaves_to_batcharchive, ct_num_slaves) 188 batcharchive_output = api.swarming.batcharchive( 189 targets=['ct-' + skia_tool + '-%s' % num for num in range( 190 slave_start_num, slave_start_num + m)]) 191 tasks_to_swarm_hashes.extend(batcharchive_output) 192 # Sort the list to go through tasks in order. 193 tasks_to_swarm_hashes.sort() 194 195 # Trigger all swarming tasks. 196 dimensions={'os': 'Ubuntu-14.04', 'cpu': 'x86-64', 'pool': 'Chrome'} 197 if 'GPU' in buildername: 198 dimensions['gpu'] = '10de:104a' 199 tasks = api.swarming.trigger_swarming_tasks( 200 tasks_to_swarm_hashes, dimensions=dimensions, io_timeout=40*60) 201 202 # Now collect all tasks. 203 env = {'AWS_CREDENTIAL_FILE': None, 'BOTO_CONFIG': None} 204 failed_tasks = [] 205 for task in tasks: 206 try: 207 api.swarming.collect_swarming_task(task) 208 209 if skia_tool == 'nanobench': 210 output_dir = api.swarming.tasks_output_dir.join( 211 task.title).join('0') 212 utc = api.time.utcnow() 213 gs_dest_dir = 'ct/%s/%d/%02d/%02d/%02d/' % ( 214 ct_page_type, utc.year, utc.month, utc.day, utc.hour) 215 for json_output in api.file.listdir('output dir', output_dir): 216 with api.step.context({'env': env}): 217 api.gsutil.upload( 218 name='upload json output', 219 source=output_dir.join(json_output), 220 bucket='skia-perf', 221 dest=gs_dest_dir, 222 args=['-R'] 223 ) 224 225 except api.step.StepFailure as e: 226 # Add SKP links for convenience. 227 api.step.active_result.presentation.links['Webpage rankings'] = ( 228 webpage_rankings_link) 229 api.step.active_result.presentation.links['Download SKPs by rank'] = ( 230 download_skps_link) 231 failed_tasks.append(e) 232 233 if failed_tasks: 234 raise api.step.StepFailure( 235 'Failed steps: %s' % ', '.join([f.name for f in failed_tasks])) 236 237 238def GenTests(api): 239 ct_num_slaves = 5 240 num_per_slave = 10 241 skia_revision = 'abc123' 242 mastername = 'client.skia' 243 slavename = 'skiabot-linux-swarm-000' 244 buildnumber = 2 245 path_config = 'kitchen' 246 247 yield( 248 api.test('CT_DM_10k_SKPs') + 249 api.properties( 250 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs', 251 mastername=mastername, 252 slavename=slavename, 253 buildnumber=buildnumber, 254 path_config=path_config, 255 swarm_out_dir='[SWARM_OUT_DIR]', 256 ct_num_slaves=ct_num_slaves, 257 num_per_slave=num_per_slave, 258 repository='https://skia.googlesource.com/skia.git', 259 revision=skia_revision, 260 ) 261 ) 262 263 yield( 264 api.test('CT_IMG_DECODE_10k_SKPs') + 265 api.properties( 266 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_' 267 '10k_SKPs', 268 mastername=mastername, 269 slavename=slavename, 270 buildnumber=buildnumber, 271 path_config=path_config, 272 swarm_out_dir='[SWARM_OUT_DIR]', 273 ct_num_slaves=ct_num_slaves, 274 num_per_slave=num_per_slave, 275 repository='https://skia.googlesource.com/skia.git', 276 revision=skia_revision, 277 ) 278 ) 279 280 yield( 281 api.test('CT_DM_100k_SKPs') + 282 api.properties( 283 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_100k_SKPs', 284 mastername=mastername, 285 slavename=slavename, 286 buildnumber=buildnumber, 287 path_config=path_config, 288 swarm_out_dir='[SWARM_OUT_DIR]', 289 ct_num_slaves=ct_num_slaves, 290 num_per_slave=num_per_slave, 291 repository='https://skia.googlesource.com/skia.git', 292 revision=skia_revision, 293 ) 294 ) 295 296 yield( 297 api.test('CT_IMG_DECODE_100k_SKPs') + 298 api.properties( 299 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_' 300 '100k_SKPs', 301 mastername=mastername, 302 slavename=slavename, 303 buildnumber=buildnumber, 304 path_config=path_config, 305 swarm_out_dir='[SWARM_OUT_DIR]', 306 ct_num_slaves=ct_num_slaves, 307 num_per_slave=num_per_slave, 308 repository='https://skia.googlesource.com/skia.git', 309 revision=skia_revision, 310 ) 311 ) 312 313 yield( 314 api.test('CT_GPU_BENCH_1k_SKPs') + 315 api.properties( 316 buildername= 317 'Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_1k_SKPs', 318 mastername=mastername, 319 slavename=slavename, 320 buildnumber=buildnumber, 321 path_config=path_config, 322 swarm_out_dir='[SWARM_OUT_DIR]', 323 ct_num_slaves=ct_num_slaves, 324 num_per_slave=num_per_slave, 325 repository='https://skia.googlesource.com/skia.git', 326 revision=skia_revision, 327 ) + 328 api.path.exists( 329 api.path['start_dir'].join('skia'), 330 api.path['start_dir'].join('src') 331 ) 332 ) 333 334 yield( 335 api.test('CT_CPU_BENCH_10k_SKPs') + 336 api.properties( 337 buildername= 338 'Perf-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Release-CT_BENCH_10k_SKPs', 339 mastername=mastername, 340 slavename=slavename, 341 buildnumber=buildnumber, 342 path_config=path_config, 343 swarm_out_dir='[SWARM_OUT_DIR]', 344 ct_num_slaves=ct_num_slaves, 345 num_per_slave=num_per_slave, 346 repository='https://skia.googlesource.com/skia.git', 347 revision=skia_revision, 348 ) + 349 api.path.exists( 350 api.path['start_dir'].join('skia'), 351 api.path['start_dir'].join('src') 352 ) 353 ) 354 355 yield( 356 api.test('CT_GPU_BENCH_10k_SKPs') + 357 api.properties( 358 buildername= 359 'Perf-Ubuntu-GCC-Golo-GPU-GT610-x86_64-Release-CT_BENCH_10k_SKPs', 360 mastername=mastername, 361 slavename=slavename, 362 buildnumber=buildnumber, 363 path_config=path_config, 364 swarm_out_dir='[SWARM_OUT_DIR]', 365 ct_num_slaves=ct_num_slaves, 366 num_per_slave=num_per_slave, 367 repository='https://skia.googlesource.com/skia.git', 368 revision=skia_revision, 369 ) + 370 api.path.exists( 371 api.path['start_dir'].join('skia'), 372 api.path['start_dir'].join('src') 373 ) 374 ) 375 376 yield( 377 api.test('CT_DM_1m_SKPs') + 378 api.properties( 379 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs', 380 mastername=mastername, 381 slavename=slavename, 382 buildnumber=buildnumber, 383 path_config=path_config, 384 swarm_out_dir='[SWARM_OUT_DIR]', 385 ct_num_slaves=ct_num_slaves, 386 num_per_slave=num_per_slave, 387 repository='https://skia.googlesource.com/skia.git', 388 revision=skia_revision, 389 ) 390 ) 391 392 yield ( 393 api.test('CT_DM_SKPs_UnknownBuilder') + 394 api.properties( 395 buildername= 396 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_UnknownRepo_SKPs', 397 mastername=mastername, 398 slavename=slavename, 399 buildnumber=buildnumber, 400 path_config=path_config, 401 swarm_out_dir='[SWARM_OUT_DIR]', 402 ct_num_slaves=ct_num_slaves, 403 num_per_slave=num_per_slave, 404 repository='https://skia.googlesource.com/skia.git', 405 revision=skia_revision, 406 ) + 407 api.expect_exception('Exception') 408 ) 409 410 yield ( 411 api.test('CT_10k_SKPs_UnknownBuilder') + 412 api.properties( 413 buildername= 414 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_UnknownTool_10k_SKPs', 415 mastername=mastername, 416 slavename=slavename, 417 buildnumber=buildnumber, 418 path_config=path_config, 419 swarm_out_dir='[SWARM_OUT_DIR]', 420 ct_num_slaves=ct_num_slaves, 421 num_per_slave=num_per_slave, 422 repository='https://skia.googlesource.com/skia.git', 423 revision=skia_revision, 424 ) + 425 api.expect_exception('Exception') 426 ) 427 428 yield( 429 api.test('CT_DM_1m_SKPs_slave3_failure') + 430 api.step_data('ct-dm-3', retcode=1) + 431 api.properties( 432 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs', 433 mastername=mastername, 434 slavename=slavename, 435 buildnumber=buildnumber, 436 path_config=path_config, 437 swarm_out_dir='[SWARM_OUT_DIR]', 438 ct_num_slaves=ct_num_slaves, 439 num_per_slave=num_per_slave, 440 repository='https://skia.googlesource.com/skia.git', 441 revision=skia_revision, 442 ) 443 ) 444 445 yield( 446 api.test('CT_DM_1m_SKPs_2slaves_failure') + 447 api.step_data('ct-dm-1', retcode=1) + 448 api.step_data('ct-dm-3', retcode=1) + 449 api.properties( 450 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_1m_SKPs', 451 mastername=mastername, 452 slavename=slavename, 453 buildnumber=buildnumber, 454 path_config=path_config, 455 swarm_out_dir='[SWARM_OUT_DIR]', 456 ct_num_slaves=ct_num_slaves, 457 num_per_slave=num_per_slave, 458 repository='https://skia.googlesource.com/skia.git', 459 revision=skia_revision, 460 ) 461 ) 462 463 yield( 464 api.test('CT_DM_10k_SKPs_Trybot') + 465 api.properties( 466 buildername= 467 'Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_DM_10k_SKPs-Trybot', 468 mastername=mastername, 469 slavename=slavename, 470 buildnumber=buildnumber, 471 path_config=path_config, 472 swarm_out_dir='[SWARM_OUT_DIR]', 473 ct_num_slaves=ct_num_slaves, 474 num_per_slave=num_per_slave, 475 rietveld='codereview.chromium.org', 476 issue=1499623002, 477 patchset=1, 478 repository='https://skia.googlesource.com/skia.git', 479 ) 480 ) 481 482 yield( 483 api.test('CT_IMG_DECODE_10k_SKPs_Trybot') + 484 api.properties( 485 buildername='Test-Ubuntu-GCC-GCE-CPU-AVX2-x86_64-Debug-CT_IMG_DECODE_' 486 '10k_SKPs_Trybot', 487 mastername=mastername, 488 slavename=slavename, 489 buildnumber=buildnumber, 490 path_config=path_config, 491 swarm_out_dir='[SWARM_OUT_DIR]', 492 ct_num_slaves=ct_num_slaves, 493 num_per_slave=num_per_slave, 494 repository='https://skia.googlesource.com/skia.git', 495 revision=skia_revision, 496 ) 497 ) 498