12a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)#!/usr/bin/env python 22a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Copyright (c) 2013 The Chromium Authors. All rights reserved. 32a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Use of this source code is governed by a BSD-style license that can be 42a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# found in the LICENSE file. 52a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 62a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import collections 72a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import glob 83551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import hashlib 94e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)import json 102a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import os 113551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)import random 12d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)import re 132a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import shutil 142a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)import sys 152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 16868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)import bb_utils 177dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport bb_annotations 18868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) 192a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)sys.path.append(os.path.join(os.path.dirname(__file__), '..')) 207dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdochimport provision_devices 212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from pylib import android_commands 222a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from pylib import constants 23a02191e04bc25c4935f804f2c080ae28663d096dBen Murdochfrom pylib.device import device_utils 242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)from pylib.gtest import gtest_config 252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2658537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)CHROME_SRC_DIR = bb_utils.CHROME_SRC 271e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)DIR_BUILD_ROOT = os.path.dirname(CHROME_SRC_DIR) 2858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)CHROME_OUT_DIR = bb_utils.CHROME_OUT_DIR 291320f92c476a1ad9d19dba2a48c72b75566198e9Primiano TucciBLINK_SCRIPTS_DIR = 'third_party/WebKit/Tools/Scripts' 302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)SLAVE_SCRIPTS_DIR = os.path.join(bb_utils.BB_BUILD_DIR, 'scripts', 'slave') 3258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles)LOGCAT_DIR = os.path.join(bb_utils.CHROME_OUT_DIR, 'logcat') 33d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)GS_URL = 'https://storage.googleapis.com' 345c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo LiuGS_AUTH_URL = 'https://storage.cloud.google.com' 352a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 362a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# Describes an instrumation test suite: 372a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# test: Name of test we're running. 382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# apk: apk to be installed. 392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# apk_package: package for the apk to be installed. 402a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# test_apk: apk to run tests on. 412a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)# test_data: data folder in format destination:source. 423240926e260ce088908e02ac07a6cf7b0c0cbf44Ben Murdoch# host_driven_root: The host-driven test root directory. 43eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch# annotation: Annotation of the tests to include. 44eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch# exclude_annotation: The annotation of the tests to exclude. 452a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)I_TEST = collections.namedtuple('InstrumentationTest', [ 46eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'name', 'apk', 'apk_package', 'test_apk', 'test_data', 'host_driven_root', 47eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'annotation', 'exclude_annotation', 'extra_flags']) 48eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 501e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles)def SrcPath(*path): 511e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) return os.path.join(CHROME_SRC_DIR, *path) 521e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 531e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) 54eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef I(name, apk, apk_package, test_apk, test_data, host_driven_root=None, 55eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch annotation=None, exclude_annotation=None, extra_flags=None): 56eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch return I_TEST(name, apk, apk_package, test_apk, test_data, host_driven_root, 57eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch annotation, exclude_annotation, extra_flags) 582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)INSTRUMENTATION_TESTS = dict((suite.name, suite) for suite in [ 60eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch I('ContentShell', 61eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'ContentShell.apk', 62eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'org.chromium.content_shell_apk', 63eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'ContentShellTest', 64eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'content:content/test/data/android/device_files'), 65a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) I('ChromeShell', 66a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 'ChromeShell.apk', 67a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 'org.chromium.chrome.shell', 68a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) 'ChromeShellTest', 69eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'chrome:chrome/test/data/android/device_files', 70a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) constants.CHROME_SHELL_HOST_DRIVEN_DIR), 71eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch I('AndroidWebView', 72eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'AndroidWebView.apk', 73eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'org.chromium.android_webview.shell', 74eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'AndroidWebViewTest', 75eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 'webview:android_webview/test/data/device_files'), 762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) ]) 772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 781320f92c476a1ad9d19dba2a48c72b75566198e9Primiano TucciVALID_TESTS = set(['chromedriver', 'chrome_proxy', 'gpu', 'mojo', 'sync', 795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 'telemetry_perf_unittests', 'ui', 'unit', 'webkit', 801320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'webkit_layout']) 812a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 82868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles)RunCmd = bb_utils.RunCmd 832a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 842a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 8568043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles)def _GetRevision(options): 8668043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) """Get the SVN revision number. 8768043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) 8868043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) Args: 8968043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) options: options object. 9068043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) 9168043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) Returns: 9268043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) The revision number. 9368043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) """ 9468043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) revision = options.build_properties.get('got_revision') 9568043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) if not revision: 9668043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) revision = options.build_properties.get('revision', 'testing') 9768043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) return revision 9868043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) 9968043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) 1006e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles)def _RunTest(options, cmd, suite): 1016e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) """Run test command with runtest.py. 1026e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) 1036e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) Args: 1046e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) options: options object. 1056e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) cmd: the command to run. 1066e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) suite: test name. 1076e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) """ 1086e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) property_args = bb_utils.EncodeProperties(options) 1096e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) args = [os.path.join(SLAVE_SCRIPTS_DIR, 'runtest.py')] + property_args 1106e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) args += ['--test-platform', 'android'] 1116e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) if options.factory_properties.get('generate_gtest_json'): 1126e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) args.append('--generate-json-file') 1136e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) args += ['-o', 'gtest-results/%s' % suite, 1146e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) '--annotate', 'gtest', 1156e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) '--build-number', str(options.build_properties.get('buildnumber', 1166e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) '')), 1176e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) '--builder-name', options.build_properties.get('buildername', '')] 1186e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) if options.target == 'Release': 1196e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) args += ['--target', 'Release'] 1206e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) else: 1216e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) args += ['--target', 'Debug'] 1226e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) args += cmd 1236e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) RunCmd(args, cwd=DIR_BUILD_ROOT) 1246e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) 1256e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) 126cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles)def RunTestSuites(options, suites, suites_options=None): 1277dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """Manages an invocation of test_runner.py for gtests. 1282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) Args: 1302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) options: options object. 131fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch suites: List of suite names to run. 132cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) suites_options: Command line options dictionary for particular suites. 133cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) For example, 134cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) {'content_browsertests', ['--num_retries=1', '--release']} 135cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) will add the options only to content_browsertests. 1362a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """ 137cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 138cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) if not suites_options: 139cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) suites_options = {} 140cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 1412a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) args = ['--verbose'] 1422a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if options.target == 'Release': 1432a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) args.append('--release') 1442a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if options.asan: 1452a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) args.append('--tool=asan') 146f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) if options.gtest_filter: 147f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) args.append('--gtest-filter=%s' % options.gtest_filter) 148cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 1492a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for suite in suites: 150fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch bb_annotations.PrintNamedStep(suite) 1516e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) cmd = [suite] + args 152cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) cmd += suites_options.get(suite, []) 153fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch if suite == 'content_browsertests': 154fb250657ef40d7500f20882d5c9909c1013367d3Ben Murdoch cmd.append('--num_retries=1') 1556e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) _RunTest(options, cmd, suite) 1562a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 15868043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles)def RunChromeDriverTests(options): 1592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Run all the steps for running chromedriver tests.""" 1607dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintNamedStep('chromedriver_annotation') 1612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) RunCmd(['chrome/test/chromedriver/run_buildbot_steps.py', 162f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) '--android-packages=%s,%s,%s,%s' % 163a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) ('chrome_shell', 164f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 'chrome_stable', 165f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 'chrome_beta', 166f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) 'chromedriver_webview_shell'), 16768043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) '--revision=%s' % _GetRevision(options), 16868043e1e95eeb07d5cae7aca370b26518b0867d6Torne (Richard Coles) '--update-log']) 1692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 1705f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles)def RunChromeProxyTests(options): 1715f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """Run the chrome_proxy tests. 1725f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) 1735f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) Args: 1745f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) options: options object. 1755f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) """ 1765f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) InstallApk(options, INSTRUMENTATION_TESTS['ChromeShell'], False) 1775f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) args = ['--browser', 'android-chrome-shell'] 1785f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) devices = android_commands.GetAttachedDevices() 1795f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) if devices: 1805f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) args = args + ['--device', devices[0]] 1815f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) bb_annotations.PrintNamedStep('chrome_proxy') 1825f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) RunCmd(['tools/chrome_proxy/run_tests'] + args) 1834e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 1841320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccidef RunChromeSyncShellTests(options): 1851320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci """Run the chrome sync shell tests""" 1861320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci test = I('ChromeSyncShell', 1871320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'ChromeSyncShell.apk', 1881320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'org.chromium.chrome.browser.sync', 1891320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'ChromeSyncShellTest.apk', 1901320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'chrome:chrome/test/data/android/device_files') 1911320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci RunInstrumentationSuite(options, test) 1921320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 1930529e5d033099cbfc42635f6f6183833b09dff6eBen Murdochdef RunTelemetryPerfUnitTests(options): 1940529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch """Runs the telemetry perf unit tests. 1950529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 1960529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch Args: 1970529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch options: options object. 1980529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch """ 1990529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch InstallApk(options, INSTRUMENTATION_TESTS['ChromeShell'], False) 20046d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) args = ['--browser', 'android-chrome-shell'] 2010529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch devices = android_commands.GetAttachedDevices() 2020529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch if devices: 2030529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch args = args + ['--device', devices[0]] 2040529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch bb_annotations.PrintNamedStep('telemetry_perf_unittests') 2050529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch RunCmd(['tools/perf/run_tests'] + args) 2060529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 2070529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 2085c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liudef RunMojoTests(options): 2095c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu """Runs the mojo unit tests. 2105c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 2115c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu Args: 2125c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu options: options object. 2135c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu """ 2145c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu test = I('MojoTest', 2155c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu None, 2165c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 'org.chromium.mojo.tests', 2175c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 'MojoTest', 2181320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci 'bindings:mojo/public/interfaces/bindings/tests/data') 2195c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu RunInstrumentationSuite(options, test) 2205c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 2215c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu 2222a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def InstallApk(options, test, print_step=False): 2232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Install an apk to all phones. 2242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) Args: 2262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) options: options object 2272a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) test: An I_TEST namedtuple 2282a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) print_step: Print a buildbot step 2292a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """ 2302a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if print_step: 2317dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintNamedStep('install_%s' % test.name.lower()) 232424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) 233f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) args = ['--apk_package', test.apk_package] 2342a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if options.target == 'Release': 2352a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) args.append('--release') 236f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) args.append(test.apk) 2372a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 238a93a17c8d99d686bd4a1511e5504e5e6cc9fcadfTorne (Richard Coles) RunCmd(['build/android/adb_install_apk.py'] + args, halt_on_failure=True) 2392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2402a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 241bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdochdef RunInstrumentationSuite(options, test, flunk_on_failure=True, 242424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) python_only=False, official_build=False): 2437dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch """Manages an invocation of test_runner.py for instrumentation tests. 2442a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2452a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) Args: 2462a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) options: options object 2472a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) test: An I_TEST namedtuple 248bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch flunk_on_failure: Flunk the step if tests fail. 249bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch Python: Run only host driven Python tests. 250424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) official_build: Run official-build tests. 2512a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """ 2527dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintNamedStep('%s_instrumentation_tests' % test.name.lower()) 2532a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2545c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu if test.apk: 2555c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu InstallApk(options, test) 2565c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu args = ['--test-apk', test.test_apk, '--verbose'] 2575c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu if test.test_data: 2585c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu args.extend(['--test_data', test.test_data]) 2592a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if options.target == 'Release': 2602a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) args.append('--release') 2612a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if options.asan: 2622a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) args.append('--tool=asan') 263bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch if options.flakiness_server: 2642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) args.append('--flakiness-dashboard-server=%s' % 265bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch options.flakiness_server) 2663551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if options.coverage_bucket: 2673551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) args.append('--coverage-dir=%s' % options.coverage_dir) 2682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if test.host_driven_root: 2693551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) args.append('--host-driven-root=%s' % test.host_driven_root) 270eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if test.annotation: 271eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch args.extend(['-A', test.annotation]) 272eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if test.exclude_annotation: 273eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch args.extend(['-E', test.exclude_annotation]) 274eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch if test.extra_flags: 275eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch args.extend(test.extra_flags) 276bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch if python_only: 277bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch args.append('-p') 278424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) if official_build: 279424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) # The option needs to be assigned 'True' as it does not have an action 280424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) # associated with it. 281424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) args.append('--official-build') 2822a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2839ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch RunCmd(['build/android/test_runner.py', 'instrumentation'] + args, 2849ab5563a3196760eb381d102cbb2bc0f7abc6a50Ben Murdoch flunk_on_failure=flunk_on_failure) 2852a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2862a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2871320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tuccidef RunWebkitLint(): 2882a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Lint WebKit's TestExpectation files.""" 2897dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintNamedStep('webkit_lint') 2901320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci RunCmd([SrcPath(os.path.join(BLINK_SCRIPTS_DIR, 'lint-test-expectations'))]) 2912a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2922a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 2932a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)def RunWebkitLayoutTests(options): 2942a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) """Run layout tests on an actual device.""" 2957dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintNamedStep('webkit_tests') 2962a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) cmd_args = [ 2974e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--no-show-results', 2984e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--no-new-test-results', 2994e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--full-results-html', 3004e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--clobber-old-results', 3014e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--exit-after-n-failures', '5000', 3024e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--exit-after-n-crashes-or-timeouts', '100', 3034e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--debug-rwt-logging', 3044e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--results-directory', '../layout-test-results', 3054e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--target', options.target, 3064e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--builder-name', options.build_properties.get('buildername', ''), 3074e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--build-number', str(options.build_properties.get('buildnumber', '')), 3084e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--master-name', 'ChromiumWebkit', # TODO: Get this from the cfg. 3094e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--build-name', options.build_properties.get('buildername', ''), 3104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--platform=android'] 3112a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3122a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for flag in 'test_results_server', 'driver_name', 'additional_drt_flag': 3132a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if flag in options.factory_properties: 3142a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) cmd_args.extend(['--%s' % flag.replace('_', '-'), 3152a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) options.factory_properties.get(flag)]) 3162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for f in options.factory_properties.get('additional_expectations', []): 3182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) cmd_args.extend( 31958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)]) 3202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # TODO(dpranke): Remove this block after 3222a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # https://codereview.chromium.org/12927002/ lands. 3232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for f in options.factory_properties.get('additional_expectations_files', []): 3242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) cmd_args.extend( 32558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) ['--additional-expectations=%s' % os.path.join(CHROME_SRC_DIR, *f)]) 3262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 3271320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci exit_code = RunCmd( 3281320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci [SrcPath(os.path.join(BLINK_SCRIPTS_DIR, 'run-webkit-tests'))] + cmd_args) 3298bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if exit_code == 255: # test_run_results.UNEXPECTED_ERROR_EXIT_STATUS 3304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) bb_annotations.PrintMsg('?? (crashed or hung)') 3318bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) elif exit_code == 254: # test_run_results.NO_DEVICES_EXIT_STATUS 3328bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bb_annotations.PrintMsg('?? (no devices found)') 3338bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) elif exit_code == 253: # test_run_results.NO_TESTS_EXIT_STATUS 3348bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) bb_annotations.PrintMsg('?? (no tests found)') 3354e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) else: 3364e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) full_results_path = os.path.join('..', 'layout-test-results', 3374e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'full_results.json') 3384e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if os.path.exists(full_results_path): 3394e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) full_results = json.load(open(full_results_path)) 3404e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) unexpected_passes, unexpected_failures, unexpected_flakes = ( 3414e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) _ParseLayoutTestResults(full_results)) 3424e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if unexpected_failures: 3434e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) _PrintDashboardLink('failed', unexpected_failures, 3444e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) max_tests=25) 3454e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) elif unexpected_passes: 3464e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) _PrintDashboardLink('unexpected passes', unexpected_passes, 3474e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) max_tests=10) 3484e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if unexpected_flakes: 3494e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) _PrintDashboardLink('unexpected flakes', unexpected_flakes, 3504e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) max_tests=10) 3514e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 3524e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if exit_code == 0 and (unexpected_passes or unexpected_flakes): 3534e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) # If exit_code != 0, RunCmd() will have already printed an error. 3544e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) bb_annotations.PrintWarning() 3554e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) else: 3564e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) bb_annotations.PrintError() 3574e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) bb_annotations.PrintMsg('?? (results missing)') 3582a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 35958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) if options.factory_properties.get('archive_webkit_results', False): 36058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) bb_annotations.PrintNamedStep('archive_webkit_results') 361d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) base = 'https://storage.googleapis.com/chromium-layout-test-archives' 362d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) builder_name = options.build_properties.get('buildername', '') 363d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) build_number = str(options.build_properties.get('buildnumber', '')) 3644e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) results_link = '%s/%s/%s/layout-test-results/results.html' % ( 3654e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) base, EscapeBuilderName(builder_name), build_number) 3664e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) bb_annotations.PrintLink('results', results_link) 367d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) bb_annotations.PrintLink('(zip)', '%s/%s/%s/layout-test-results.zip' % ( 368d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) base, EscapeBuilderName(builder_name), build_number)) 36958537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) gs_bucket = 'gs://chromium-layout-test-archives' 37058537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) RunCmd([os.path.join(SLAVE_SCRIPTS_DIR, 'chromium', 37158537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 'archive_layout_test_results.py'), 3724e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--results-dir', '../../layout-test-results', 3734e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--build-number', build_number, 3744e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--builder-name', builder_name, 3751e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) '--gs-bucket', gs_bucket], 3761e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) cwd=DIR_BUILD_ROOT) 3774e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 3784e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 3794e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)def _ParseLayoutTestResults(results): 3804e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) """Extract the failures from the test run.""" 3814e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) # Cloned from third_party/WebKit/Tools/Scripts/print-json-test-results 3824e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) tests = _ConvertTrieToFlatPaths(results['tests']) 3834e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) failures = {} 3844e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) flakes = {} 3854e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) passes = {} 3864e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for (test, result) in tests.iteritems(): 3874e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if result.get('is_unexpected'): 3881e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) actual_results = result['actual'].split() 3891e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) expected_results = result['expected'].split() 3901e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if len(actual_results) > 1: 3911e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) # We report the first failure type back, even if the second 3921e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) # was more severe. 3931e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) if actual_results[1] in expected_results: 3941e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) flakes[test] = actual_results[0] 3951e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) else: 3961e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) failures[test] = actual_results[0] 3971e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) elif actual_results[0] == 'PASS': 3984e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) passes[test] = result 3994e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) else: 4001e9bf3e0803691d0a228da41fc608347b6db4340Torne (Richard Coles) failures[test] = actual_results[0] 4014e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 4024e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return (passes, failures, flakes) 4034e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 4044e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 4054e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)def _ConvertTrieToFlatPaths(trie, prefix=None): 4064e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) """Flatten the trie of failures into a list.""" 4074e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) # Cloned from third_party/WebKit/Tools/Scripts/print-json-test-results 4084e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) result = {} 4094e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) for name, data in trie.iteritems(): 4104e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if prefix: 4114e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) name = prefix + '/' + name 4124e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 4134e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if len(data) and 'actual' not in data and 'expected' not in data: 4144e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) result.update(_ConvertTrieToFlatPaths(data, name)) 4154e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) else: 4164e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) result[name] = data 4174e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 4184e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) return result 4194e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 4204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 4214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles)def _PrintDashboardLink(link_text, tests, max_tests): 4224e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) """Add a link to the flakiness dashboard in the step annotations.""" 4234e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) if len(tests) > max_tests: 4244e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_list_text = ' '.join(tests[:max_tests]) + ' and more' 4254e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) else: 4264e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) test_list_text = ' '.join(tests) 4274e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 4284e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) dashboard_base = ('http://test-results.appspot.com' 4294e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '/dashboards/flakiness_dashboard.html#' 4304e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'master=ChromiumWebkit&tests=') 4314e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 4324e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) bb_annotations.PrintLink('%d %s: %s' % 4334e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) (len(tests), link_text, test_list_text), 4344e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) dashboard_base + ','.join(tests)) 43558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 4362a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 437d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)def EscapeBuilderName(builder_name): 438d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) return re.sub('[ ()]', '_', builder_name) 439d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 440d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 441eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef SpawnLogcatMonitor(): 442eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch shutil.rmtree(LOGCAT_DIR, ignore_errors=True) 443eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch bb_utils.SpawnCmd([ 44458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) os.path.join(CHROME_SRC_DIR, 'build', 'android', 'adb_logcat_monitor.py'), 445eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch LOGCAT_DIR]) 446eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 447eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch # Wait for logcat_monitor to pull existing logcat 448eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch RunCmd(['sleep', '5']) 449eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 4504e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 451eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef ProvisionDevices(options): 4527dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintNamedStep('provision_devices') 4533551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 4543551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if not bb_utils.TESTING: 4553551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) # Restart adb to work around bugs, sleep to wait for usb discovery. 456cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) device_utils.RestartServer() 4573551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) RunCmd(['sleep', '1']) 4587dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch provision_cmd = ['build/android/provision_devices.py', '-t', options.target] 4597dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch if options.auto_reconnect: 4607dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch provision_cmd.append('--auto-reconnect') 461f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) if options.skip_wipe: 462f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) provision_cmd.append('--skip-wipe') 463116680a4aac90f2aa7413d9095a592090648e557Ben Murdoch RunCmd(provision_cmd, halt_on_failure=True) 4642a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 465eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 4668bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles)def DeviceStatusCheck(options): 4677dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintNamedStep('device_status_check') 4688bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) cmd = ['build/android/buildbot/bb_device_status_check.py'] 4698bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) if options.restart_usb: 4708bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) cmd.append('--restart-usb') 4718bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) RunCmd(cmd, halt_on_failure=True) 4722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 474eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef GetDeviceSetupStepCmds(): 475eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch return [ 4764e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) ('device_status_check', DeviceStatusCheck), 477a3f6a49ab37290eeeb8db0f41ec0f1cb74a68be7Torne (Richard Coles) ('provision_devices', ProvisionDevices), 478eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch ] 4792a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 4802a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 481eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef RunUnitTests(options): 482a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) suites = gtest_config.STABLE_TEST_SUITES 483a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if options.asan: 484a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) suites = [s for s in suites 485a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) if s not in gtest_config.ASAN_EXCLUDED_TEST_SUITES] 486a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) RunTestSuites(options, suites) 487eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 488eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 489eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef RunInstrumentationTests(options): 490eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch for test in INSTRUMENTATION_TESTS.itervalues(): 491eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch RunInstrumentationSuite(options, test) 492eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 493eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 494eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef RunWebkitTests(options): 4955d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) RunTestSuites(options, ['webkit_unit_tests', 'blink_heap_unittests']) 4961320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci RunWebkitLint() 497ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch 498ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch 499d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)def RunGPUTests(options): 5005d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) revision = _GetRevision(options) 501cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) builder_name = options.build_properties.get('buildername', 'noname') 502cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) 503cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) bb_annotations.PrintNamedStep('pixel_tests') 504a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) RunCmd(['content/test/gpu/run_gpu_test.py', 5055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'pixel', 5065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '--browser', 5075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'android-content-shell', 5085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '--build-revision', 5095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) str(revision), 5105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '--upload-refimg-to-cloud-storage', 5115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '--refimg-cloud-storage-bucket', 5125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'chromium-gpu-archive/reference-images', 5135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '--os-type', 5145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'android', 5155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '--test-machine-name', 516cedac228d2dd51db4b79ea1e72c7f249408ee061Torne (Richard Coles) EscapeBuilderName(builder_name)]) 5178bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) 5184e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) bb_annotations.PrintNamedStep('webgl_conformance_tests') 519a1401311d1ab56c4ed0a474bd38c108f75cb0cd9Torne (Richard Coles) RunCmd(['content/test/gpu/run_gpu_test.py', 5204e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--browser=android-content-shell', 'webgl_conformance', 5214e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '--webgl-conformance-version=1.0.1']) 52258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 52346d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) bb_annotations.PrintNamedStep('gpu_rasterization_tests') 52446d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) RunCmd(['content/test/gpu/run_gpu_test.py', 52546d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 'gpu_rasterization', 52646d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) '--browser', 52746d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 'android-content-shell', 52846d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) '--build-revision', 52946d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) str(revision), 53046d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) '--test-machine-name', 53146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) EscapeBuilderName(builder_name)]) 53246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 53358537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) 534eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef GetTestStepCmds(): 535eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch return [ 536eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch ('chromedriver', RunChromeDriverTests), 5375f1c94371a64b3196d4be9466099bb892df9b88eTorne (Richard Coles) ('chrome_proxy', RunChromeProxyTests), 53858537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) ('gpu', RunGPUTests), 5395c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu ('mojo', RunMojoTests), 5401320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci ('sync', RunChromeSyncShellTests), 5410529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch ('telemetry_perf_unittests', RunTelemetryPerfUnitTests), 542eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch ('ui', RunInstrumentationTests), 5431320f92c476a1ad9d19dba2a48c72b75566198e9Primiano Tucci ('unit', RunUnitTests), 544eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch ('webkit', RunWebkitTests), 545ba5b9a6411cb1792fd21f0a078d7a25cd1ceec16Ben Murdoch ('webkit_layout', RunWebkitLayoutTests), 546eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch ] 547eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 548eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 5490529e5d033099cbfc42635f6f6183833b09dff6eBen Murdochdef MakeGSPath(options, gs_base_dir): 5500529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch revision = _GetRevision(options) 5510529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch bot_id = options.build_properties.get('buildername', 'testing') 5520529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch randhash = hashlib.sha1(str(random.random())).hexdigest() 5530529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch gs_path = '%s/%s/%s/%s' % (gs_base_dir, bot_id, revision, randhash) 5545c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu # remove double slashes, happens with blank revisions and confuses gsutil 5555c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu gs_path = re.sub('/+', '/', gs_path) 5560529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch return gs_path 5570529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch 558d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles)def UploadHTML(options, gs_base_dir, dir_to_upload, link_text, 559d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) link_rel_path='index.html', gs_url=GS_URL): 560d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) """Uploads directory at |dir_to_upload| to Google Storage and output a link. 5613551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 5623551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) Args: 5633551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) options: Command line options. 564d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) gs_base_dir: The Google Storage base directory (e.g. 565d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 'chromium-code-coverage/java') 566d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) dir_to_upload: Absolute path to the directory to be uploaded. 567d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) link_text: Link text to be displayed on the step. 568d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) link_rel_path: Link path relative to |dir_to_upload|. 569d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) gs_url: Google storage URL. 5703551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) """ 5710529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch gs_path = MakeGSPath(options, gs_base_dir) 572d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) RunCmd([bb_utils.GSUTIL_PATH, 'cp', '-R', dir_to_upload, 'gs://%s' % gs_path]) 573d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) bb_annotations.PrintLink(link_text, 5744e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) '%s/%s/%s' % (gs_url, gs_path, link_rel_path)) 5753551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 5763551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 5773551c9c881056c480085172ff9840cab31610854Torne (Richard Coles)def GenerateJavaCoverageReport(options): 5783551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) """Generates an HTML coverage report using EMMA and uploads it.""" 5793551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) bb_annotations.PrintNamedStep('java_coverage_report') 5803551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 5813551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) coverage_html = os.path.join(options.coverage_dir, 'coverage_html') 5823551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) RunCmd(['build/android/generate_emma_html.py', 5833551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) '--coverage-dir', options.coverage_dir, 58458537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) '--metadata-dir', os.path.join(CHROME_OUT_DIR, options.target), 585424c4d7b64af9d0d8fd9624f381f469654d5e3d2Torne (Richard Coles) '--cleanup', 5863551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) '--output', os.path.join(coverage_html, 'index.html')]) 587d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) return coverage_html 5883551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 5893551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 590eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef LogcatDump(options): 5912a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) # Print logcat, kill logcat monitor 5927dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintNamedStep('logcat_dump') 593010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) logcat_file = os.path.join(CHROME_OUT_DIR, options.target, 'full_log.txt') 594f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) RunCmd([SrcPath('build' , 'android', 'adb_logcat_printer.py'), 595f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) '--output-path', logcat_file, LOGCAT_DIR]) 5960529e5d033099cbfc42635f6f6183833b09dff6eBen Murdoch gs_path = MakeGSPath(options, 'chromium-android/logcat_dumps') 597010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) RunCmd([bb_utils.GSUTIL_PATH, 'cp', '-z', 'txt', logcat_file, 598010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) 'gs://%s' % gs_path]) 5995c02ac1a9c1b504631c0a3d2b6e737b5d738bae1Bo Liu bb_annotations.PrintLink('logcat dump', '%s/%s' % (GS_AUTH_URL, gs_path)) 600eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 6012a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 6025d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles)def RunStackToolSteps(options): 6035d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) """Run stack tool steps. 6045d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 6055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) Stack tool is run for logcat dump, optionally for ASAN. 6065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) """ 6075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bb_annotations.PrintNamedStep('Run stack tool with logcat dump') 608010d83a9304c5a91596085d917d248abff47903aTorne (Richard Coles) logcat_file = os.path.join(CHROME_OUT_DIR, options.target, 'full_log.txt') 6095d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) RunCmd([os.path.join(CHROME_SRC_DIR, 'third_party', 'android_platform', 6105d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 'development', 'scripts', 'stack'), 6115d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '--more-info', logcat_file]) 6125d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if options.asan_symbolize: 6135d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) bb_annotations.PrintNamedStep('Run stack tool for ASAN') 6145d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) RunCmd([ 6155d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) os.path.join(CHROME_SRC_DIR, 'build', 'android', 'asan_symbolize.py'), 6165d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) '-l', logcat_file]) 6175d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 6185d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) 619eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef GenerateTestReport(options): 6207dbb3d5cf0c15f500944d211057644d6a2f37371Ben Murdoch bb_annotations.PrintNamedStep('test_report') 6212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) for report in glob.glob( 62258537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) os.path.join(CHROME_OUT_DIR, options.target, 'test_logs', '*.log')): 6232a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) RunCmd(['cat', report]) 6242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) os.remove(report) 6252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 6262a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 627eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef MainTestWrapper(options): 628a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) try: 629a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) # Spawn logcat monitor 630a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) SpawnLogcatMonitor() 631eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 632a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) # Run all device setup steps 633a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) for _, cmd in GetDeviceSetupStepCmds(): 634a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) cmd(options) 635eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 636a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) if options.install: 637a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) test_obj = INSTRUMENTATION_TESTS[options.install] 638a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) InstallApk(options, test_obj, print_step=True) 639eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 640a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) if options.test_filter: 641a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) bb_utils.RunSteps(options.test_filter, GetTestStepCmds(), options) 642eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 6433551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if options.coverage_bucket: 644d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) coverage_html = GenerateJavaCoverageReport(options) 645d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) UploadHTML(options, '%s/java' % options.coverage_bucket, coverage_html, 646d0247b1b59f9c528cb6df88b4f2b9afaf80d181eTorne (Richard Coles) 'Coverage Report') 6475d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) shutil.rmtree(coverage_html, ignore_errors=True) 6483551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 649a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) if options.experimental: 650a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) RunTestSuites(options, gtest_config.EXPERIMENTAL_TEST_SUITES) 651eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 652a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) finally: 653a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) # Run all post test steps 654a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) LogcatDump(options) 6555d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) if not options.disable_stack_tool: 6565d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) RunStackToolSteps(options) 657a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) GenerateTestReport(options) 658a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) # KillHostHeartbeat() has logic to check if heartbeat process is running, 659a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) # and kills only if it finds the process is running on the host. 660a36e5920737c6adbddd3e43b760e5de8431db6e0Torne (Richard Coles) provision_devices.KillHostHeartbeat() 6616e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) if options.cleanup: 6626e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) shutil.rmtree(os.path.join(CHROME_OUT_DIR, options.target), 6636e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) ignore_errors=True) 664eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 665eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 666eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef GetDeviceStepsOptParser(): 667868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) parser = bb_utils.GetParser() 6682a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parser.add_option('--experimental', action='store_true', 6692a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) help='Run experiemental tests') 6702a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parser.add_option('-f', '--test-filter', metavar='<filter>', default=[], 6712a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) action='append', 6722a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) help=('Run a test suite. Test suites: "%s"' % 6732a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) '", "'.join(VALID_TESTS))) 674f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) parser.add_option('--gtest-filter', 675f2477e01787aa58f445919b809d89e252beef54fTorne (Richard Coles) help='Filter for running a subset of tests of a gtest test') 6762a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parser.add_option('--asan', action='store_true', help='Run tests with asan.') 6772a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) parser.add_option('--install', metavar='<apk name>', 6782a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) help='Install an apk by name') 6795d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser.add_option('--no-reboot', action='store_true', 6805d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) help='Do not reboot devices during provisioning.') 6813551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) parser.add_option('--coverage-bucket', 6823551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) help=('Bucket name to store coverage results. Coverage is ' 6833551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) 'only run if this is set.')) 6848bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) parser.add_option('--restart-usb', action='store_true', 6858bcbed890bc3ce4d7a057a8f32cab53fa534672eTorne (Richard Coles) help='Restart usb ports before device status check.') 686bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch parser.add_option( 687bbcdd45c55eb7c4641ab97aef9889b0fc828e7d3Ben Murdoch '--flakiness-server', 6884e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) help=('The flakiness dashboard server to which the results should be ' 6894e180b6a0b4720a9b8e9e959a882386f690f08ffTorne (Richard Coles) 'uploaded.')) 690c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) parser.add_option( 691c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) '--auto-reconnect', action='store_true', 692c2e0dbddbe15c98d52c4786dac06cb8952a8ae6dTorne (Richard Coles) help='Push script to device which restarts adbd on disconnections.') 693f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) parser.add_option('--skip-wipe', action='store_true', 694f8ee788a64d60abd8f2d742a5fdedde054ecd910Torne (Richard Coles) help='Do not wipe devices during provisioning.') 695eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch parser.add_option( 696eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch '--logcat-dump-output', 697eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch help='The logcat dump output will be "tee"-ed into this file') 69846d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) # During processing perf bisects, a seperate working directory created under 69946d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) # which builds are produced. Therefore we should look for relevent output 70046d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) # file under this directory.(/b/build/slave/<slave_name>/build/bisect/src/out) 70146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) parser.add_option( 70246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) '--chrome-output-dir', 70346d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) help='Chrome output directory to be used while bisecting.') 70446d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 7055d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser.add_option('--disable-stack-tool', action='store_true', 7065d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) help='Do not run stack tool.') 7075d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) parser.add_option('--asan-symbolize', action='store_true', 7085d1f7b1de12d16ceb2c938c56701a3e8bfa558f7Torne (Richard Coles) help='Run stack tool for ASAN') 7096e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) parser.add_option('--cleanup', action='store_true', 7106e8cce623b6e4fe0c9e4af605d675dd9d0338c38Torne (Richard Coles) help='Delete out/<target> directory at the end of the run.') 711eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch return parser 712eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 713eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch 714eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdochdef main(argv): 715eb525c5499e34cc9c4b825d6d9e75bb07cc06aceBen Murdoch parser = GetDeviceStepsOptParser() 7162a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) options, args = parser.parse_args(argv[1:]) 7172a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 7182a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if args: 719868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return sys.exit('Unused args %s' % args) 7202a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 7212a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) unknown_tests = set(options.test_filter) - VALID_TESTS 7222a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) if unknown_tests: 723868fa2fe829687343ffae624259930155e16dbd8Torne (Richard Coles) return sys.exit('Unknown tests %s' % list(unknown_tests)) 7242a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 7252a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) setattr(options, 'target', options.factory_properties.get('target', 'Debug')) 72646d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 72746d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) if options.chrome_output_dir: 72846d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) global CHROME_OUT_DIR 72946d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) global LOGCAT_DIR 73046d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) CHROME_OUT_DIR = options.chrome_output_dir 73146d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) LOGCAT_DIR = os.path.join(CHROME_OUT_DIR, 'logcat') 73246d4c2bc3267f3f028f39e7e311b0f89aba2e4fdTorne (Richard Coles) 7333551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) if options.coverage_bucket: 7343551c9c881056c480085172ff9840cab31610854Torne (Richard Coles) setattr(options, 'coverage_dir', 73558537e28ecd584eab876aee8be7156509866d23aTorne (Richard Coles) os.path.join(CHROME_OUT_DIR, options.target, 'coverage')) 7362a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 7372a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) MainTestWrapper(options) 7382a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 7392a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) 7402a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles)if __name__ == '__main__': 7412a99a7e74a7f215066514fe81d2bfa6639d9edddTorne (Richard Coles) sys.exit(main(sys.argv)) 742