15c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Copyright (C) 2012 Google Inc. All rights reserved. 25c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 35c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# Redistribution and use in source and binary forms, with or without 45c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# modification, are permitted provided that the following conditions are 55c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# met: 65c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 75c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions of source code must retain the above copyright 85c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# notice, this list of conditions and the following disclaimer. 95c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Redistributions in binary form must reproduce the above 105c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# copyright notice, this list of conditions and the following disclaimer 115c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# in the documentation and/or other materials provided with the 125c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# distribution. 135c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# * Neither the name of Google Inc. nor the names of its 145c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# contributors may be used to endorse or promote products derived from 155c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# this software without specific prior written permission. 165c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# 175c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 185c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 195c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 205c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 215c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 225c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 235c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 245c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 255c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 265c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 275c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 285c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 297757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch"""Run Inspector's perf tests in perf mode.""" 305c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 315c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import os 327757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochimport json 335c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import logging 345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)import optparse 357757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochimport time 367757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochimport datetime 375c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 387757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.common import find_files 397757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.common.checkout.scm.detection import SCMDetector 407757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.common.config.urls import view_source_url 415c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)from webkitpy.common.host import Host 427757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.common.net.file_uploader import FileUploader 437757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.performance_tests.perftest import PerfTestFactory 447757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdochfrom webkitpy.performance_tests.perftest import DEFAULT_TEST_RUNNER_COUNT 457757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 465c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 475c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)_log = logging.getLogger(__name__) 485c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 495c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 505c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles)class PerfTestsRunner(object): 517757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _default_branch = 'webkit-trunk' 527757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch EXIT_CODE_BAD_BUILD = -1 537757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch EXIT_CODE_BAD_SOURCE_JSON = -2 547757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch EXIT_CODE_BAD_MERGE = -3 557757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch EXIT_CODE_FAILED_UPLOADING = -4 567757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch EXIT_CODE_BAD_PREPARATION = -5 577757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 587757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _DEFAULT_JSON_FILENAME = 'PerformanceTestsResults.json' 595c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 605c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def __init__(self, args=None, port=None): 615c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._options, self._args = PerfTestsRunner._parse_args(args) 625c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) if port: 635c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._port = port 645c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._host = self._port.host 655c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) else: 665c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) self._host = Host() 677757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._port = self._host.port_factory.get(self._options.platform, self._options) 687757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._host.initialize_scm() 697757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._webkit_base_dir_len = len(self._port.webkit_base()) 707757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._base_path = self._port.perf_tests_dir() 717757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._timestamp = time.time() 727757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._utc_timestamp = datetime.datetime.utcnow() 737757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 745c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 755c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) @staticmethod 765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _parse_args(args=None): 775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def _expand_path(option, opt_str, value, parser): 785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) path = os.path.expandvars(os.path.expanduser(value)) 795c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) setattr(parser.values, option.dest, path) 805c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) perf_option_list = [ 817757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option('--debug', action='store_const', const='Debug', dest="configuration", 827757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help='Set the configuration to Debug'), 837757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option('--release', action='store_const', const='Release', dest="configuration", 847757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help='Set the configuration to Release'), 857757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--platform", 867757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Specify port/platform being tested (e.g. mac)"), 877757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--chromium", 887757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch action="store_const", const='chromium', dest='platform', help='Alias for --platform=chromium'), 89e69819bd8e388ea4ad1636a19aa6b2eed4952191Ben Murdoch optparse.make_option("--android", 907757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch action="store_const", const='android', dest='platform', help='Alias for --platform=android'), 917757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--builder-name", 927757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help=("The name of the builder shown on the waterfall running this script e.g. google-mac-2.")), 937757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--build-number", 947757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help=("The build number of the builder running this script.")), 957757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--build", dest="build", action="store_true", default=True, 967757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Check to ensure the DumpRenderTree build is up-to-date (default)."), 977757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--no-build", dest="build", action="store_false", 987757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Don't check to see if the DumpRenderTree build is up-to-date."), 997757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--build-directory", 1007757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Path to the directory under which build files are kept (should not include configuration)"), 1017757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--time-out-ms", default=600 * 1000, 1027757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Set the timeout for each test"), 1037757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--no-results", action="store_false", dest="generate_results", default=True, 1047757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Do no generate results JSON and results page."), 1057757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--output-json-path", action='callback', callback=_expand_path, type="str", 1067757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Path to generate a JSON file at; may contain previous results if it already exists."), 1077757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--reset-results", action="store_true", 1087757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Clears the content in the generated JSON file before adding the results."), 1097757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--slave-config-json-path", action='callback', callback=_expand_path, type="str", 1107757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Only used on bots. Path to a slave configuration file."), 1117757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--description", 1127757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Add a description to the output JSON file if one is generated"), 1137757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--no-show-results", action="store_false", default=True, dest="show_results", 1147757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Don't launch a browser with results after the tests are done"), 1157757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--test-results-server", 1167757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Upload the generated JSON file to the specified server when --output-json-path is present."), 1177757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--force", dest="use_skipped_list", action="store_false", default=True, 1187757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Run all tests, including the ones in the Skipped list."), 1197757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--profile", action="store_true", 1207757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Output per-test profile information."), 121926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) optparse.make_option("--profiler", action="store", 1227757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Output per-test profile information, using the specified profiler."), 1237757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--additional-drt-flag", action="append", 1247757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch default=[], help="Additional command line flag to pass to DumpRenderTree " 125926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) "Specify multiple times to add multiple flags."), 1267757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--driver-name", type="string", 1277757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Alternative DumpRenderTree binary to use"), 1287757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--content-shell", action="store_true", 1297757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Use Content Shell instead of DumpRenderTree"), 1307757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--repeat", default=1, type="int", 1317757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Specify number of times to run test set (default: 1)."), 1327757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch optparse.make_option("--test-runner-count", default=DEFAULT_TEST_RUNNER_COUNT, type="int", 1337757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch help="Specify number of times to invoke test runner for each performance test."), 1345c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) ] 1355c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) return optparse.OptionParser(option_list=(perf_option_list)).parse_args(args) 1365c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1377757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _collect_tests(self): 1387757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch test_extensions = ['.html', '.svg'] 1397757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1407757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _is_test_file(filesystem, dirname, filename): 1417757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return filesystem.splitext(filename)[1] in test_extensions 1427757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1437757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem = self._host.filesystem 1447757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1457757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch paths = [] 1467757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for arg in self._args: 1477757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if filesystem.exists(filesystem.join(self._base_path, arg)): 1487757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch paths.append(arg) 1497757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch else: 1507757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch relpath = filesystem.relpath(arg, self._base_path) 1517757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if filesystem.exists(filesystem.join(self._base_path, relpath)): 1527757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch paths.append(filesystem.normpath(relpath)) 1537757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch else: 1547757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.warn('Path was not found:' + arg) 1557757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1567757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch skipped_directories = set(['.svn', 'resources']) 1577757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch test_files = find_files.find(filesystem, self._base_path, paths, skipped_directories, _is_test_file) 1587757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests = [] 1597757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for path in test_files: 1607757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch relative_path = filesystem.relpath(path, self._base_path).replace('\\', '/') 1617757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if self._options.use_skipped_list and self._port.skips_perf_test(relative_path) and filesystem.normpath(relative_path) not in paths: 1627757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch continue 1637757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch test = PerfTestFactory.create_perf_test(self._port, relative_path, path, test_runner_count=self._options.test_runner_count) 1647757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests.append(test) 1657757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1667757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return tests 1677757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1687757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _start_http_servers(self): 1697757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._port.acquire_http_lock() 1707757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._port.start_http_server(number_of_servers=2) 1717757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1727757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _stop_http_servers(self): 1737757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._port.stop_http_server() 1747757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._port.release_http_lock() 1757757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 1765c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) def run(self): 1777757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch needs_http = self._port.requires_http_server() 1785c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1799bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) class FakePrinter(object): 1809bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) def write_update(self, msg): 1819bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) print msg 1829bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) 1839bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) def write_throttled_update(self, msg): 1849bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) pass 1859bbd2f5e390b01907d97ecffde80aa1b06113aacTorne (Richard Coles) 186f79f16f17ddc4f842d7b7a38603e280e94be826aTorne (Richard Coles) if self._port.check_build(needs_http=needs_http, printer=FakePrinter()): 1877757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.error("Build not up to date for %s" % self._port._path_to_driver()) 1887757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return self.EXIT_CODE_BAD_BUILD 189926b001d589ce2f10facb93dd4b87578ea35a855Torne (Richard Coles) 1907757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch run_count = 0 1917757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch repeat = self._options.repeat 1927757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch while (run_count < repeat): 1937757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch run_count += 1 1945c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1957757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests = self._collect_tests() 1967757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch runs = ' (Run %d of %d)' % (run_count, repeat) if repeat > 1 else '' 1977757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.info("Running %d tests%s" % (len(tests), runs)) 1985c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 1997757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for test in tests: 2007757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if not test.prepare(self._options.time_out_ms): 2017757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return self.EXIT_CODE_BAD_PREPARATION 2027757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2037757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch try: 2047757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if needs_http: 2057757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._start_http_servers() 2067757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch unexpected = self._run_tests_set(sorted(list(tests), key=lambda test: test.test_name())) 2077757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2087757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch finally: 2097757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if needs_http: 2107757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._stop_http_servers() 2117757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2127757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if self._options.generate_results and not self._options.profile: 2137757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch exit_code = self._generate_results() 2147757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if exit_code: 2157757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return exit_code 2167757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2177757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if self._options.generate_results and not self._options.profile: 2187757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch test_results_server = self._options.test_results_server 2197757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if test_results_server and not self._upload_json(test_results_server, self._output_json_path()): 2207757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return self.EXIT_CODE_FAILED_UPLOADING 2217757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2227757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if self._options.show_results: 2237757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._port.show_results_html_file(self._results_page_path()) 2247757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2257757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return unexpected 2267757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2277757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _output_json_path(self): 2287757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output_json_path = self._options.output_json_path 2297757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if output_json_path: 2307757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return output_json_path 2317757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return self._host.filesystem.join(self._port.perf_results_directory(), self._DEFAULT_JSON_FILENAME) 2327757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2337757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _results_page_path(self): 2347757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return self._host.filesystem.splitext(self._output_json_path())[0] + '.html' 2357757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2367757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _generate_results(self): 2377757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch options = self._options 2387757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output_json_path = self._output_json_path() 2397757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = self._generate_results_dict(self._timestamp, options.description, options.platform, options.builder_name, options.build_number) 2407757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2417757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if options.slave_config_json_path: 2427757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = self._merge_slave_config_json(options.slave_config_json_path, output) 2437757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if not output: 2447757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return self.EXIT_CODE_BAD_SOURCE_JSON 2457757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2467757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch output = self._merge_outputs_if_needed(output_json_path, output) 2477757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if not output: 2487757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return self.EXIT_CODE_BAD_MERGE 2497757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2507757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem = self._host.filesystem 2517757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch json_output = json.dumps(output) 2527757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem.write_text_file(output_json_path, json_output) 2537757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2547757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch template_path = filesystem.join(self._port.perf_tests_dir(), 'resources/results-template.html') 2557757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch template = filesystem.read_text_file(template_path) 2567757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2577757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch absolute_path_to_trunk = filesystem.dirname(self._port.perf_tests_dir()) 2587757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch results_page = template.replace('%AbsolutePathToWebKitTrunk%', absolute_path_to_trunk) 2597757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch results_page = results_page.replace('%PeformanceTestsResultsJSON%', json_output) 2607757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2617757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch filesystem.write_text_file(self._results_page_path(), results_page) 2627757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2637757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _generate_results_dict(self, timestamp, description, platform, builder_name, build_number): 2647757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch revisions = {} 2657757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for (name, path) in self._port.repository_paths(): 2667757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch scm = SCMDetector(self._host.filesystem, self._host.executive).detect_scm_system(path) or self._host.scm() 2677757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch revision = scm.svn_revision(path) 2687757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch revisions[name] = {'revision': revision, 'timestamp': scm.timestamp_of_revision(path, revision)} 2697757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2707757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch meta_info = { 2717757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'description': description, 2727757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'buildTime': self._datetime_in_ES5_compatible_iso_format(self._utc_timestamp), 2737757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'platform': platform, 2747757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'revisions': revisions, 2757757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'builderName': builder_name, 2767757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 'buildNumber': int(build_number) if build_number else None} 2777757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2787757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch contents = {'tests': {}} 2797757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for key, value in meta_info.items(): 2807757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if value: 2817757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch contents[key] = value 2827757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2837757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for test, metrics in self._results: 2847757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for metric_name, iteration_values in metrics.iteritems(): 2857757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if not isinstance(iteration_values, list): # We can't reports results without individual measurements. 2867757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch continue 2877757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 2887757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests = contents['tests'] 2897757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch path = test.test_name_without_file_extension().split('/') 2907757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for i in range(0, len(path)): 2917757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch is_last_token = i + 1 == len(path) 2927757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch url = view_source_url('PerformanceTests/' + (test.test_name() if is_last_token else '/'.join(path[0:i + 1]))) 2937757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests.setdefault(path[i], {'url': url}) 2947757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch current_test = tests[path[i]] 2957757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if is_last_token: 2967757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch current_test.setdefault('metrics', {}) 2977757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch assert metric_name not in current_test['metrics'] 2987757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch current_test['metrics'][metric_name] = {'current': iteration_values} 2997757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch else: 3007757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch current_test.setdefault('tests', {}) 3017757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch tests = current_test['tests'] 3027757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3037757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return contents 3047757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3057757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch @staticmethod 3067757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _datetime_in_ES5_compatible_iso_format(datetime): 3077757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return datetime.strftime('%Y-%m-%dT%H:%M:%S.%f') 3087757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3097757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _merge_slave_config_json(self, slave_config_json_path, contents): 3107757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if not self._host.filesystem.isfile(slave_config_json_path): 3117757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.error("Missing slave configuration JSON file: %s" % slave_config_json_path) 3127757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return None 3137757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3147757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch try: 3157757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch slave_config_json = self._host.filesystem.open_text_file_for_reading(slave_config_json_path) 3167757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch slave_config = json.load(slave_config_json) 3177757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for key in slave_config: 3187757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch contents['builder' + key.capitalize()] = slave_config[key] 3197757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return contents 3207757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch except Exception, error: 3217757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.error("Failed to merge slave configuration JSON file %s: %s" % (slave_config_json_path, error)) 3227757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return None 3237757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3247757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _merge_outputs_if_needed(self, output_json_path, output): 3257757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if self._options.reset_results or not self._host.filesystem.isfile(output_json_path): 3267757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return [output] 3277757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch try: 3287757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch existing_outputs = json.loads(self._host.filesystem.read_text_file(output_json_path)) 3297757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return existing_outputs + [output] 3307757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch except Exception, error: 3317757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.error("Failed to merge output JSON file %s: %s" % (output_json_path, error)) 3327757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return None 3337757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3347757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _upload_json(self, test_results_server, json_path, host_path="/api/report", file_uploader=FileUploader): 3357757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch url = "https://%s%s" % (test_results_server, host_path) 3367757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch uploader = file_uploader(url, 120) 3377757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch try: 3387757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch response = uploader.upload_single_text_file(self._host.filesystem, 'application/json', json_path) 3397757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch except Exception, error: 3407757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.error("Failed to upload JSON file to %s in 120s: %s" % (url, error)) 3417757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return False 3427757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3437757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch response_body = [line.strip('\n') for line in response] 3447757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if response_body != ['OK']: 3457757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch try: 3467757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch parsed_response = json.loads('\n'.join(response_body)) 3477757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch except: 3487757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.error("Uploaded JSON to %s but got a bad response:" % url) 3497757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for line in response_body: 3507757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.error(line) 3517757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return False 3527757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if parsed_response.get('status') != 'OK': 3537757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.error("Uploaded JSON to %s but got an error:" % url) 3547757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.error(json.dumps(parsed_response, indent=4)) 3557757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return False 3567757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3577757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.info("JSON file uploaded to %s." % url) 3587757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return True 3597757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3607757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch def _run_tests_set(self, tests): 3617757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch result_count = len(tests) 3627757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch failures = 0 3637757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._results = [] 3647757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3657757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch for i, test in enumerate(tests): 3667757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.info('Running %s (%d of %d)' % (test.test_name(), i + 1, len(tests))) 3677757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch start_time = time.time() 3687757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch metrics = test.run(self._options.time_out_ms) 3697757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch if metrics: 3707757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch self._results.append((test, metrics)) 3717757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch else: 3727757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch failures += 1 3737757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.error('FAILED') 3747757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch 3757757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.info('Finished: %f s' % (time.time() - start_time)) 3767757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch _log.info('') 3775c87bf8b86a7c82ef50fb7a89697d8e02e2553beTorne (Richard Coles) 3787757ec2eadfa2dd8ac2aeed0a4399e9b07ec38cbBen Murdoch return failures 379