1ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org#!/usr/bin/env python 2ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org# Copyright 2014 the V8 project authors. All rights reserved. 3ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org# Use of this source code is governed by a BSD-style license that can be 4ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org# found in the LICENSE file. 5ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 6ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgfrom collections import namedtuple 7ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgimport coverage 8ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgimport json 9ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgfrom mock import DEFAULT 10ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgfrom mock import MagicMock 11ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgimport os 12ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgfrom os import path, sys 13ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgimport shutil 14ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgimport tempfile 15ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgimport unittest 16ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 17ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org# Requires python-coverage and python-mock. Native python coverage 18ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org# version >= 3.7.1 should be installed to get the best speed. 19ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 20ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgTEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-perf") 21ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 22ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgV8_JSON = { 23ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "path": ["."], 24ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "binary": "d7", 25ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "flags": ["--flag"], 26ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "main": "run.js", 27ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "run_count": 1, 28ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "results_regexp": "^%s: (.+)$", 29ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "tests": [ 30ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Richards"}, 31ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "DeltaBlue"}, 32ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ] 33ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 34ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 35ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgV8_NESTED_SUITES_JSON = { 36ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "path": ["."], 37ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "flags": ["--flag"], 38ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "run_count": 1, 39ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "units": "score", 40ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "tests": [ 41ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Richards", 42ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "path": ["richards"], 43ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "binary": "d7", 44ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "main": "run.js", 45ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "resources": ["file1.js", "file2.js"], 46ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "run_count": 2, 47ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "results_regexp": "^Richards: (.+)$"}, 48ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Sub", 49ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "path": ["sub"], 50ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "tests": [ 51ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Leaf", 52ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "path": ["leaf"], 53ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "run_count_x64": 3, 54ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "units": "ms", 55ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "main": "run.js", 56ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "results_regexp": "^Simple: (.+) ms.$"}, 57ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ] 58ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org }, 59ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "DeltaBlue", 60ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "path": ["delta_blue"], 61ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "main": "run.js", 62ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "flags": ["--flag2"], 63ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "results_regexp": "^DeltaBlue: (.+)$"}, 64ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "ShouldntRun", 65ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "path": ["."], 66ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "archs": ["arm"], 67ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "main": "run.js"}, 68ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ] 69ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 70ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 71ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgV8_GENERIC_JSON = { 72ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "path": ["."], 73ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "binary": "cc", 74ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "flags": ["--flag"], 75ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "generic": True, 76ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "run_count": 1, 77ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "units": "ms", 78ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 79ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 809e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.orgOutput = namedtuple("Output", "stdout, stderr, timed_out") 81ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 82ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgclass PerfTest(unittest.TestCase): 83ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org @classmethod 84ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def setUpClass(cls): 85ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org cls.base = path.dirname(path.dirname(path.abspath(__file__))) 86ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org sys.path.append(cls.base) 87ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org cls._cov = coverage.coverage( 88ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org include=([os.path.join(cls.base, "run_perf.py")])) 89ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org cls._cov.start() 90ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org import run_perf 91ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org from testrunner.local import commands 92ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org global commands 93ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org global run_perf 94ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 95ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org @classmethod 96ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def tearDownClass(cls): 97ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org cls._cov.stop() 98ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org print "" 99ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org print cls._cov.report() 100ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 101ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def setUp(self): 102ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.maxDiff = None 103ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if path.exists(TEST_WORKSPACE): 104ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org shutil.rmtree(TEST_WORKSPACE) 105ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org os.makedirs(TEST_WORKSPACE) 106ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 107ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def tearDown(self): 108ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if path.exists(TEST_WORKSPACE): 109ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org shutil.rmtree(TEST_WORKSPACE) 110ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 111ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def _WriteTestInput(self, json_content): 112ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._test_input = path.join(TEST_WORKSPACE, "test.json") 113ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org with open(self._test_input, "w") as f: 114ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org f.write(json.dumps(json_content)) 115ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 1169e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org def _MockCommand(self, *args, **kwargs): 117ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org # Fake output for each test run. 1189e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org test_outputs = [Output(stdout=arg, 1199e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org stderr=None, 1209e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org timed_out=kwargs.get("timed_out", False)) 1219e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org for arg in args[1]] 122ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def execute(*args, **kwargs): 123ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return test_outputs.pop() 124ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org commands.Execute = MagicMock(side_effect=execute) 125ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 126ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org # Check that d8 is called from the correct cwd for each test run. 127ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org dirs = [path.join(TEST_WORKSPACE, arg) for arg in args[0]] 128ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def chdir(*args, **kwargs): 129ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(dirs.pop(), args[0]) 130ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org os.chdir = MagicMock(side_effect=chdir) 131ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 132ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def _CallMain(self, *args): 133ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._test_output = path.join(TEST_WORKSPACE, "results.json") 134ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org all_args=[ 135ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "--json-test-results", 136ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._test_output, 137ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._test_input, 138ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ] 139ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org all_args += args 140ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return run_perf.Main(all_args) 141ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 142ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def _LoadResults(self): 143ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org with open(self._test_output) as f: 144ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return json.load(f) 145ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 146ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def _VerifyResults(self, suite, units, traces): 147ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals([ 148ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"units": units, 149ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "graphs": [suite, trace["name"]], 150ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "results": trace["results"], 151ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "stddev": trace["stddev"]} for trace in traces], 152ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._LoadResults()["traces"]) 153ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 154ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def _VerifyErrors(self, errors): 155ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(errors, self._LoadResults()["errors"]) 156ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 1579e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org def _VerifyMock(self, binary, *args, **kwargs): 158ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org arg = [path.join(path.dirname(self.base), binary)] 159ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org arg += args 1609e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org commands.Execute.assert_called_with( 1619e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org arg, timeout=kwargs.get("timeout", 60)) 162ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 1639e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org def _VerifyMockMultiple(self, *args, **kwargs): 164ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org expected = [] 165ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org for arg in args: 166ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org a = [path.join(path.dirname(self.base), arg[0])] 167ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org a += arg[1:] 1689e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org expected.append(((a,), {"timeout": kwargs.get("timeout", 60)})) 169ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(expected, commands.Execute.call_args_list) 170ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 171ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def testOneRun(self): 172ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._WriteTestInput(V8_JSON) 173ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"]) 174ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(0, self._CallMain()) 175ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyResults("test", "score", [ 176ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Richards", "results": ["1.234"], "stddev": ""}, 177ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 178ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ]) 179ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyErrors([]) 180ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 181ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 182ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def testTwoRuns_Units_SuiteName(self): 183ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input = dict(V8_JSON) 184ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input["run_count"] = 2 185ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input["name"] = "v8" 186ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input["units"] = "ms" 187ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._WriteTestInput(test_input) 188ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._MockCommand([".", "."], 189ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ["Richards: 100\nDeltaBlue: 200\n", 190ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Richards: 50\nDeltaBlue: 300\n"]) 191ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(0, self._CallMain()) 192ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyResults("v8", "ms", [ 193ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Richards", "results": ["50", "100"], "stddev": ""}, 194ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""}, 195ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ]) 196ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyErrors([]) 197ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 198ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 199ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def testTwoRuns_SubRegexp(self): 200ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input = dict(V8_JSON) 201ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input["run_count"] = 2 202ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org del test_input["results_regexp"] 203ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input["tests"][0]["results_regexp"] = "^Richards: (.+)$" 204ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input["tests"][1]["results_regexp"] = "^DeltaBlue: (.+)$" 205ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._WriteTestInput(test_input) 206ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._MockCommand([".", "."], 207ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ["Richards: 100\nDeltaBlue: 200\n", 208ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Richards: 50\nDeltaBlue: 300\n"]) 209ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(0, self._CallMain()) 210ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyResults("test", "score", [ 211ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Richards", "results": ["50", "100"], "stddev": ""}, 212ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""}, 213ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ]) 214ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyErrors([]) 215ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 216ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 217ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def testNestedSuite(self): 218ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._WriteTestInput(V8_NESTED_SUITES_JSON) 219ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._MockCommand(["delta_blue", "sub/leaf", "richards"], 220ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ["DeltaBlue: 200\n", 221ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Simple: 1 ms.\n", 222ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Simple: 2 ms.\n", 223ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Simple: 3 ms.\n", 224ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Richards: 100\n", 225ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Richards: 50\n"]) 226ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(0, self._CallMain()) 227ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals([ 228ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"units": "score", 229ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "graphs": ["test", "Richards"], 230ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "results": ["50", "100"], 231ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "stddev": ""}, 232ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"units": "ms", 233ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "graphs": ["test", "Sub", "Leaf"], 234ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "results": ["3", "2", "1"], 235ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "stddev": ""}, 236ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"units": "score", 237ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "graphs": ["test", "DeltaBlue"], 238ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "results": ["200"], 239ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "stddev": ""}, 240ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ], self._LoadResults()["traces"]) 241ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyErrors([]) 242ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyMockMultiple( 243ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org (path.join("out", "x64.release", "d7"), "--flag", "file1.js", 244ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "file2.js", "run.js"), 245ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org (path.join("out", "x64.release", "d7"), "--flag", "file1.js", 246ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "file2.js", "run.js"), 247ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 248ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 249ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 250ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js")) 251ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 252ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def testOneRunStdDevRegExp(self): 253ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input = dict(V8_JSON) 254ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" 255ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._WriteTestInput(test_input) 256ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._MockCommand(["."], ["Richards: 1.234\nRichards-stddev: 0.23\n" 257ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n"]) 258ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(0, self._CallMain()) 259ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyResults("test", "score", [ 260ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Richards", "results": ["1.234"], "stddev": "0.23"}, 261ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "DeltaBlue", "results": ["10657567"], "stddev": "106"}, 262ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ]) 263ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyErrors([]) 264ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 265ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 266ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def testTwoRunsStdDevRegExp(self): 267ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input = dict(V8_JSON) 268ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" 269ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input["run_count"] = 2 270ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._WriteTestInput(test_input) 271ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n" 272ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n", 273ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Richards: 2\nRichards-stddev: 0.5\n" 274ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"]) 275ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(1, self._CallMain()) 276ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyResults("test", "score", [ 277ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Richards", "results": ["2", "3"], "stddev": "0.7"}, 278ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "DeltaBlue", "results": ["5", "6"], "stddev": "0.8"}, 279ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ]) 280ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyErrors( 281ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ["Test Richards should only run once since a stddev is provided " 282ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "by the test.", 283ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Test DeltaBlue should only run once since a stddev is provided " 284ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "by the test.", 285ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for test " 286ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "DeltaBlue."]) 287ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 288ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 289ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def testBuildbot(self): 290ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._WriteTestInput(V8_JSON) 291ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) 292ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(0, self._CallMain("--buildbot")) 293ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyResults("test", "score", [ 294ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Richards", "results": ["1.234"], "stddev": ""}, 295ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 296ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ]) 297ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyErrors([]) 298ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 299ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 300ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def testBuildbotWithTotal(self): 301ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input = dict(V8_JSON) 302ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input["total"] = True 303ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._WriteTestInput(test_input) 304ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) 305ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(0, self._CallMain("--buildbot")) 306ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyResults("test", "score", [ 307ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Richards", "results": ["1.234"], "stddev": ""}, 308ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 309ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Total", "results": ["3626.49109719"], "stddev": ""}, 310ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ]) 311ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyErrors([]) 312ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 313ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 314ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def testBuildbotWithTotalAndErrors(self): 315ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input = dict(V8_JSON) 316ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input["total"] = True 317ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._WriteTestInput(test_input) 318ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) 319ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(1, self._CallMain("--buildbot")) 320ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyResults("test", "score", [ 321ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Richards", "results": [], "stddev": ""}, 322ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 323ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ]) 324ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyErrors( 325ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ["Regexp \"^Richards: (.+)$\" didn't match for test Richards.", 326ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Not all traces have the same number of results."]) 327ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 328ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 329ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def testRegexpNoMatch(self): 330ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._WriteTestInput(V8_JSON) 331ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) 332ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(1, self._CallMain()) 333ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyResults("test", "score", [ 334ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Richards", "results": [], "stddev": ""}, 335ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 336ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ]) 337ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyErrors( 338ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ["Regexp \"^Richards: (.+)$\" didn't match for test Richards."]) 339ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 340ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 341ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org def testOneRunGeneric(self): 342ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org test_input = dict(V8_GENERIC_JSON) 343ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._WriteTestInput(test_input) 344ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._MockCommand(["."], [ 345ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Trace(Test1), Result(1.234), StdDev(0.23)\n" 346ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org "Trace(Test2), Result(10657567), StdDev(106)\n"]) 347ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self.assertEquals(0, self._CallMain()) 348ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyResults("test", "ms", [ 349ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Test1", "results": ["1.234"], "stddev": "0.23"}, 350ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org {"name": "Test2", "results": ["10657567"], "stddev": "106"}, 351ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ]) 352ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyErrors([]) 353ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "") 3549e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org 3559e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org def testOneRunTimingOut(self): 3569e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org test_input = dict(V8_JSON) 3579e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org test_input["timeout"] = 70 3589e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org self._WriteTestInput(test_input) 3599e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org self._MockCommand(["."], [""], timed_out=True) 3609e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org self.assertEquals(1, self._CallMain()) 3619e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org self._VerifyResults("test", "score", [ 3629e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org {"name": "Richards", "results": [], "stddev": ""}, 3639e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org {"name": "DeltaBlue", "results": [], "stddev": ""}, 3649e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org ]) 3659e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org self._VerifyErrors([ 3669e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org "Regexp \"^Richards: (.+)$\" didn't match for test Richards.", 3679e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org "Regexp \"^DeltaBlue: (.+)$\" didn't match for test DeltaBlue.", 3689e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org ]) 3699e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org self._VerifyMock( 3709e2b466e4b4a2026caefa79afe6737f1bad83a19machenbach@chromium.org path.join("out", "x64.release", "d7"), "--flag", "run.js", timeout=70) 371