1#!/usr/bin/env python 2# Copyright 2014 the V8 project authors. All rights reserved. 3# Use of this source code is governed by a BSD-style license that can be 4# found in the LICENSE file. 5 6from collections import namedtuple 7import coverage 8import json 9from mock import DEFAULT 10from mock import MagicMock 11import os 12from os import path, sys 13import shutil 14import tempfile 15import unittest 16 17# Requires python-coverage and python-mock. Native python coverage 18# version >= 3.7.1 should be installed to get the best speed. 19 20TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-perf") 21 22V8_JSON = { 23 "path": ["."], 24 "binary": "d7", 25 "flags": ["--flag"], 26 "main": "run.js", 27 "run_count": 1, 28 "results_regexp": "^%s: (.+)$", 29 "tests": [ 30 {"name": "Richards"}, 31 {"name": "DeltaBlue"}, 32 ] 33} 34 35V8_NESTED_SUITES_JSON = { 36 "path": ["."], 37 "flags": ["--flag"], 38 "run_count": 1, 39 "units": "score", 40 "tests": [ 41 {"name": "Richards", 42 "path": ["richards"], 43 "binary": "d7", 44 "main": "run.js", 45 "resources": ["file1.js", "file2.js"], 46 "run_count": 2, 47 "results_regexp": "^Richards: (.+)$"}, 48 {"name": "Sub", 49 "path": ["sub"], 50 "tests": [ 51 {"name": "Leaf", 52 "path": ["leaf"], 53 "run_count_x64": 3, 54 "units": "ms", 55 "main": "run.js", 56 "results_regexp": "^Simple: (.+) ms.$"}, 57 ] 58 }, 59 {"name": "DeltaBlue", 60 "path": ["delta_blue"], 61 "main": "run.js", 62 "flags": ["--flag2"], 63 "results_regexp": "^DeltaBlue: (.+)$"}, 64 {"name": "ShouldntRun", 65 "path": ["."], 66 "archs": ["arm"], 67 "main": "run.js"}, 68 ] 69} 70 71V8_GENERIC_JSON = { 72 "path": ["."], 73 "binary": "cc", 74 "flags": ["--flag"], 75 "generic": True, 76 "run_count": 1, 77 "units": "ms", 78} 79 80Output = namedtuple("Output", "stdout, stderr, timed_out") 81 82class PerfTest(unittest.TestCase): 83 @classmethod 84 def setUpClass(cls): 85 cls.base = path.dirname(path.dirname(path.abspath(__file__))) 86 sys.path.append(cls.base) 87 cls._cov = coverage.coverage( 88 include=([os.path.join(cls.base, "run_perf.py")])) 89 cls._cov.start() 90 import run_perf 91 from testrunner.local import commands 92 global commands 93 global run_perf 94 95 @classmethod 96 def tearDownClass(cls): 97 cls._cov.stop() 98 print "" 99 print cls._cov.report() 100 101 def setUp(self): 102 self.maxDiff = None 103 if path.exists(TEST_WORKSPACE): 104 shutil.rmtree(TEST_WORKSPACE) 105 os.makedirs(TEST_WORKSPACE) 106 107 def tearDown(self): 108 if path.exists(TEST_WORKSPACE): 109 shutil.rmtree(TEST_WORKSPACE) 110 111 def _WriteTestInput(self, json_content): 112 self._test_input = path.join(TEST_WORKSPACE, "test.json") 113 with open(self._test_input, "w") as f: 114 f.write(json.dumps(json_content)) 115 116 def _MockCommand(self, *args, **kwargs): 117 # Fake output for each test run. 118 test_outputs = [Output(stdout=arg, 119 stderr=None, 120 timed_out=kwargs.get("timed_out", False)) 121 for arg in args[1]] 122 def execute(*args, **kwargs): 123 return test_outputs.pop() 124 commands.Execute = MagicMock(side_effect=execute) 125 126 # Check that d8 is called from the correct cwd for each test run. 127 dirs = [path.join(TEST_WORKSPACE, arg) for arg in args[0]] 128 def chdir(*args, **kwargs): 129 self.assertEquals(dirs.pop(), args[0]) 130 os.chdir = MagicMock(side_effect=chdir) 131 132 def _CallMain(self, *args): 133 self._test_output = path.join(TEST_WORKSPACE, "results.json") 134 all_args=[ 135 "--json-test-results", 136 self._test_output, 137 self._test_input, 138 ] 139 all_args += args 140 return run_perf.Main(all_args) 141 142 def _LoadResults(self): 143 with open(self._test_output) as f: 144 return json.load(f) 145 146 def _VerifyResults(self, suite, units, traces): 147 self.assertEquals([ 148 {"units": units, 149 "graphs": [suite, trace["name"]], 150 "results": trace["results"], 151 "stddev": trace["stddev"]} for trace in traces], 152 self._LoadResults()["traces"]) 153 154 def _VerifyErrors(self, errors): 155 self.assertEquals(errors, self._LoadResults()["errors"]) 156 157 def _VerifyMock(self, binary, *args, **kwargs): 158 arg = [path.join(path.dirname(self.base), binary)] 159 arg += args 160 commands.Execute.assert_called_with( 161 arg, timeout=kwargs.get("timeout", 60)) 162 163 def _VerifyMockMultiple(self, *args, **kwargs): 164 expected = [] 165 for arg in args: 166 a = [path.join(path.dirname(self.base), arg[0])] 167 a += arg[1:] 168 expected.append(((a,), {"timeout": kwargs.get("timeout", 60)})) 169 self.assertEquals(expected, commands.Execute.call_args_list) 170 171 def testOneRun(self): 172 self._WriteTestInput(V8_JSON) 173 self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"]) 174 self.assertEquals(0, self._CallMain()) 175 self._VerifyResults("test", "score", [ 176 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 177 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 178 ]) 179 self._VerifyErrors([]) 180 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 181 182 def testTwoRuns_Units_SuiteName(self): 183 test_input = dict(V8_JSON) 184 test_input["run_count"] = 2 185 test_input["name"] = "v8" 186 test_input["units"] = "ms" 187 self._WriteTestInput(test_input) 188 self._MockCommand([".", "."], 189 ["Richards: 100\nDeltaBlue: 200\n", 190 "Richards: 50\nDeltaBlue: 300\n"]) 191 self.assertEquals(0, self._CallMain()) 192 self._VerifyResults("v8", "ms", [ 193 {"name": "Richards", "results": ["50", "100"], "stddev": ""}, 194 {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""}, 195 ]) 196 self._VerifyErrors([]) 197 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 198 199 def testTwoRuns_SubRegexp(self): 200 test_input = dict(V8_JSON) 201 test_input["run_count"] = 2 202 del test_input["results_regexp"] 203 test_input["tests"][0]["results_regexp"] = "^Richards: (.+)$" 204 test_input["tests"][1]["results_regexp"] = "^DeltaBlue: (.+)$" 205 self._WriteTestInput(test_input) 206 self._MockCommand([".", "."], 207 ["Richards: 100\nDeltaBlue: 200\n", 208 "Richards: 50\nDeltaBlue: 300\n"]) 209 self.assertEquals(0, self._CallMain()) 210 self._VerifyResults("test", "score", [ 211 {"name": "Richards", "results": ["50", "100"], "stddev": ""}, 212 {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""}, 213 ]) 214 self._VerifyErrors([]) 215 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 216 217 def testNestedSuite(self): 218 self._WriteTestInput(V8_NESTED_SUITES_JSON) 219 self._MockCommand(["delta_blue", "sub/leaf", "richards"], 220 ["DeltaBlue: 200\n", 221 "Simple: 1 ms.\n", 222 "Simple: 2 ms.\n", 223 "Simple: 3 ms.\n", 224 "Richards: 100\n", 225 "Richards: 50\n"]) 226 self.assertEquals(0, self._CallMain()) 227 self.assertEquals([ 228 {"units": "score", 229 "graphs": ["test", "Richards"], 230 "results": ["50", "100"], 231 "stddev": ""}, 232 {"units": "ms", 233 "graphs": ["test", "Sub", "Leaf"], 234 "results": ["3", "2", "1"], 235 "stddev": ""}, 236 {"units": "score", 237 "graphs": ["test", "DeltaBlue"], 238 "results": ["200"], 239 "stddev": ""}, 240 ], self._LoadResults()["traces"]) 241 self._VerifyErrors([]) 242 self._VerifyMockMultiple( 243 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", 244 "file2.js", "run.js"), 245 (path.join("out", "x64.release", "d7"), "--flag", "file1.js", 246 "file2.js", "run.js"), 247 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 248 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 249 (path.join("out", "x64.release", "d8"), "--flag", "run.js"), 250 (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js")) 251 252 def testOneRunStdDevRegExp(self): 253 test_input = dict(V8_JSON) 254 test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" 255 self._WriteTestInput(test_input) 256 self._MockCommand(["."], ["Richards: 1.234\nRichards-stddev: 0.23\n" 257 "DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n"]) 258 self.assertEquals(0, self._CallMain()) 259 self._VerifyResults("test", "score", [ 260 {"name": "Richards", "results": ["1.234"], "stddev": "0.23"}, 261 {"name": "DeltaBlue", "results": ["10657567"], "stddev": "106"}, 262 ]) 263 self._VerifyErrors([]) 264 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 265 266 def testTwoRunsStdDevRegExp(self): 267 test_input = dict(V8_JSON) 268 test_input["stddev_regexp"] = "^%s\-stddev: (.+)$" 269 test_input["run_count"] = 2 270 self._WriteTestInput(test_input) 271 self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n" 272 "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n", 273 "Richards: 2\nRichards-stddev: 0.5\n" 274 "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"]) 275 self.assertEquals(1, self._CallMain()) 276 self._VerifyResults("test", "score", [ 277 {"name": "Richards", "results": ["2", "3"], "stddev": "0.7"}, 278 {"name": "DeltaBlue", "results": ["5", "6"], "stddev": "0.8"}, 279 ]) 280 self._VerifyErrors( 281 ["Test Richards should only run once since a stddev is provided " 282 "by the test.", 283 "Test DeltaBlue should only run once since a stddev is provided " 284 "by the test.", 285 "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for test " 286 "DeltaBlue."]) 287 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 288 289 def testBuildbot(self): 290 self._WriteTestInput(V8_JSON) 291 self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) 292 self.assertEquals(0, self._CallMain("--buildbot")) 293 self._VerifyResults("test", "score", [ 294 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 295 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 296 ]) 297 self._VerifyErrors([]) 298 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 299 300 def testBuildbotWithTotal(self): 301 test_input = dict(V8_JSON) 302 test_input["total"] = True 303 self._WriteTestInput(test_input) 304 self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"]) 305 self.assertEquals(0, self._CallMain("--buildbot")) 306 self._VerifyResults("test", "score", [ 307 {"name": "Richards", "results": ["1.234"], "stddev": ""}, 308 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 309 {"name": "Total", "results": ["3626.49109719"], "stddev": ""}, 310 ]) 311 self._VerifyErrors([]) 312 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 313 314 def testBuildbotWithTotalAndErrors(self): 315 test_input = dict(V8_JSON) 316 test_input["total"] = True 317 self._WriteTestInput(test_input) 318 self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) 319 self.assertEquals(1, self._CallMain("--buildbot")) 320 self._VerifyResults("test", "score", [ 321 {"name": "Richards", "results": [], "stddev": ""}, 322 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 323 ]) 324 self._VerifyErrors( 325 ["Regexp \"^Richards: (.+)$\" didn't match for test Richards.", 326 "Not all traces have the same number of results."]) 327 self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js") 328 329 def testRegexpNoMatch(self): 330 self._WriteTestInput(V8_JSON) 331 self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"]) 332 self.assertEquals(1, self._CallMain()) 333 self._VerifyResults("test", "score", [ 334 {"name": "Richards", "results": [], "stddev": ""}, 335 {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""}, 336 ]) 337 self._VerifyErrors( 338 ["Regexp \"^Richards: (.+)$\" didn't match for test Richards."]) 339 self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js") 340 341 def testOneRunGeneric(self): 342 test_input = dict(V8_GENERIC_JSON) 343 self._WriteTestInput(test_input) 344 self._MockCommand(["."], [ 345 "Trace(Test1), Result(1.234), StdDev(0.23)\n" 346 "Trace(Test2), Result(10657567), StdDev(106)\n"]) 347 self.assertEquals(0, self._CallMain()) 348 self._VerifyResults("test", "ms", [ 349 {"name": "Test1", "results": ["1.234"], "stddev": "0.23"}, 350 {"name": "Test2", "results": ["10657567"], "stddev": "106"}, 351 ]) 352 self._VerifyErrors([]) 353 self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "") 354 355 def testOneRunTimingOut(self): 356 test_input = dict(V8_JSON) 357 test_input["timeout"] = 70 358 self._WriteTestInput(test_input) 359 self._MockCommand(["."], [""], timed_out=True) 360 self.assertEquals(1, self._CallMain()) 361 self._VerifyResults("test", "score", [ 362 {"name": "Richards", "results": [], "stddev": ""}, 363 {"name": "DeltaBlue", "results": [], "stddev": ""}, 364 ]) 365 self._VerifyErrors([ 366 "Regexp \"^Richards: (.+)$\" didn't match for test Richards.", 367 "Regexp \"^DeltaBlue: (.+)$\" didn't match for test DeltaBlue.", 368 ]) 369 self._VerifyMock( 370 path.join("out", "x64.release", "d7"), "--flag", "run.js", timeout=70) 371