1#!/usr/bin/env python
2# Copyright 2014 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5
6from collections import namedtuple
7import coverage
8import json
9from mock import DEFAULT
10from mock import MagicMock
11import os
12from os import path, sys
13import shutil
14import tempfile
15import unittest
16
17# Requires python-coverage and python-mock. Native python coverage
18# version >= 3.7.1 should be installed to get the best speed.
19
20TEST_WORKSPACE = path.join(tempfile.gettempdir(), "test-v8-run-perf")
21
22V8_JSON = {
23  "path": ["."],
24  "binary": "d7",
25  "flags": ["--flag"],
26  "main": "run.js",
27  "run_count": 1,
28  "results_regexp": "^%s: (.+)$",
29  "tests": [
30    {"name": "Richards"},
31    {"name": "DeltaBlue"},
32  ]
33}
34
35V8_NESTED_SUITES_JSON = {
36  "path": ["."],
37  "flags": ["--flag"],
38  "run_count": 1,
39  "units": "score",
40  "tests": [
41    {"name": "Richards",
42     "path": ["richards"],
43     "binary": "d7",
44     "main": "run.js",
45     "resources": ["file1.js", "file2.js"],
46     "run_count": 2,
47     "results_regexp": "^Richards: (.+)$"},
48    {"name": "Sub",
49     "path": ["sub"],
50     "tests": [
51       {"name": "Leaf",
52        "path": ["leaf"],
53        "run_count_x64": 3,
54        "units": "ms",
55        "main": "run.js",
56        "results_regexp": "^Simple: (.+) ms.$"},
57     ]
58    },
59    {"name": "DeltaBlue",
60     "path": ["delta_blue"],
61     "main": "run.js",
62     "flags": ["--flag2"],
63     "results_regexp": "^DeltaBlue: (.+)$"},
64    {"name": "ShouldntRun",
65     "path": ["."],
66     "archs": ["arm"],
67     "main": "run.js"},
68  ]
69}
70
71V8_GENERIC_JSON = {
72  "path": ["."],
73  "binary": "cc",
74  "flags": ["--flag"],
75  "generic": True,
76  "run_count": 1,
77  "units": "ms",
78}
79
80Output = namedtuple("Output", "stdout, stderr, timed_out")
81
82class PerfTest(unittest.TestCase):
83  @classmethod
84  def setUpClass(cls):
85    cls.base = path.dirname(path.dirname(path.abspath(__file__)))
86    sys.path.append(cls.base)
87    cls._cov = coverage.coverage(
88        include=([os.path.join(cls.base, "run_perf.py")]))
89    cls._cov.start()
90    import run_perf
91    from testrunner.local import commands
92    global commands
93    global run_perf
94
95  @classmethod
96  def tearDownClass(cls):
97    cls._cov.stop()
98    print ""
99    print cls._cov.report()
100
101  def setUp(self):
102    self.maxDiff = None
103    if path.exists(TEST_WORKSPACE):
104      shutil.rmtree(TEST_WORKSPACE)
105    os.makedirs(TEST_WORKSPACE)
106
107  def tearDown(self):
108    if path.exists(TEST_WORKSPACE):
109      shutil.rmtree(TEST_WORKSPACE)
110
111  def _WriteTestInput(self, json_content):
112    self._test_input = path.join(TEST_WORKSPACE, "test.json")
113    with open(self._test_input, "w") as f:
114      f.write(json.dumps(json_content))
115
116  def _MockCommand(self, *args, **kwargs):
117    # Fake output for each test run.
118    test_outputs = [Output(stdout=arg,
119                           stderr=None,
120                           timed_out=kwargs.get("timed_out", False))
121                    for arg in args[1]]
122    def execute(*args, **kwargs):
123      return test_outputs.pop()
124    commands.Execute = MagicMock(side_effect=execute)
125
126    # Check that d8 is called from the correct cwd for each test run.
127    dirs = [path.join(TEST_WORKSPACE, arg) for arg in args[0]]
128    def chdir(*args, **kwargs):
129      self.assertEquals(dirs.pop(), args[0])
130    os.chdir = MagicMock(side_effect=chdir)
131
132  def _CallMain(self, *args):
133    self._test_output = path.join(TEST_WORKSPACE, "results.json")
134    all_args=[
135      "--json-test-results",
136      self._test_output,
137      self._test_input,
138    ]
139    all_args += args
140    return run_perf.Main(all_args)
141
142  def _LoadResults(self):
143    with open(self._test_output) as f:
144      return json.load(f)
145
146  def _VerifyResults(self, suite, units, traces):
147    self.assertEquals([
148      {"units": units,
149       "graphs": [suite, trace["name"]],
150       "results": trace["results"],
151       "stddev": trace["stddev"]} for trace in traces],
152      self._LoadResults()["traces"])
153
154  def _VerifyErrors(self, errors):
155    self.assertEquals(errors, self._LoadResults()["errors"])
156
157  def _VerifyMock(self, binary, *args, **kwargs):
158    arg = [path.join(path.dirname(self.base), binary)]
159    arg += args
160    commands.Execute.assert_called_with(
161        arg, timeout=kwargs.get("timeout", 60))
162
163  def _VerifyMockMultiple(self, *args, **kwargs):
164    expected = []
165    for arg in args:
166      a = [path.join(path.dirname(self.base), arg[0])]
167      a += arg[1:]
168      expected.append(((a,), {"timeout": kwargs.get("timeout", 60)}))
169    self.assertEquals(expected, commands.Execute.call_args_list)
170
171  def testOneRun(self):
172    self._WriteTestInput(V8_JSON)
173    self._MockCommand(["."], ["x\nRichards: 1.234\nDeltaBlue: 10657567\ny\n"])
174    self.assertEquals(0, self._CallMain())
175    self._VerifyResults("test", "score", [
176      {"name": "Richards", "results": ["1.234"], "stddev": ""},
177      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
178    ])
179    self._VerifyErrors([])
180    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
181
182  def testTwoRuns_Units_SuiteName(self):
183    test_input = dict(V8_JSON)
184    test_input["run_count"] = 2
185    test_input["name"] = "v8"
186    test_input["units"] = "ms"
187    self._WriteTestInput(test_input)
188    self._MockCommand([".", "."],
189                      ["Richards: 100\nDeltaBlue: 200\n",
190                       "Richards: 50\nDeltaBlue: 300\n"])
191    self.assertEquals(0, self._CallMain())
192    self._VerifyResults("v8", "ms", [
193      {"name": "Richards", "results": ["50", "100"], "stddev": ""},
194      {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
195    ])
196    self._VerifyErrors([])
197    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
198
199  def testTwoRuns_SubRegexp(self):
200    test_input = dict(V8_JSON)
201    test_input["run_count"] = 2
202    del test_input["results_regexp"]
203    test_input["tests"][0]["results_regexp"] = "^Richards: (.+)$"
204    test_input["tests"][1]["results_regexp"] = "^DeltaBlue: (.+)$"
205    self._WriteTestInput(test_input)
206    self._MockCommand([".", "."],
207                      ["Richards: 100\nDeltaBlue: 200\n",
208                       "Richards: 50\nDeltaBlue: 300\n"])
209    self.assertEquals(0, self._CallMain())
210    self._VerifyResults("test", "score", [
211      {"name": "Richards", "results": ["50", "100"], "stddev": ""},
212      {"name": "DeltaBlue", "results": ["300", "200"], "stddev": ""},
213    ])
214    self._VerifyErrors([])
215    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
216
217  def testNestedSuite(self):
218    self._WriteTestInput(V8_NESTED_SUITES_JSON)
219    self._MockCommand(["delta_blue", "sub/leaf", "richards"],
220                      ["DeltaBlue: 200\n",
221                       "Simple: 1 ms.\n",
222                       "Simple: 2 ms.\n",
223                       "Simple: 3 ms.\n",
224                       "Richards: 100\n",
225                       "Richards: 50\n"])
226    self.assertEquals(0, self._CallMain())
227    self.assertEquals([
228      {"units": "score",
229       "graphs": ["test", "Richards"],
230       "results": ["50", "100"],
231       "stddev": ""},
232      {"units": "ms",
233       "graphs": ["test", "Sub", "Leaf"],
234       "results": ["3", "2", "1"],
235       "stddev": ""},
236      {"units": "score",
237       "graphs": ["test", "DeltaBlue"],
238       "results": ["200"],
239       "stddev": ""},
240      ], self._LoadResults()["traces"])
241    self._VerifyErrors([])
242    self._VerifyMockMultiple(
243        (path.join("out", "x64.release", "d7"), "--flag", "file1.js",
244         "file2.js", "run.js"),
245        (path.join("out", "x64.release", "d7"), "--flag", "file1.js",
246         "file2.js", "run.js"),
247        (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
248        (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
249        (path.join("out", "x64.release", "d8"), "--flag", "run.js"),
250        (path.join("out", "x64.release", "d8"), "--flag", "--flag2", "run.js"))
251
252  def testOneRunStdDevRegExp(self):
253    test_input = dict(V8_JSON)
254    test_input["stddev_regexp"] = "^%s\-stddev: (.+)$"
255    self._WriteTestInput(test_input)
256    self._MockCommand(["."], ["Richards: 1.234\nRichards-stddev: 0.23\n"
257                              "DeltaBlue: 10657567\nDeltaBlue-stddev: 106\n"])
258    self.assertEquals(0, self._CallMain())
259    self._VerifyResults("test", "score", [
260      {"name": "Richards", "results": ["1.234"], "stddev": "0.23"},
261      {"name": "DeltaBlue", "results": ["10657567"], "stddev": "106"},
262    ])
263    self._VerifyErrors([])
264    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
265
266  def testTwoRunsStdDevRegExp(self):
267    test_input = dict(V8_JSON)
268    test_input["stddev_regexp"] = "^%s\-stddev: (.+)$"
269    test_input["run_count"] = 2
270    self._WriteTestInput(test_input)
271    self._MockCommand(["."], ["Richards: 3\nRichards-stddev: 0.7\n"
272                              "DeltaBlue: 6\nDeltaBlue-boom: 0.9\n",
273                              "Richards: 2\nRichards-stddev: 0.5\n"
274                              "DeltaBlue: 5\nDeltaBlue-stddev: 0.8\n"])
275    self.assertEquals(1, self._CallMain())
276    self._VerifyResults("test", "score", [
277      {"name": "Richards", "results": ["2", "3"], "stddev": "0.7"},
278      {"name": "DeltaBlue", "results": ["5", "6"], "stddev": "0.8"},
279    ])
280    self._VerifyErrors(
281        ["Test Richards should only run once since a stddev is provided "
282         "by the test.",
283         "Test DeltaBlue should only run once since a stddev is provided "
284         "by the test.",
285         "Regexp \"^DeltaBlue\-stddev: (.+)$\" didn't match for test "
286         "DeltaBlue."])
287    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
288
289  def testBuildbot(self):
290    self._WriteTestInput(V8_JSON)
291    self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"])
292    self.assertEquals(0, self._CallMain("--buildbot"))
293    self._VerifyResults("test", "score", [
294      {"name": "Richards", "results": ["1.234"], "stddev": ""},
295      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
296    ])
297    self._VerifyErrors([])
298    self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
299
300  def testBuildbotWithTotal(self):
301    test_input = dict(V8_JSON)
302    test_input["total"] = True
303    self._WriteTestInput(test_input)
304    self._MockCommand(["."], ["Richards: 1.234\nDeltaBlue: 10657567\n"])
305    self.assertEquals(0, self._CallMain("--buildbot"))
306    self._VerifyResults("test", "score", [
307      {"name": "Richards", "results": ["1.234"], "stddev": ""},
308      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
309      {"name": "Total", "results": ["3626.49109719"], "stddev": ""},
310    ])
311    self._VerifyErrors([])
312    self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
313
314  def testBuildbotWithTotalAndErrors(self):
315    test_input = dict(V8_JSON)
316    test_input["total"] = True
317    self._WriteTestInput(test_input)
318    self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"])
319    self.assertEquals(1, self._CallMain("--buildbot"))
320    self._VerifyResults("test", "score", [
321      {"name": "Richards", "results": [], "stddev": ""},
322      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
323    ])
324    self._VerifyErrors(
325        ["Regexp \"^Richards: (.+)$\" didn't match for test Richards.",
326         "Not all traces have the same number of results."])
327    self._VerifyMock(path.join("out", "Release", "d7"), "--flag", "run.js")
328
329  def testRegexpNoMatch(self):
330    self._WriteTestInput(V8_JSON)
331    self._MockCommand(["."], ["x\nRichaards: 1.234\nDeltaBlue: 10657567\ny\n"])
332    self.assertEquals(1, self._CallMain())
333    self._VerifyResults("test", "score", [
334      {"name": "Richards", "results": [], "stddev": ""},
335      {"name": "DeltaBlue", "results": ["10657567"], "stddev": ""},
336    ])
337    self._VerifyErrors(
338        ["Regexp \"^Richards: (.+)$\" didn't match for test Richards."])
339    self._VerifyMock(path.join("out", "x64.release", "d7"), "--flag", "run.js")
340
341  def testOneRunGeneric(self):
342    test_input = dict(V8_GENERIC_JSON)
343    self._WriteTestInput(test_input)
344    self._MockCommand(["."], [
345      "Trace(Test1), Result(1.234), StdDev(0.23)\n"
346      "Trace(Test2), Result(10657567), StdDev(106)\n"])
347    self.assertEquals(0, self._CallMain())
348    self._VerifyResults("test", "ms", [
349      {"name": "Test1", "results": ["1.234"], "stddev": "0.23"},
350      {"name": "Test2", "results": ["10657567"], "stddev": "106"},
351    ])
352    self._VerifyErrors([])
353    self._VerifyMock(path.join("out", "x64.release", "cc"), "--flag", "")
354
355  def testOneRunTimingOut(self):
356    test_input = dict(V8_JSON)
357    test_input["timeout"] = 70
358    self._WriteTestInput(test_input)
359    self._MockCommand(["."], [""], timed_out=True)
360    self.assertEquals(1, self._CallMain())
361    self._VerifyResults("test", "score", [
362      {"name": "Richards", "results": [], "stddev": ""},
363      {"name": "DeltaBlue", "results": [], "stddev": ""},
364    ])
365    self._VerifyErrors([
366      "Regexp \"^Richards: (.+)$\" didn't match for test Richards.",
367      "Regexp \"^DeltaBlue: (.+)$\" didn't match for test DeltaBlue.",
368    ])
369    self._VerifyMock(
370        path.join("out", "x64.release", "d7"), "--flag", "run.js", timeout=70)
371