1# Copyright 2013 the V8 project authors. All rights reserved.
2# Redistribution and use in source and binary forms, with or without
3# modification, are permitted provided that the following conditions are
4# met:
5#
6#     * Redistributions of source code must retain the above copyright
7#       notice, this list of conditions and the following disclaimer.
8#     * Redistributions in binary form must reproduce the above
9#       copyright notice, this list of conditions and the following
10#       disclaimer in the documentation and/or other materials provided
11#       with the distribution.
12#     * Neither the name of Google Inc. nor the names of its
13#       contributors may be used to endorse or promote products derived
14#       from this software without specific prior written permission.
15#
16# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
17# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
18# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
19# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
20# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
21# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
22# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
23# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
24# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
25# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
26# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
27
28
29import os
30import shutil
31import subprocess
32import tarfile
33
34from testrunner.local import testsuite
35from testrunner.objects import testcase
36
37
38class BenchmarksTestSuite(testsuite.TestSuite):
39
40  def __init__(self, name, root):
41    super(BenchmarksTestSuite, self).__init__(name, root)
42    self.testroot = root
43
44  def ListTests(self, context):
45    tests = []
46    for test in [
47        "kraken/ai-astar",
48        "kraken/audio-beat-detection",
49        "kraken/audio-dft",
50        "kraken/audio-fft",
51        "kraken/audio-oscillator",
52        "kraken/imaging-darkroom",
53        "kraken/imaging-desaturate",
54        "kraken/imaging-gaussian-blur",
55        "kraken/json-parse-financial",
56        "kraken/json-stringify-tinderbox",
57        "kraken/stanford-crypto-aes",
58        "kraken/stanford-crypto-ccm",
59        "kraken/stanford-crypto-pbkdf2",
60        "kraken/stanford-crypto-sha256-iterative",
61
62        "octane/box2d",
63        "octane/code-load",
64        "octane/crypto",
65        "octane/deltablue",
66        "octane/earley-boyer",
67        "octane/gbemu-part1",
68        "octane/mandreel",
69        "octane/navier-stokes",
70        "octane/pdfjs",
71        "octane/raytrace",
72        "octane/regexp",
73        "octane/richards",
74        "octane/splay",
75        "octane/typescript",
76        "octane/zlib",
77
78        "sunspider/3d-cube",
79        "sunspider/3d-morph",
80        "sunspider/3d-raytrace",
81        "sunspider/access-binary-trees",
82        "sunspider/access-fannkuch",
83        "sunspider/access-nbody",
84        "sunspider/access-nsieve",
85        "sunspider/bitops-3bit-bits-in-byte",
86        "sunspider/bitops-bits-in-byte",
87        "sunspider/bitops-bitwise-and",
88        "sunspider/bitops-nsieve-bits",
89        "sunspider/controlflow-recursive",
90        "sunspider/crypto-aes",
91        "sunspider/crypto-md5",
92        "sunspider/crypto-sha1",
93        "sunspider/date-format-tofte",
94        "sunspider/date-format-xparb",
95        "sunspider/math-cordic",
96        "sunspider/math-partial-sums",
97        "sunspider/math-spectral-norm",
98        "sunspider/regexp-dna",
99        "sunspider/string-base64",
100        "sunspider/string-fasta",
101        "sunspider/string-tagcloud",
102        "sunspider/string-unpack-code",
103        "sunspider/string-validate-input"]:
104      tests.append(testcase.TestCase(self, test))
105    return tests
106
107  def GetFlagsForTestCase(self, testcase, context):
108    result = []
109    result += context.mode_flags
110    if testcase.path.startswith("kraken"):
111      result.append(os.path.join(self.testroot, "%s-data.js" % testcase.path))
112      result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
113    elif testcase.path.startswith("octane"):
114      result.append(os.path.join(self.testroot, "octane/base.js"))
115      result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
116      if testcase.path.startswith("octane/gbemu"):
117        result.append(os.path.join(self.testroot, "octane/gbemu-part2.js"))
118      elif testcase.path.startswith("octane/typescript"):
119        result.append(os.path.join(self.testroot,
120                                   "octane/typescript-compiler.js"))
121        result.append(os.path.join(self.testroot, "octane/typescript-input.js"))
122      elif testcase.path.startswith("octane/zlib"):
123        result.append(os.path.join(self.testroot, "octane/zlib-data.js"))
124      result += ["-e", "BenchmarkSuite.RunSuites({});"]
125    elif testcase.path.startswith("sunspider"):
126      result.append(os.path.join(self.testroot, "%s.js" % testcase.path))
127    return testcase.flags + result
128
129  def GetSourceForTest(self, testcase):
130    filename = os.path.join(self.testroot, testcase.path + ".js")
131    with open(filename) as f:
132      return f.read()
133
134  def _DownloadIfNecessary(self, url, revision, target_dir):
135    # Maybe we're still up to date?
136    revision_file = "CHECKED_OUT_%s" % target_dir
137    checked_out_revision = None
138    if os.path.exists(revision_file):
139      with open(revision_file) as f:
140        checked_out_revision = f.read()
141    if checked_out_revision == revision:
142      return
143
144    # If we have a local archive file with the test data, extract it.
145    if os.path.exists(target_dir):
146      shutil.rmtree(target_dir)
147    archive_file = "downloaded_%s_%s.tar.gz" % (target_dir, revision)
148    if os.path.exists(archive_file):
149      with tarfile.open(archive_file, "r:gz") as tar:
150        tar.extractall()
151      with open(revision_file, "w") as f:
152        f.write(revision)
153      return
154
155    # No cached copy. Check out via SVN, and pack as .tar.gz for later use.
156    command = "svn co %s -r %s %s" % (url, revision, target_dir)
157    code = subprocess.call(command, shell=True)
158    if code != 0:
159      raise Exception("Error checking out %s benchmark" % target_dir)
160    with tarfile.open(archive_file, "w:gz") as tar:
161      tar.add("%s" % target_dir)
162    with open(revision_file, "w") as f:
163      f.write(revision)
164
165  def DownloadData(self):
166    old_cwd = os.getcwd()
167    os.chdir(os.path.abspath(self.root))
168
169    self._DownloadIfNecessary(
170        ("http://svn.webkit.org/repository/webkit/trunk/PerformanceTests/"
171         "SunSpider/tests/sunspider-1.0.2/"),
172        "159499", "sunspider")
173
174    self._DownloadIfNecessary(
175        ("http://kraken-mirror.googlecode.com/svn/trunk/kraken/tests/"
176         "kraken-1.1/"),
177        "8", "kraken")
178
179    self._DownloadIfNecessary(
180        "http://octane-benchmark.googlecode.com/svn/trunk/",
181        "26", "octane")
182
183    os.chdir(old_cwd)
184
185  def VariantFlags(self, testcase, default_flags):
186    # Both --nocrankshaft and --stressopt are very slow. Add TF but without
187    # always opt to match the way the benchmarks are run for performance
188    # testing.
189    return [[], ["--turbo-filter=*"]]
190
191
192def GetSuite(name, root):
193  return BenchmarksTestSuite(name, root)
194