1b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch# Copyright 2013 the V8 project authors. All rights reserved. 244f0eee88ff00398ff7f715fab053374d808c90dSteve Block# Redistribution and use in source and binary forms, with or without 344f0eee88ff00398ff7f715fab053374d808c90dSteve Block# modification, are permitted provided that the following conditions are 444f0eee88ff00398ff7f715fab053374d808c90dSteve Block# met: 544f0eee88ff00398ff7f715fab053374d808c90dSteve Block# 644f0eee88ff00398ff7f715fab053374d808c90dSteve Block# * Redistributions of source code must retain the above copyright 744f0eee88ff00398ff7f715fab053374d808c90dSteve Block# notice, this list of conditions and the following disclaimer. 844f0eee88ff00398ff7f715fab053374d808c90dSteve Block# * Redistributions in binary form must reproduce the above 944f0eee88ff00398ff7f715fab053374d808c90dSteve Block# copyright notice, this list of conditions and the following 1044f0eee88ff00398ff7f715fab053374d808c90dSteve Block# disclaimer in the documentation and/or other materials provided 1144f0eee88ff00398ff7f715fab053374d808c90dSteve Block# with the distribution. 1244f0eee88ff00398ff7f715fab053374d808c90dSteve Block# * Neither the name of Google Inc. nor the names of its 1344f0eee88ff00398ff7f715fab053374d808c90dSteve Block# contributors may be used to endorse or promote products derived 1444f0eee88ff00398ff7f715fab053374d808c90dSteve Block# from this software without specific prior written permission. 1544f0eee88ff00398ff7f715fab053374d808c90dSteve Block# 1644f0eee88ff00398ff7f715fab053374d808c90dSteve Block# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS 1744f0eee88ff00398ff7f715fab053374d808c90dSteve Block# "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT 1844f0eee88ff00398ff7f715fab053374d808c90dSteve Block# LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR 1944f0eee88ff00398ff7f715fab053374d808c90dSteve Block# A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT 2044f0eee88ff00398ff7f715fab053374d808c90dSteve Block# OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, 2144f0eee88ff00398ff7f715fab053374d808c90dSteve Block# SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT 2244f0eee88ff00398ff7f715fab053374d808c90dSteve Block# LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 2344f0eee88ff00398ff7f715fab053374d808c90dSteve Block# DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 2444f0eee88ff00398ff7f715fab053374d808c90dSteve Block# THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 2544f0eee88ff00398ff7f715fab053374d808c90dSteve Block# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 2644f0eee88ff00398ff7f715fab053374d808c90dSteve Block# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 2744f0eee88ff00398ff7f715fab053374d808c90dSteve Block 2844f0eee88ff00398ff7f715fab053374d808c90dSteve Block 2944f0eee88ff00398ff7f715fab053374d808c90dSteve Blockimport os 30b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport shutil 31b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport subprocess 32b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochimport tarfile 33b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 34b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom testrunner.local import testsuite 35b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochfrom testrunner.objects import testcase 36b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 37b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 38b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochclass BenchmarksTestSuite(testsuite.TestSuite): 39b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 40b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def __init__(self, name, root): 41b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch super(BenchmarksTestSuite, self).__init__(name, root) 42b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self.testroot = root 43b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 44b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def ListTests(self, context): 45b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch tests = [] 46b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch for test in [ 47b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/ai-astar", 48b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/audio-beat-detection", 49b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/audio-dft", 50b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/audio-fft", 51b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/audio-oscillator", 52b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/imaging-darkroom", 53b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/imaging-desaturate", 54b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/imaging-gaussian-blur", 55b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/json-parse-financial", 56b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/json-stringify-tinderbox", 57b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/stanford-crypto-aes", 58b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/stanford-crypto-ccm", 59b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/stanford-crypto-pbkdf2", 60b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken/stanford-crypto-sha256-iterative", 61b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 62b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/box2d", 63b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/code-load", 64b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/crypto", 65b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/deltablue", 66b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/earley-boyer", 67b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/gbemu-part1", 68b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/mandreel", 69b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/navier-stokes", 70b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/pdfjs", 71b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/raytrace", 72b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/regexp", 73b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/richards", 74b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/splay", 75b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/typescript", 76b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/zlib", 77b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 78b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/3d-cube", 79b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/3d-morph", 80b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/3d-raytrace", 81b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/access-binary-trees", 82b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/access-fannkuch", 83b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/access-nbody", 84b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/access-nsieve", 85b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/bitops-3bit-bits-in-byte", 86b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/bitops-bits-in-byte", 87b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/bitops-bitwise-and", 88b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/bitops-nsieve-bits", 89b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/controlflow-recursive", 90b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/crypto-aes", 91b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/crypto-md5", 92b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/crypto-sha1", 93b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/date-format-tofte", 94b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/date-format-xparb", 95b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/math-cordic", 96b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/math-partial-sums", 97b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/math-spectral-norm", 98b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/regexp-dna", 99b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/string-base64", 100b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/string-fasta", 101b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/string-tagcloud", 102b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/string-unpack-code", 103b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "sunspider/string-validate-input"]: 104b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch tests.append(testcase.TestCase(self, test)) 105b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return tests 106b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 107b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def GetFlagsForTestCase(self, testcase, context): 108b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result = [] 109b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result += context.mode_flags 110b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if testcase.path.startswith("kraken"): 111b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.append(os.path.join(self.testroot, "%s-data.js" % testcase.path)) 112b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.append(os.path.join(self.testroot, "%s.js" % testcase.path)) 113b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch elif testcase.path.startswith("octane"): 114b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.append(os.path.join(self.testroot, "octane/base.js")) 115b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.append(os.path.join(self.testroot, "%s.js" % testcase.path)) 116b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if testcase.path.startswith("octane/gbemu"): 117b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.append(os.path.join(self.testroot, "octane/gbemu-part2.js")) 118b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch elif testcase.path.startswith("octane/typescript"): 119b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.append(os.path.join(self.testroot, 120b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "octane/typescript-compiler.js")) 121b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.append(os.path.join(self.testroot, "octane/typescript-input.js")) 122b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch elif testcase.path.startswith("octane/zlib"): 123b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.append(os.path.join(self.testroot, "octane/zlib-data.js")) 124b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result += ["-e", "BenchmarkSuite.RunSuites({});"] 125b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch elif testcase.path.startswith("sunspider"): 126b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch result.append(os.path.join(self.testroot, "%s.js" % testcase.path)) 127b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return testcase.flags + result 128b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 129b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def GetSourceForTest(self, testcase): 130b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch filename = os.path.join(self.testroot, testcase.path + ".js") 131b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch with open(filename) as f: 132b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return f.read() 133b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 134b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def _DownloadIfNecessary(self, url, revision, target_dir): 135b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Maybe we're still up to date? 136b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch revision_file = "CHECKED_OUT_%s" % target_dir 137b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch checked_out_revision = None 138b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if os.path.exists(revision_file): 139b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch with open(revision_file) as f: 140b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch checked_out_revision = f.read() 141b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if checked_out_revision == revision: 142b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return 143b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 144b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # If we have a local archive file with the test data, extract it. 145b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if os.path.exists(target_dir): 146b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch shutil.rmtree(target_dir) 147b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch archive_file = "downloaded_%s_%s.tar.gz" % (target_dir, revision) 148b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if os.path.exists(archive_file): 149b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch with tarfile.open(archive_file, "r:gz") as tar: 150b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch tar.extractall() 151b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch with open(revision_file, "w") as f: 152b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch f.write(revision) 153b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return 154b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 155b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # No cached copy. Check out via SVN, and pack as .tar.gz for later use. 156b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch command = "svn co %s -r %s %s" % (url, revision, target_dir) 157b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch code = subprocess.call(command, shell=True) 158b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch if code != 0: 159b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch raise Exception("Error checking out %s benchmark" % target_dir) 160b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch with tarfile.open(archive_file, "w:gz") as tar: 161b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch tar.add("%s" % target_dir) 162b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch with open(revision_file, "w") as f: 163b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch f.write(revision) 164b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 165b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def DownloadData(self): 166b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch old_cwd = os.getcwd() 167b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch os.chdir(os.path.abspath(self.root)) 168b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 169b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self._DownloadIfNecessary( 170b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ("http://svn.webkit.org/repository/webkit/trunk/PerformanceTests/" 171b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "SunSpider/tests/sunspider-1.0.2/"), 172b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "159499", "sunspider") 173b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 174b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self._DownloadIfNecessary( 175b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch ("http://kraken-mirror.googlecode.com/svn/trunk/kraken/tests/" 176b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "kraken-1.1/"), 177b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "8", "kraken") 178b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 179b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch self._DownloadIfNecessary( 180b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "http://octane-benchmark.googlecode.com/svn/trunk/", 181b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch "26", "octane") 182b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 183b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch os.chdir(old_cwd) 184b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 185b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch def VariantFlags(self, testcase, default_flags): 186b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # Both --nocrankshaft and --stressopt are very slow. Add TF but without 187b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # always opt to match the way the benchmarks are run for performance 188b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch # testing. 189b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return [[], ["--turbo-filter=*"]] 190b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 191b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch 192b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdochdef GetSuite(name, root): 193b8a8cc1952d61a2f3a2568848933943a543b5d3eBen Murdoch return BenchmarksTestSuite(name, root) 194