1#!/usr/bin/perl -w 2 3# Copyright (C) 2007 Apple Inc. All rights reserved. 4# Copyright (C) 2007 Eric Seidel <eric@webkit.org> 5# 6# Redistribution and use in source and binary forms, with or without 7# modification, are permitted provided that the following conditions 8# are met: 9# 1. Redistributions of source code must retain the above copyright 10# notice, this list of conditions and the following disclaimer. 11# 2. Redistributions in binary form must reproduce the above copyright 12# notice, this list of conditions and the following disclaimer in the 13# documentation and/or other materials provided with the distribution. 14# 15# THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY 16# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 17# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 18# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE COMPUTER, INC. OR 19# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 20# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 21# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR 22# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY 23# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE 25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 26 27use strict; 28use Getopt::Long; 29use File::Basename; 30use File::Spec; 31use Cwd; 32use POSIX qw(strftime); 33use Time::HiRes qw(gettimeofday tv_interval); 34 35my $showHelp = 0; 36my $runInstruments = 0; 37my $ubench = 0; 38my $v8suite = 0; 39my $suite = ""; 40my $parseOnly = 0; 41my $jsShellPath; 42my $jsShellArgs = ""; 43my $setBaseline = 0; 44my $testsPattern; 45my $testRuns = 10; 46 47my $programName = basename($0); 48my $usage = <<EOF; 49Usage: $programName --shell=[path] [options] 50 --help Show this help message 51 --set-baseline Set baseline for future comparisons 52 --shell Path to JavaScript shell 53 --args Arguments to pass to JavaScript shell 54 --runs Number of times to run tests (default: $testRuns) 55 --tests Only run tests matching provided pattern 56 --instruments Sample execution time with the Mac OS X "Instruments" tool (Time Profile) (implies --runs=1) 57 --suite Select a specific benchmark suite. The default is sunspider-1.0 58 --ubench Use microbenchmark suite instead of regular tests. Same as --suite=ubench 59 --v8-suite Use the V8 benchmark suite. Same as --suite=v8-v4 60 --parse-only Use the parse-only benchmark suite. Same as --suite=parse-only 61EOF 62 63GetOptions('runs=i' => \$testRuns, 64 'set-baseline' => \$setBaseline, 65 'shell=s' => \$jsShellPath, 66 'args=s' => \$jsShellArgs, 67 'instruments' => \$runInstruments, 68 'suite=s' => \$suite, 69 'ubench' => \$ubench, 70 'v8-suite' => \$v8suite, 71 'parse-only' => \$parseOnly, 72 'tests=s' => \$testsPattern, 73 'help' => \$showHelp); 74 75 76$suite = "ubench" if ($ubench); 77$suite = "v8-v4" if ($v8suite); 78$suite = "parse-only" if ($parseOnly); 79$suite = "sunspider-1.0" if (!$suite); 80 81my $resultDirectory = "${suite}-results"; 82 83my $suitePath = $suite; 84$suitePath = "tests/" . $suitePath unless ($suite =~ /\//); 85 86$testRuns = 1 if $runInstruments; 87 88if (!$jsShellPath || $showHelp) { 89 print STDERR $usage; 90 exit 1; 91} 92 93sub dumpToFile($$) 94{ 95 my ($contents, $path) = @_; 96 open FILE, ">", $path or die "Failed to open $path"; 97 print FILE $contents; 98 close FILE; 99} 100 101my @tests = (); 102my @categories = (); 103my %uniqueCategories = (); 104 105sub loadTestsList() 106{ 107 open TESTLIST, "<", "${suitePath}/LIST" or die "Can't find ${suitePath}/LIST"; 108 while (<TESTLIST>) { 109 chomp; 110 next unless !$testsPattern || /$testsPattern/; 111 112 push @tests, $_; 113 my $category = $_; 114 $category =~ s/-.*//; 115 if (!$uniqueCategories{$category}) { 116 push @categories, $category; 117 $uniqueCategories{$category} = $category; 118 } 119 } 120 close TESTLIST; 121} 122 123my $timeString = strftime "%Y-%m-%d-%H.%M.%S", localtime $^T; 124my $prefixFile = "$resultDirectory/sunspider-test-prefix.js"; 125my $resultsFile = "$resultDirectory/sunspider-results-$timeString.js"; 126 127sub writePrefixFile() 128{ 129 my $prefix = "var suitePath = " . '"' . $suitePath . '"' . ";\n"; 130 $prefix .= "var tests = [ " . join(", ", map { '"' . $_ . '"' } @tests) . " ];\n"; 131 $prefix .= "var categories = [ " . join(", ", map { '"' . $_ . '"' } @categories) . " ];\n"; 132 133 mkdir "$resultDirectory"; 134 dumpToFile($prefix, $prefixFile); 135} 136 137sub runTestsOnce($) 138{ 139 my ($useInstruments) = @_; 140 my $shellArgs = $jsShellArgs . " -f $prefixFile -f resources/sunspider-standalone-driver.js 2> " . File::Spec->devnull(); 141 my $output; 142 if ($useInstruments) { 143 $output = `instruments -t "resources/TimeProfile20us.tracetemplate" "$jsShellPath" $shellArgs`; 144 } else { 145 $output = `"$jsShellPath" $shellArgs | grep -v break`; 146 } 147 return $output; 148} 149 150sub newestFile($$) 151{ 152 my ($dir, $pattern) = @_; 153 154 my $newestAge; 155 my $newestFile = ""; 156 opendir DIR, $dir or die; 157 for my $file (readdir DIR) { 158 if ($file =~ $pattern) { 159 my $age = -M "$dir/$file"; 160 if (!defined $newestAge || $age < $newestAge) { 161 $newestFile = $file; 162 $newestAge = $age; 163 } 164 } 165 } 166 closedir DIR; 167 168 return "$dir/$newestFile"; 169} 170 171loadTestsList(); 172if ($testsPattern) { 173 print STDERR "Found " . scalar(@tests) . " tests matching '" . $testsPattern . "'\n"; 174} else { 175 print STDERR "Found " . scalar(@tests) . " tests\n"; 176} 177die "No tests to run" unless scalar(@tests); 178print STDERR "Running SunSpider once for warmup, then " . ($runInstruments ? "under Instruments" : "$testRuns time" . ($testRuns == 1 ? "" : "s")) . "\n"; 179writePrefixFile(); 180 181runTestsOnce(0); 182print "Discarded first run.\n"; 183 184my $result; 185my $count = 0; 186my @results = (); 187my $total = 0; 188print "["; 189while ($count++ < $testRuns) { 190 $result = runTestsOnce($runInstruments); 191 $result =~ s/\r\n/\n/g; 192 chomp $result; 193 push @results, $result; 194 print $result; 195 print ",\n" unless ($count == $testRuns); 196} 197print "]\n"; 198 199my $output = "var output = [\n" . join(",\n", @results) . "\n];\n"; 200dumpToFile($output, $resultsFile); 201dumpToFile(File::Spec->rel2abs($resultsFile), "$resultDirectory/baseline-filename.txt") if $setBaseline; 202 203system("$jsShellPath", "-f", $prefixFile, "-f", $resultsFile, "-f", "resources/sunspider-analyze-results.js"); 204 205print("\nResults are located at $resultsFile\n"); 206 207if ($runInstruments) { 208 my $newestTrace = newestFile(".", qr/\.trace$/); 209 if ($newestTrace) { 210 my $profileFile = "$resultDirectory/sunspider-profile-$timeString.trace"; 211 rename $newestTrace, $profileFile or die; 212 exec "/usr/bin/open", $profileFile; 213 } 214} 215