1#!/usr/bin/perl -w
2
3# Copyright (C) 2007 Apple Inc. All rights reserved.
4# Copyright (C) 2007 Eric Seidel <eric@webkit.org>
5#
6# Redistribution and use in source and binary forms, with or without
7# modification, are permitted provided that the following conditions
8# are met:
9# 1. Redistributions of source code must retain the above copyright
10#    notice, this list of conditions and the following disclaimer.
11# 2. Redistributions in binary form must reproduce the above copyright
12#    notice, this list of conditions and the following disclaimer in the
13#    documentation and/or other materials provided with the distribution.
14#
15# THIS SOFTWARE IS PROVIDED BY APPLE COMPUTER, INC. ``AS IS'' AND ANY
16# EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
17# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
18# PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL APPLE COMPUTER, INC. OR
19# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
20# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
21# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
22# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
23# OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
24# (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
25# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 
26
27use strict;
28use Getopt::Long;
29use File::Basename;
30use File::Spec;
31use Cwd;
32use POSIX qw(strftime);
33use Time::HiRes qw(gettimeofday tv_interval);
34
35my $showHelp = 0;
36my $runInstruments = 0;
37my $ubench = 0;
38my $v8suite = 0;
39my $suite = "";
40my $parseOnly = 0;
41my $jsShellPath;
42my $jsShellArgs = "";
43my $setBaseline = 0;
44my $testsPattern;
45my $testRuns = 10;
46
47my $programName = basename($0);
48my $usage = <<EOF;
49Usage: $programName --shell=[path] [options]
50  --help            Show this help message
51  --set-baseline    Set baseline for future comparisons
52  --shell           Path to JavaScript shell
53  --args            Arguments to pass to JavaScript shell
54  --runs            Number of times to run tests (default: $testRuns)
55  --tests           Only run tests matching provided pattern
56  --instruments     Sample execution time with the Mac OS X "Instruments" tool (Time Profile) (implies --runs=1)
57  --suite           Select a specific benchmark suite. The default is sunspider-1.0
58  --ubench          Use microbenchmark suite instead of regular tests. Same as --suite=ubench
59  --v8-suite        Use the V8 benchmark suite. Same as --suite=v8-v4
60  --parse-only      Use the parse-only benchmark suite. Same as --suite=parse-only
61EOF
62
63GetOptions('runs=i' => \$testRuns,
64           'set-baseline' => \$setBaseline,
65           'shell=s' => \$jsShellPath,
66           'args=s' => \$jsShellArgs,
67           'instruments' => \$runInstruments,
68           'suite=s' => \$suite,
69           'ubench' => \$ubench,
70           'v8-suite' => \$v8suite,
71           'parse-only' => \$parseOnly,
72           'tests=s' => \$testsPattern,
73           'help' => \$showHelp);
74
75
76$suite = "ubench" if ($ubench);
77$suite = "v8-v4" if ($v8suite);
78$suite = "parse-only" if ($parseOnly);
79$suite = "sunspider-1.0" if (!$suite);
80
81my $resultDirectory = "${suite}-results";
82
83my $suitePath = $suite;
84$suitePath = "tests/" . $suitePath unless ($suite =~ /\//);
85
86$testRuns = 1 if $runInstruments;
87
88if (!$jsShellPath || $showHelp) {
89   print STDERR $usage;
90   exit 1;
91}
92
93sub dumpToFile($$)
94{
95    my ($contents, $path) = @_;
96    open FILE, ">", $path or die "Failed to open $path";
97    print FILE $contents;
98    close FILE;
99}
100
101my @tests = ();
102my @categories = ();
103my %uniqueCategories = ();
104
105sub loadTestsList()
106{
107    open TESTLIST, "<", "${suitePath}/LIST" or die "Can't find ${suitePath}/LIST";
108    while (<TESTLIST>) {
109        chomp;
110        next unless !$testsPattern || /$testsPattern/;
111        
112        push @tests, $_;
113        my $category = $_;
114        $category =~ s/-.*//;
115        if (!$uniqueCategories{$category}) {
116            push @categories, $category;
117            $uniqueCategories{$category} = $category;
118        }
119    }
120    close TESTLIST;
121}
122
123my $timeString = strftime "%Y-%m-%d-%H.%M.%S", localtime $^T;
124my $prefixFile = "$resultDirectory/sunspider-test-prefix.js";
125my $resultsFile = "$resultDirectory/sunspider-results-$timeString.js";
126
127sub writePrefixFile()
128{
129    my $prefix = "var suitePath = " . '"' . $suitePath . '"' . ";\n";
130    $prefix .= "var tests = [ " . join(", ", map { '"' . $_ . '"' } @tests) . " ];\n";
131    $prefix .= "var categories = [ " . join(", ", map { '"' . $_ . '"' } @categories) . " ];\n";
132
133    mkdir "$resultDirectory";
134    dumpToFile($prefix, $prefixFile);
135}
136
137sub runTestsOnce($)
138{
139    my ($useInstruments) = @_;
140    my $shellArgs = $jsShellArgs . " -f $prefixFile -f resources/sunspider-standalone-driver.js 2> " . File::Spec->devnull();
141    my $output;
142    if ($useInstruments) {
143        $output = `instruments -t "resources/TimeProfile20us.tracetemplate" "$jsShellPath" $shellArgs`;
144    } else {
145        $output = `"$jsShellPath" $shellArgs | grep -v break`;
146    }
147    return $output;
148}
149
150sub newestFile($$)
151{
152    my ($dir, $pattern) = @_;
153
154    my $newestAge;
155    my $newestFile = "";
156    opendir DIR, $dir or die;
157    for my $file (readdir DIR) {
158        if ($file =~ $pattern) {
159            my $age = -M "$dir/$file";
160            if (!defined $newestAge || $age < $newestAge) {
161                $newestFile = $file;
162                $newestAge = $age;
163            }
164        }
165    }
166    closedir DIR;
167
168    return "$dir/$newestFile";
169}
170
171loadTestsList();
172if ($testsPattern) {
173    print STDERR "Found " . scalar(@tests) . " tests matching '" . $testsPattern . "'\n";
174} else {
175    print STDERR "Found " . scalar(@tests) . " tests\n";
176}
177die "No tests to run"  unless scalar(@tests);
178print STDERR "Running SunSpider once for warmup, then " . ($runInstruments ? "under Instruments" : "$testRuns time" . ($testRuns == 1 ? "" : "s")) . "\n";
179writePrefixFile();
180
181runTestsOnce(0);
182print "Discarded first run.\n";
183
184my $result;
185my $count = 0;
186my @results = ();
187my $total = 0;
188print "[";
189while ($count++ < $testRuns) {
190    $result = runTestsOnce($runInstruments);
191    $result =~ s/\r\n/\n/g;
192    chomp $result;
193    push @results, $result;
194    print $result;
195    print ",\n" unless ($count == $testRuns);
196}
197print "]\n";
198
199my $output = "var output = [\n" . join(",\n", @results) . "\n];\n";
200dumpToFile($output, $resultsFile);
201dumpToFile(File::Spec->rel2abs($resultsFile), "$resultDirectory/baseline-filename.txt") if $setBaseline;
202
203system("$jsShellPath", "-f", $prefixFile, "-f", $resultsFile, "-f", "resources/sunspider-analyze-results.js");
204
205print("\nResults are located at $resultsFile\n");
206
207if ($runInstruments) {
208    my $newestTrace = newestFile(".", qr/\.trace$/);
209    if ($newestTrace) {
210        my $profileFile = "$resultDirectory/sunspider-profile-$timeString.trace";
211        rename $newestTrace, $profileFile or die;
212        exec "/usr/bin/open", $profileFile;
213    }
214}
215