1#!/usr/bin/env ruby
2
3# Copyright (C) 2011 Apple Inc. All rights reserved.
4#
5# Redistribution and use in source and binary forms, with or without
6# modification, are permitted provided that the following conditions
7# are met:
8# 1. Redistributions of source code must retain the above copyright
9#    notice, this list of conditions and the following disclaimer.
10# 2. Redistributions in binary form must reproduce the above copyright
11#    notice, this list of conditions and the following disclaimer in the
12#    documentation and/or other materials provided with the distribution.
13#
14# THIS SOFTWARE IS PROVIDED BY APPLE INC. AND ITS CONTRIBUTORS ``AS IS''
15# AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO,
16# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
17# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL APPLE INC. OR ITS CONTRIBUTORS
18# BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
19# CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
20# SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
21# INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
22# CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
23# ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF
24# THE POSSIBILITY OF SUCH DAMAGE.
25
26require 'rubygems'
27
28require 'getoptlong'
29require 'pathname'
30require 'tempfile'
31require 'socket'
32
33begin
34  require 'json'
35rescue LoadError => e
36  $stderr.puts "It does not appear that you have the 'json' package installed.  Try running 'sudo gem install json'."
37  exit 1
38end
39
40# Configuration
41
42CONFIGURATION_FLNM = ENV["HOME"]+"/.bencher"
43
44unless FileTest.exist? CONFIGURATION_FLNM
45  $stderr.puts "Error: no configuration file at ~/.bencher."
46  $stderr.puts "This file should contain paths to SunSpider, V8, and Kraken, as well as a"
47  $stderr.puts "temporary directory that bencher can use for its remote mode. It should be"
48  $stderr.puts "formatted in JSON.  For example:"
49  $stderr.puts "{"
50  $stderr.puts "    \"sunSpiderPath\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/sunspider-1.0\","
51  $stderr.puts "    \"v8Path\": \"/Volumes/Data/pizlo/OpenSource/PerformanceTests/SunSpider/tests/v8-v6\","
52  $stderr.puts "    \"krakenPath\": \"/Volumes/Data/pizlo/kraken/kraken-e119421cb325/tests/kraken-1.1\","
53  $stderr.puts "    \"tempPath\": \"/Volumes/Data/pizlo/bencher/temp\""
54  $stderr.puts "}"
55  exit 1
56end
57
58CONFIGURATION = JSON.parse(File::read(CONFIGURATION_FLNM))
59
60SUNSPIDER_PATH = CONFIGURATION["sunSpiderPath"]
61V8_PATH = CONFIGURATION["v8Path"]
62KRAKEN_PATH = CONFIGURATION["krakenPath"]
63TEMP_PATH = CONFIGURATION["tempPath"]
64BENCH_DATA_PATH = TEMP_PATH + "/benchdata"
65
66IBR_LOOKUP=[0.00615583, 0.0975, 0.22852, 0.341628, 0.430741, 0.500526, 0.555933,
67            0.600706, 0.637513, 0.668244, 0.694254, 0.716537, 0.735827, 0.752684,
68            0.767535, 0.780716, 0.792492, 0.803074, 0.812634, 0.821313, 0.829227,
69            0.836472, 0.843129, 0.849267, 0.854943, 0.860209, 0.865107, 0.869674,
70            0.873942, 0.877941, 0.881693, 0.885223, 0.888548, 0.891686, 0.894652,
71            0.897461, 0.900124, 0.902652, 0.905056, 0.907343, 0.909524, 0.911604,
72            0.91359, 0.91549, 0.917308, 0.919049, 0.920718, 0.92232, 0.923859, 0.925338,
73            0.926761, 0.92813, 0.929449, 0.930721, 0.931948, 0.933132, 0.934275, 0.93538,
74            0.936449, 0.937483, 0.938483, 0.939452, 0.940392, 0.941302, 0.942185,
75            0.943042, 0.943874, 0.944682, 0.945467, 0.94623, 0.946972, 0.947694,
76            0.948396, 0.94908, 0.949746, 0.950395, 0.951027, 0.951643, 0.952244,
77            0.952831, 0.953403, 0.953961, 0.954506, 0.955039, 0.955559, 0.956067,
78            0.956563, 0.957049, 0.957524, 0.957988, 0.958443, 0.958887, 0.959323,
79            0.959749, 0.960166, 0.960575, 0.960975, 0.961368, 0.961752, 0.962129,
80            0.962499, 0.962861, 0.963217, 0.963566, 0.963908, 0.964244, 0.964574,
81            0.964897, 0.965215, 0.965527, 0.965834, 0.966135, 0.966431, 0.966722,
82            0.967007, 0.967288, 0.967564, 0.967836, 0.968103, 0.968366, 0.968624,
83            0.968878, 0.969128, 0.969374, 0.969617, 0.969855, 0.97009, 0.970321,
84            0.970548, 0.970772, 0.970993, 0.97121, 0.971425, 0.971636, 0.971843,
85            0.972048, 0.97225, 0.972449, 0.972645, 0.972839, 0.973029, 0.973217,
86            0.973403, 0.973586, 0.973766, 0.973944, 0.97412, 0.974293, 0.974464,
87            0.974632, 0.974799, 0.974963, 0.975125, 0.975285, 0.975443, 0.975599,
88            0.975753, 0.975905, 0.976055, 0.976204, 0.97635, 0.976495, 0.976638,
89            0.976779, 0.976918, 0.977056, 0.977193, 0.977327, 0.97746, 0.977592,
90            0.977722, 0.97785, 0.977977, 0.978103, 0.978227, 0.978349, 0.978471,
91            0.978591, 0.978709, 0.978827, 0.978943, 0.979058, 0.979171, 0.979283,
92            0.979395, 0.979504, 0.979613, 0.979721, 0.979827, 0.979933, 0.980037,
93            0.98014, 0.980242, 0.980343, 0.980443, 0.980543, 0.980641, 0.980738,
94            0.980834, 0.980929, 0.981023, 0.981116, 0.981209, 0.9813, 0.981391, 0.981481,
95            0.981569, 0.981657, 0.981745, 0.981831, 0.981916, 0.982001, 0.982085,
96            0.982168, 0.982251, 0.982332, 0.982413, 0.982493, 0.982573, 0.982651,
97            0.982729, 0.982807, 0.982883, 0.982959, 0.983034, 0.983109, 0.983183,
98            0.983256, 0.983329, 0.983401, 0.983472, 0.983543, 0.983613, 0.983683,
99            0.983752, 0.98382, 0.983888, 0.983956, 0.984022, 0.984089, 0.984154,
100            0.984219, 0.984284, 0.984348, 0.984411, 0.984474, 0.984537, 0.984599,
101            0.98466, 0.984721, 0.984782, 0.984842, 0.984902, 0.984961, 0.985019,
102            0.985077, 0.985135, 0.985193, 0.985249, 0.985306, 0.985362, 0.985417,
103            0.985472, 0.985527, 0.985582, 0.985635, 0.985689, 0.985742, 0.985795,
104            0.985847, 0.985899, 0.985951, 0.986002, 0.986053, 0.986103, 0.986153,
105            0.986203, 0.986252, 0.986301, 0.98635, 0.986398, 0.986446, 0.986494,
106            0.986541, 0.986588, 0.986635, 0.986681, 0.986727, 0.986773, 0.986818,
107            0.986863, 0.986908, 0.986953, 0.986997, 0.987041, 0.987084, 0.987128,
108            0.987171, 0.987213, 0.987256, 0.987298, 0.98734, 0.987381, 0.987423,
109            0.987464, 0.987504, 0.987545, 0.987585, 0.987625, 0.987665, 0.987704,
110            0.987744, 0.987783, 0.987821, 0.98786, 0.987898, 0.987936, 0.987974,
111            0.988011, 0.988049, 0.988086, 0.988123, 0.988159, 0.988196, 0.988232,
112            0.988268, 0.988303, 0.988339, 0.988374, 0.988409, 0.988444, 0.988479,
113            0.988513, 0.988547, 0.988582, 0.988615, 0.988649, 0.988682, 0.988716,
114            0.988749, 0.988782, 0.988814, 0.988847, 0.988879, 0.988911, 0.988943,
115            0.988975, 0.989006, 0.989038, 0.989069, 0.9891, 0.989131, 0.989161, 0.989192,
116            0.989222, 0.989252, 0.989282, 0.989312, 0.989342, 0.989371, 0.989401,
117            0.98943, 0.989459, 0.989488, 0.989516, 0.989545, 0.989573, 0.989602, 0.98963,
118            0.989658, 0.989685, 0.989713, 0.98974, 0.989768, 0.989795, 0.989822,
119            0.989849, 0.989876, 0.989902, 0.989929, 0.989955, 0.989981, 0.990007,
120            0.990033, 0.990059, 0.990085, 0.99011, 0.990136, 0.990161, 0.990186,
121            0.990211, 0.990236, 0.990261, 0.990285, 0.99031, 0.990334, 0.990358,
122            0.990383, 0.990407, 0.99043, 0.990454, 0.990478, 0.990501, 0.990525,
123            0.990548, 0.990571, 0.990594, 0.990617, 0.99064, 0.990663, 0.990686,
124            0.990708, 0.990731, 0.990753, 0.990775, 0.990797, 0.990819, 0.990841,
125            0.990863, 0.990885, 0.990906, 0.990928, 0.990949, 0.99097, 0.990991,
126            0.991013, 0.991034, 0.991054, 0.991075, 0.991096, 0.991116, 0.991137,
127            0.991157, 0.991178, 0.991198, 0.991218, 0.991238, 0.991258, 0.991278,
128            0.991298, 0.991317, 0.991337, 0.991356, 0.991376, 0.991395, 0.991414,
129            0.991433, 0.991452, 0.991471, 0.99149, 0.991509, 0.991528, 0.991547,
130            0.991565, 0.991584, 0.991602, 0.99162, 0.991639, 0.991657, 0.991675,
131            0.991693, 0.991711, 0.991729, 0.991746, 0.991764, 0.991782, 0.991799,
132            0.991817, 0.991834, 0.991851, 0.991869, 0.991886, 0.991903, 0.99192,
133            0.991937, 0.991954, 0.991971, 0.991987, 0.992004, 0.992021, 0.992037,
134            0.992054, 0.99207, 0.992086, 0.992103, 0.992119, 0.992135, 0.992151,
135            0.992167, 0.992183, 0.992199, 0.992215, 0.99223, 0.992246, 0.992262,
136            0.992277, 0.992293, 0.992308, 0.992324, 0.992339, 0.992354, 0.992369,
137            0.992384, 0.9924, 0.992415, 0.992429, 0.992444, 0.992459, 0.992474, 0.992489,
138            0.992503, 0.992518, 0.992533, 0.992547, 0.992561, 0.992576, 0.99259,
139            0.992604, 0.992619, 0.992633, 0.992647, 0.992661, 0.992675, 0.992689,
140            0.992703, 0.992717, 0.99273, 0.992744, 0.992758, 0.992771, 0.992785,
141            0.992798, 0.992812, 0.992825, 0.992839, 0.992852, 0.992865, 0.992879,
142            0.992892, 0.992905, 0.992918, 0.992931, 0.992944, 0.992957, 0.99297,
143            0.992983, 0.992995, 0.993008, 0.993021, 0.993034, 0.993046, 0.993059,
144            0.993071, 0.993084, 0.993096, 0.993109, 0.993121, 0.993133, 0.993145,
145            0.993158, 0.99317, 0.993182, 0.993194, 0.993206, 0.993218, 0.99323, 0.993242,
146            0.993254, 0.993266, 0.993277, 0.993289, 0.993301, 0.993312, 0.993324,
147            0.993336, 0.993347, 0.993359, 0.99337, 0.993382, 0.993393, 0.993404,
148            0.993416, 0.993427, 0.993438, 0.993449, 0.99346, 0.993472, 0.993483,
149            0.993494, 0.993505, 0.993516, 0.993527, 0.993538, 0.993548, 0.993559,
150            0.99357, 0.993581, 0.993591, 0.993602, 0.993613, 0.993623, 0.993634,
151            0.993644, 0.993655, 0.993665, 0.993676, 0.993686, 0.993697, 0.993707,
152            0.993717, 0.993727, 0.993738, 0.993748, 0.993758, 0.993768, 0.993778,
153            0.993788, 0.993798, 0.993808, 0.993818, 0.993828, 0.993838, 0.993848,
154            0.993858, 0.993868, 0.993877, 0.993887, 0.993897, 0.993907, 0.993916,
155            0.993926, 0.993935, 0.993945, 0.993954, 0.993964, 0.993973, 0.993983,
156            0.993992, 0.994002, 0.994011, 0.99402, 0.99403, 0.994039, 0.994048, 0.994057,
157            0.994067, 0.994076, 0.994085, 0.994094, 0.994103, 0.994112, 0.994121,
158            0.99413, 0.994139, 0.994148, 0.994157, 0.994166, 0.994175, 0.994183,
159            0.994192, 0.994201, 0.99421, 0.994218, 0.994227, 0.994236, 0.994244,
160            0.994253, 0.994262, 0.99427, 0.994279, 0.994287, 0.994296, 0.994304,
161            0.994313, 0.994321, 0.994329, 0.994338, 0.994346, 0.994354, 0.994363,
162            0.994371, 0.994379, 0.994387, 0.994395, 0.994404, 0.994412, 0.99442,
163            0.994428, 0.994436, 0.994444, 0.994452, 0.99446, 0.994468, 0.994476,
164            0.994484, 0.994492, 0.9945, 0.994508, 0.994516, 0.994523, 0.994531, 0.994539,
165            0.994547, 0.994554, 0.994562, 0.99457, 0.994577, 0.994585, 0.994593, 0.9946,
166            0.994608, 0.994615, 0.994623, 0.994631, 0.994638, 0.994645, 0.994653,
167            0.99466, 0.994668, 0.994675, 0.994683, 0.99469, 0.994697, 0.994705, 0.994712,
168            0.994719, 0.994726, 0.994734, 0.994741, 0.994748, 0.994755, 0.994762,
169            0.994769, 0.994777, 0.994784, 0.994791, 0.994798, 0.994805, 0.994812,
170            0.994819, 0.994826, 0.994833, 0.99484, 0.994847, 0.994854, 0.99486, 0.994867,
171            0.994874, 0.994881, 0.994888, 0.994895, 0.994901, 0.994908, 0.994915,
172            0.994922, 0.994928, 0.994935, 0.994942, 0.994948, 0.994955, 0.994962,
173            0.994968, 0.994975, 0.994981, 0.994988, 0.994994, 0.995001, 0.995007,
174            0.995014, 0.99502, 0.995027, 0.995033, 0.99504, 0.995046, 0.995052, 0.995059,
175            0.995065, 0.995071, 0.995078, 0.995084, 0.99509, 0.995097, 0.995103,
176            0.995109, 0.995115, 0.995121, 0.995128, 0.995134, 0.99514, 0.995146,
177            0.995152, 0.995158, 0.995164, 0.995171, 0.995177, 0.995183, 0.995189,
178            0.995195, 0.995201, 0.995207, 0.995213, 0.995219, 0.995225, 0.995231,
179            0.995236, 0.995242, 0.995248, 0.995254, 0.99526, 0.995266, 0.995272,
180            0.995277, 0.995283, 0.995289, 0.995295, 0.995301, 0.995306, 0.995312,
181            0.995318, 0.995323, 0.995329, 0.995335, 0.99534, 0.995346, 0.995352,
182            0.995357, 0.995363, 0.995369, 0.995374, 0.99538, 0.995385, 0.995391,
183            0.995396, 0.995402, 0.995407, 0.995413, 0.995418, 0.995424, 0.995429,
184            0.995435, 0.99544, 0.995445, 0.995451, 0.995456, 0.995462, 0.995467,
185            0.995472, 0.995478, 0.995483, 0.995488, 0.995493, 0.995499, 0.995504,
186            0.995509, 0.995515, 0.99552, 0.995525, 0.99553, 0.995535, 0.995541, 0.995546,
187            0.995551, 0.995556, 0.995561, 0.995566, 0.995571, 0.995577, 0.995582,
188            0.995587, 0.995592, 0.995597, 0.995602, 0.995607, 0.995612, 0.995617,
189            0.995622, 0.995627, 0.995632, 0.995637, 0.995642, 0.995647, 0.995652,
190            0.995657, 0.995661, 0.995666, 0.995671, 0.995676, 0.995681, 0.995686,
191            0.995691, 0.995695, 0.9957, 0.995705, 0.99571, 0.995715, 0.995719, 0.995724,
192            0.995729, 0.995734, 0.995738, 0.995743, 0.995748, 0.995753, 0.995757,
193            0.995762, 0.995767, 0.995771, 0.995776, 0.995781, 0.995785, 0.99579,
194            0.995794, 0.995799, 0.995804, 0.995808, 0.995813, 0.995817, 0.995822,
195            0.995826, 0.995831, 0.995835, 0.99584, 0.995844, 0.995849, 0.995853,
196            0.995858, 0.995862, 0.995867, 0.995871, 0.995876, 0.99588, 0.995885,
197            0.995889, 0.995893, 0.995898, 0.995902, 0.995906, 0.995911, 0.995915,
198            0.99592, 0.995924, 0.995928, 0.995932, 0.995937, 0.995941, 0.995945, 0.99595,
199            0.995954, 0.995958, 0.995962, 0.995967, 0.995971, 0.995975, 0.995979,
200            0.995984, 0.995988, 0.995992, 0.995996, 0.996, 0.996004, 0.996009, 0.996013,
201            0.996017, 0.996021, 0.996025, 0.996029, 0.996033, 0.996037, 0.996041,
202            0.996046, 0.99605, 0.996054, 0.996058, 0.996062, 0.996066, 0.99607, 0.996074,
203            0.996078, 0.996082, 0.996086, 0.99609, 0.996094, 0.996098, 0.996102,
204            0.996106, 0.99611, 0.996114, 0.996117, 0.996121, 0.996125, 0.996129,
205            0.996133, 0.996137, 0.996141, 0.996145, 0.996149, 0.996152, 0.996156,
206            0.99616, 0.996164]
207
208# Run-time configuration parameters (can be set with command-line options)
209
210$rerun=1
211$inner=3
212$warmup=1
213$outer=4
214$includeSunSpider=true
215$includeV8=true
216$includeKraken=true
217$measureGC=false
218$benchmarkPattern=nil
219$verbosity=0
220$timeMode=:preciseTime
221$forceVMKind=nil
222$brief=false
223$silent=false
224$remoteHosts=[]
225$alsoLocal=false
226$sshOptions=[]
227$vms = []
228$needToCopyVMs = false
229$dontCopyVMs = false
230
231$prepare = true
232$run = true
233$analyze = []
234
235# Helpful functions and classes
236
237def smallUsage
238  puts "Use the --help option to get basic usage information."
239  exit 1
240end
241
242def usage
243  puts "bencher [options] <vm1> [<vm2> ...]"
244  puts
245  puts "Runs one or more JavaScript runtimes against SunSpider, V8, and/or Kraken"
246  puts "benchmarks, and reports detailed statistics.  What makes bencher special is"
247  puts "that each benchmark/VM configuration is run in a single VM invocation, and"
248  puts "the invocations are run in random order.  This minimizes systematics due to"
249  puts "one benchmark polluting the running time of another.  The fine-grained"
250  puts "interleaving of VM invocations further minimizes systematics due to changes in"
251  puts "the performance or behavior of your machine."
252  puts
253  puts "Bencher is highly configurable.  You can compare as many VMs as you like.  You"
254  puts "can change the amount of warm-up iterations, number of iterations executed per"
255  puts "VM invocation, and the number of VM invocations per benchmark.  By default,"
256  puts "SunSpider, VM, and Kraken are all run; but you can run any combination of these"
257  puts "suites."
258  puts
259  puts "The <vm> should be either a path to a JavaScript runtime executable (such as"
260  puts "jsc), or a string of the form <name>:<path>, where the <path> is the path to"
261  puts "the executable and <name> is the name that you would like to give the"
262  puts "configuration for the purposeof reporting.  If no name is given, a generic name"
263  puts "of the form Conf#<n> will be ascribed to the configuration automatically."
264  puts
265  puts "Options:"
266  puts "--rerun <n>          Set the number of iterations of the benchmark that"
267  puts "                     contribute to the measured run time.  Default is #{$rerun}."
268  puts "--inner <n>          Set the number of inner (per-runtime-invocation)"
269  puts "                     iterations.  Default is #{$inner}."
270  puts "--outer <n>          Set the number of runtime invocations for each benchmark."
271  puts "                     Default is #{$outer}."
272  puts "--warmup <n>         Set the number of warm-up runs per invocation.  Default"
273  puts "                     is #{$warmup}."
274  puts "--timing-mode        Set the way that bencher measures time.  Possible values"
275  puts "                     are 'preciseTime' and 'date'.  Default is 'preciseTime'."
276  puts "--force-vm-kind      Turn off auto-detection of VM kind, and assume that it is"
277  puts "                     the one specified.  Valid arguments are 'jsc' or"
278  puts "                     'DumpRenderTree'."
279  puts "--force-vm-copy      Force VM builds to be copied to bencher's working directory."
280  puts "                     This may reduce pathologies resulting from path names."
281  puts "--dont-copy-vms      Don't copy VMs even when doing a remote benchmarking run;"
282  puts "                     instead assume that they are already there."
283  puts "--v8-only            Only run V8."
284  puts "--sunspider-only     Only run SunSpider."
285  puts "--kraken-only        Only run Kraken."
286  puts "--exclude-v8         Exclude V8 (only run SunSpider and Kraken)."
287  puts "--exclude-sunspider  Exclude SunSpider (only run V8 and Kraken)."
288  puts "--exclude-kraken     Exclude Kraken (only run SunSpider and V8)."
289  puts "--benchmarks         Only run benchmarks matching the given regular expression."
290  puts "--measure-gc         Turn off manual calls to gc(), so that GC time is measured."
291  puts "                     Works best with large values of --inner.  You can also say"
292  puts "                     --measure-gc <conf>, which turns this on for one"
293  puts "                     configuration only."
294  puts "--verbose or -v      Print more stuff."
295  puts "--brief              Print only the final result for each VM."
296  puts "--silent             Don't print progress. This might slightly reduce some"
297  puts "                     performance perturbation."
298  puts "--remote <sshhosts>  Performance performance measurements remotely, on the given"
299  puts "                     SSH host(s). Easiest way to use this is to specify the SSH"
300  puts "                     user@host string. However, you can also supply a comma-"
301  puts "                     separated list of SSH hosts. Alternatively, you can use this"
302  puts "                     option multiple times to specify multiple hosts. This"
303  puts "                     automatically copies the WebKit release builds of the VMs"
304  puts "                     you specified to all of the hosts."
305  puts "--ssh-options        Pass additional options to SSH."
306  puts "--local              Also do a local benchmark run even when doing --remote."
307  puts "--prepare-only       Only prepare the bencher runscript (a shell script that"
308  puts "                     invokes the VMs to run benchmarks) but don't run it."
309  puts "--analyze            Only read the output of the runscript but don't do anything"
310  puts "                     else. This requires passing the same arguments to bencher"
311  puts "                     that you passed when running --prepare-only."
312  puts "--help or -h         Display this message."
313  puts
314  puts "Example:"
315  puts "bencher TipOfTree:/Volumes/Data/pizlo/OpenSource/WebKitBuild/Release/jsc MyChanges:/Volumes/Data/pizlo/secondary/OpenSource/WebKitBuild/Release/jsc"
316  exit 1
317end
318
319def fail(reason)
320  if reason.respond_to? :backtrace
321    puts "FAILED: #{reason}"
322    puts "Stack trace:"
323    puts reason.backtrace.join("\n")
324  else
325    puts "FAILED: #{reason}"
326  end
327  smallUsage
328end
329
330def quickFail(r1,r2)
331  $stderr.puts "#{$0}: #{r1}"
332  puts
333  fail(r2)
334end
335
336def intArg(argName,arg,min,max)
337  result=arg.to_i
338  unless result.to_s == arg
339    quickFail("Expected an integer value for #{argName}, but got #{arg}.",
340              "Invalid argument for command-line option")
341  end
342  if min and result<min
343    quickFail("Argument for #{argName} cannot be smaller than #{min}.",
344              "Invalid argument for command-line option")
345  end
346  if max and result>max
347    quickFail("Argument for #{argName} cannot be greater than #{max}.",
348              "Invalid argument for command-line option")
349  end
350  result
351end
352
353def computeMean(array)
354  sum=0.0
355  array.each {
356    | value |
357    sum += value
358  }
359  sum/array.length
360end
361
362def computeGeometricMean(array)
363  mult=1.0
364  array.each {
365    | value |
366    mult*=value
367  }
368  mult**(1.0/array.length)
369end
370
371def computeHarmonicMean(array)
372  1.0 / computeMean(array.collect{ | value | 1.0 / value })
373end
374
375def computeStdDev(array)
376  case array.length
377  when 0
378    0.0/0.0
379  when 1
380    0.0
381  else
382    begin
383      mean=computeMean(array)
384      sum=0.0
385      array.each {
386        | value |
387        sum += (value-mean)**2
388      }
389      Math.sqrt(sum/(array.length-1))
390    rescue
391      0.0/0.0
392    end
393  end
394end
395
396class Array
397  def shuffle!
398    size.downto(1) { |n| push delete_at(rand(n)) }
399    self
400  end
401end
402
403def inverseBetaRegularized(n)
404  IBR_LOOKUP[n-1]
405end
406
407def numToStr(num)
408  "%.4f"%(num.to_f)
409end
410
411class NoChange
412  attr_reader :amountFaster
413
414  def initialize(amountFaster)
415    @amountFaster = amountFaster
416  end
417
418  def shortForm
419    " "
420  end
421
422  def longForm
423    "  might be #{numToStr(@amountFaster)}x faster"
424  end
425
426  def to_s
427    if @amountFaster < 1.01
428      ""
429    else
430      longForm
431    end
432  end
433end
434
435class Faster
436  attr_reader :amountFaster
437
438  def initialize(amountFaster)
439    @amountFaster = amountFaster
440  end
441
442  def shortForm
443    "^"
444  end
445
446  def longForm
447    "^ definitely #{numToStr(@amountFaster)}x faster"
448  end
449
450  def to_s
451    longForm
452  end
453end
454
455class Slower
456  attr_reader :amountSlower
457
458  def initialize(amountSlower)
459    @amountSlower = amountSlower
460  end
461
462  def shortForm
463    "!"
464  end
465
466  def longForm
467    "! definitely #{numToStr(@amountSlower)}x slower"
468  end
469
470  def to_s
471    longForm
472  end
473end
474
475class MayBeSlower
476  attr_reader :amountSlower
477
478  def initialize(amountSlower)
479    @amountSlower = amountSlower
480  end
481
482  def shortForm
483    "?"
484  end
485
486  def longForm
487    "? might be #{numToStr(@amountSlower)}x slower"
488  end
489
490  def to_s
491    if @amountSlower < 1.01
492      "?"
493    else
494      longForm
495    end
496  end
497end
498
499class Stats
500  def initialize
501    @array = []
502  end
503
504  def add(value)
505    if value.is_a? Stats
506      add(value.array)
507    elsif value.respond_to? :each
508      value.each {
509        | v |
510        add(v)
511      }
512    else
513      @array << value.to_f
514    end
515  end
516
517  def array
518    @array
519  end
520
521  def sum
522    result=0
523    @array.each {
524      | value |
525      result += value
526    }
527    result
528  end
529
530  def min
531    @array.min
532  end
533
534  def max
535    @array.max
536  end
537
538  def size
539    @array.length
540  end
541
542  def mean
543    computeMean(array)
544  end
545
546  def arithmeticMean
547    mean
548  end
549
550  def stdDev
551    computeStdDev(array)
552  end
553
554  def stdErr
555    stdDev/Math.sqrt(size)
556  end
557
558  # Computes a 95% Student's t distribution confidence interval
559  def confInt
560    if size < 2
561      0.0/0.0
562    else
563      raise if size > 1000
564      Math.sqrt(size-1.0)*stdErr*Math.sqrt(-1.0+1.0/inverseBetaRegularized(size-1))
565    end
566  end
567
568  def lower
569    mean-confInt
570  end
571
572  def upper
573    mean+confInt
574  end
575
576  def geometricMean
577    computeGeometricMean(array)
578  end
579
580  def harmonicMean
581    computeHarmonicMean(array)
582  end
583
584  def compareTo(other)
585    if upper < other.lower
586      Faster.new(other.mean/mean)
587    elsif lower > other.upper
588      Slower.new(mean/other.mean)
589    elsif mean > other.mean
590      MayBeSlower.new(mean/other.mean)
591    else
592      NoChange.new(other.mean/mean)
593    end
594  end
595
596  def to_s
597    "size = #{size}, mean = #{mean}, stdDev = #{stdDev}, stdErr = #{stdErr}, confInt = #{confInt}"
598  end
599end
600
601def doublePuts(out1,out2,msg)
602  out1.puts "#{out2.path}: #{msg}" if $verbosity>=3
603  out2.puts msg
604end
605
606class Benchfile < File
607  @@counter = 0
608
609  attr_reader :filename, :basename
610
611  def initialize(name)
612    @basename, @filename = Benchfile.uniqueFilename(name)
613    super(@filename, "w")
614  end
615
616  def self.uniqueFilename(name)
617    if name.is_a? Array
618      basename = name[0] + @@counter.to_s + name[1]
619    else
620      basename = name + @@counter.to_s
621    end
622    filename = BENCH_DATA_PATH + "/" + basename
623    @@counter += 1
624    raise "Benchfile #{filename} already exists" if FileTest.exist?(filename)
625    [basename, filename]
626  end
627
628  def self.create(name)
629    file = Benchfile.new(name)
630    yield file
631    file.close
632    file.basename
633  end
634end
635
636$dataFiles={}
637def ensureFile(key, filename)
638  unless $dataFiles[key]
639    $dataFiles[key] = Benchfile.create(key) {
640      | outp |
641      doublePuts($stderr,outp,IO::read(filename))
642    }
643  end
644  $dataFiles[key]
645end
646
647def emitBenchRunCodeFile(name, plan, benchDataPath, benchPath)
648  case plan.vm.vmType
649  when :jsc
650    Benchfile.create("bencher") {
651      | file |
652      case $timeMode
653      when :preciseTime
654        doublePuts($stderr,file,"function __bencher_curTimeMS() {")
655        doublePuts($stderr,file,"   return preciseTime()*1000")
656        doublePuts($stderr,file,"}")
657      when :date
658        doublePuts($stderr,file,"function __bencher_curTimeMS() {")
659        doublePuts($stderr,file,"   return Date.now()")
660        doublePuts($stderr,file,"}")
661      else
662        raise
663      end
664
665      if benchDataPath
666        doublePuts($stderr,file,"load(#{benchDataPath.inspect});")
667        doublePuts($stderr,file,"gc();")
668        doublePuts($stderr,file,"for (var __bencher_index = 0; __bencher_index < #{$warmup+$inner}; ++__bencher_index) {")
669        doublePuts($stderr,file,"   before = __bencher_curTimeMS();")
670        $rerun.times {
671          doublePuts($stderr,file,"   load(#{benchPath.inspect});")
672        }
673        doublePuts($stderr,file,"   after = __bencher_curTimeMS();")
674        doublePuts($stderr,file,"   if (__bencher_index >= #{$warmup}) print(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_index - #{$warmup}) + \": Time: \"+(after-before));");
675        doublePuts($stderr,file,"   gc();") unless plan.vm.shouldMeasureGC
676        doublePuts($stderr,file,"}")
677      else
678        doublePuts($stderr,file,"function __bencher_run(__bencher_what) {")
679        doublePuts($stderr,file,"   var __bencher_before = __bencher_curTimeMS();")
680        $rerun.times {
681          doublePuts($stderr,file,"   run(__bencher_what);")
682        }
683        doublePuts($stderr,file,"   var __bencher_after = __bencher_curTimeMS();")
684        doublePuts($stderr,file,"   return __bencher_after - __bencher_before;")
685        doublePuts($stderr,file,"}")
686        $warmup.times {
687          doublePuts($stderr,file,"__bencher_run(#{benchPath.inspect})")
688          doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
689        }
690        $inner.times {
691          | innerIndex |
692          doublePuts($stderr,file,"print(\"#{name}: #{plan.vm}: #{plan.iteration}: #{innerIndex}: Time: \"+__bencher_run(#{benchPath.inspect}));")
693          doublePuts($stderr,file,"gc();") unless plan.vm.shouldMeasureGC
694        }
695      end
696    }
697  when :dumpRenderTree
698    mainCode = Benchfile.create("bencher") {
699      | file |
700      doublePuts($stderr,file,"__bencher_count = 0;")
701      doublePuts($stderr,file,"function __bencher_doNext(result) {")
702      doublePuts($stderr,file,"    if (__bencher_count >= #{$warmup})")
703      doublePuts($stderr,file,"        debug(\"#{name}: #{plan.vm}: #{plan.iteration}: \" + (__bencher_count - #{$warmup}) + \": Time: \" + result);")
704      doublePuts($stderr,file,"    __bencher_count++;")
705      doublePuts($stderr,file,"    if (__bencher_count < #{$inner+$warmup})")
706      doublePuts($stderr,file,"        __bencher_runImpl(__bencher_doNext);")
707      doublePuts($stderr,file,"    else")
708      doublePuts($stderr,file,"        quit();")
709      doublePuts($stderr,file,"}")
710      doublePuts($stderr,file,"__bencher_runImpl(__bencher_doNext);")
711    }
712
713    cssCode = Benchfile.create("bencher-css") {
714      | file |
715      doublePuts($stderr,file,".pass {\n    font-weight: bold;\n    color: green;\n}\n.fail {\n    font-weight: bold;\n    color: red;\n}\n\#console {\n    white-space: pre-wrap;\n    font-family: monospace;\n}")
716    }
717
718    preCode = Benchfile.create("bencher-pre") {
719      | file |
720      doublePuts($stderr,file,"if (window.testRunner) {")
721      doublePuts($stderr,file,"    if (window.enablePixelTesting) {")
722      doublePuts($stderr,file,"        testRunner.dumpAsTextWithPixelResults();")
723      doublePuts($stderr,file,"    } else {")
724      doublePuts($stderr,file,"        testRunner.dumpAsText();")
725      doublePuts($stderr,file,"    }")
726      doublePuts($stderr,file,"}")
727      doublePuts($stderr,file,"")
728      doublePuts($stderr,file,"function debug(msg)")
729      doublePuts($stderr,file,"{")
730      doublePuts($stderr,file,"    var span = document.createElement(\"span\");")
731      doublePuts($stderr,file,"    document.getElementById(\"console\").appendChild(span); // insert it first so XHTML knows the namespace")
732      doublePuts($stderr,file,"    span.innerHTML = msg + '<br />';")
733      doublePuts($stderr,file,"}")
734      doublePuts($stderr,file,"")
735      doublePuts($stderr,file,"function quit() {")
736      doublePuts($stderr,file,"    testRunner.notifyDone();")
737      doublePuts($stderr,file,"}")
738      doublePuts($stderr,file,"")
739      doublePuts($stderr,file,"__bencher_continuation=null;")
740      doublePuts($stderr,file,"")
741      doublePuts($stderr,file,"function reportResult(result) {")
742      doublePuts($stderr,file,"    __bencher_continuation(result);")
743      doublePuts($stderr,file,"}")
744      doublePuts($stderr,file,"")
745      doublePuts($stderr,file,"function __bencher_runImpl(continuation) {")
746      doublePuts($stderr,file,"    function doit() {")
747      doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"\";")
748      doublePuts($stderr,file,"        document.getElementById(\"frameparent\").innerHTML = \"<iframe id='testframe'>\";")
749      doublePuts($stderr,file,"        var testFrame = document.getElementById(\"testframe\");")
750      doublePuts($stderr,file,"        testFrame.contentDocument.open();")
751      doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<!DOCTYPE html>\\n<head></head><body><div id=\\\"console\\\"></div>\");")
752      if benchDataPath
753        doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script src=\\\"#{benchDataPath}\\\"></script>\");")
754      end
755      doublePuts($stderr,file,"        testFrame.contentDocument.write(\"<script type=\\\"text/javascript\\\">__bencher_before = Date.now();</script><script src=\\\"#{benchPath}\\\"></script><script type=\\\"text/javascript\\\">window.parent.reportResult(Date.now() - __bencher_before);</script></body></html>\");")
756      doublePuts($stderr,file,"        testFrame.contentDocument.close();")
757      doublePuts($stderr,file,"    }")
758      doublePuts($stderr,file,"    __bencher_continuation = continuation;")
759      doublePuts($stderr,file,"    window.setTimeout(doit, 10);")
760      doublePuts($stderr,file,"}")
761    }
762
763    Benchfile.create(["bencher-htmldoc",".html"]) {
764      | file |
765      doublePuts($stderr,file,"<!DOCTYPE HTML PUBLIC \"-//IETF//DTD HTML//EN\">\n<html><head><link rel=\"stylesheet\" href=\"#{cssCode}\"><script src=\"#{preCode}\"></script></head><body><div id=\"console\"></div><div id=\"frameparent\"></div><script src=\"#{mainCode}\"></script></body></html>")
766    }
767  else
768    raise
769  end
770end
771
772def emitBenchRunCode(name, plan, benchDataPath, benchPath)
773  plan.vm.emitRunCode(emitBenchRunCodeFile(name, plan, benchDataPath, benchPath))
774end
775
776def planForDescription(plans, benchFullname, vmName, iteration)
777  raise unless benchFullname =~ /\//
778  suiteName = $~.pre_match
779  benchName = $~.post_match
780  result = plans.select{|v| v.suite.name == suiteName and v.benchmark.name == benchName and v.vm.name == vmName and v.iteration == iteration}
781  raise unless result.size == 1
782  result[0]
783end
784
785class ParsedResult
786  attr_reader :plan, :innerIndex, :time
787
788  def initialize(plan, innerIndex, time)
789    @plan = plan
790    @innerIndex = innerIndex
791    @time = time
792
793    raise unless @plan.is_a? BenchPlan
794    raise unless @innerIndex.is_a? Integer
795    raise unless @time.is_a? Numeric
796  end
797
798  def benchmark
799    plan.benchmark
800  end
801
802  def suite
803    plan.suite
804  end
805
806  def vm
807    plan.vm
808  end
809
810  def outerIndex
811    plan.iteration
812  end
813
814  def self.parse(plans, string)
815    if string =~ /([a-zA-Z0-9\/-]+): ([a-zA-Z0-9_# ]+): ([0-9]+): ([0-9]+): Time: /
816      benchFullname = $1
817      vmName = $2
818      outerIndex = $3.to_i
819      innerIndex = $4.to_i
820      time = $~.post_match.to_f
821      ParsedResult.new(planForDescription(plans, benchFullname, vmName, outerIndex), innerIndex, time)
822    else
823      nil
824    end
825  end
826end
827
828class VM
829  def initialize(origPath, name, nameKind, svnRevision)
830    @origPath = origPath.to_s
831    @path = origPath.to_s
832    @name = name
833    @nameKind = nameKind
834
835    if $forceVMKind
836      @vmType = $forceVMKind
837    else
838      if @origPath =~ /DumpRenderTree$/
839        @vmType = :dumpRenderTree
840      else
841        @vmType = :jsc
842      end
843    end
844
845    @svnRevision = svnRevision
846
847    # Try to detect information about the VM.
848    if path =~ /\/WebKitBuild\/Release\/([a-zA-Z]+)$/
849      @checkoutPath = $~.pre_match
850      # FIXME: Use some variant of this:
851      # <bdash>   def retrieve_revision
852      # <bdash>     `perl -I#{@path}/Tools/Scripts -MVCSUtils -e 'print svnRevisionForDirectory("#{@path}");'`.to_i
853      # <bdash>   end
854      unless @svnRevision
855        begin
856          Dir.chdir(@checkoutPath) {
857            $stderr.puts ">> cd #{@checkoutPath} && svn info" if $verbosity>=2
858            IO.popen("svn info", "r") {
859              | inp |
860              inp.each_line {
861                | line |
862                if line =~ /Revision: ([0-9]+)/
863                  @svnRevision = $1
864                end
865              }
866            }
867          }
868          unless @svnRevision
869            $stderr.puts "Warning: running svn info for #{name} silently failed."
870          end
871        rescue => e
872          # Failed to detect svn revision.
873          $stderr.puts "Warning: could not get svn revision information for #{name}: #{e}"
874        end
875      end
876    else
877      $stderr.puts "Warning: could not identify checkout location for #{name}"
878    end
879
880    if @path =~ /\/Release\/([a-zA-Z]+)$/
881      @libPath, @relativeBinPath = $~.pre_match+"/Release", "./#{$1}"
882    elsif @path =~ /\/Contents\/Resources\/([a-zA-Z]+)$/
883      @libPath = $~.pre_match
884    elsif @path =~ /\/JavaScriptCore.framework\/Resources\/([a-zA-Z]+)$/
885      @libPath, @relativeBinPath = $~.pre_match, $&[1..-1]
886    end
887  end
888
889  def canCopyIntoBenchPath
890    if @libPath and @relativeBinPath
891      true
892    else
893      false
894    end
895  end
896
897  def copyIntoBenchPath
898    raise unless canCopyIntoBenchPath
899    basename, filename = Benchfile.uniqueFilename("vm")
900    raise unless Dir.mkdir(filename)
901    cmd = "cp -a #{@libPath.inspect}/* #{filename.inspect}"
902    $stderr.puts ">> #{cmd}" if $verbosity>=2
903    raise unless system(cmd)
904    @path = "#{basename}/#{@relativeBinPath}"
905    @libPath = basename
906  end
907
908  def to_s
909    @name
910  end
911
912  def name
913    @name
914  end
915
916  def shouldMeasureGC
917    $measureGC == true or ($measureGC == name)
918  end
919
920  def origPath
921    @origPath
922  end
923
924  def path
925    @path
926  end
927
928  def nameKind
929    @nameKind
930  end
931
932  def vmType
933    @vmType
934  end
935
936  def checkoutPath
937    @checkoutPath
938  end
939
940  def svnRevision
941    @svnRevision
942  end
943
944  def printFunction
945    case @vmType
946    when :jsc
947      "print"
948    when :dumpRenderTree
949      "debug"
950    else
951      raise @vmType
952    end
953  end
954
955  def emitRunCode(fileToRun)
956    myLibPath = @libPath
957    myLibPath = "" unless myLibPath
958    $script.puts "export DYLD_LIBRARY_PATH=#{myLibPath.to_s.inspect}"
959    $script.puts "export DYLD_FRAMEWORK_PATH=#{myLibPath.to_s.inspect}"
960    $script.puts "#{path} #{fileToRun}"
961  end
962end
963
964class StatsAccumulator
965  def initialize
966    @stats = []
967    ($outer*$inner).times {
968      @stats << Stats.new
969    }
970  end
971
972  def statsForIteration(outerIteration, innerIteration)
973    @stats[outerIteration*$inner + innerIteration]
974  end
975
976  def stats
977    result = Stats.new
978    @stats.each {
979      | stat |
980      result.add(yield stat)
981    }
982    result
983  end
984
985  def geometricMeanStats
986    stats {
987      | stat |
988      stat.geometricMean
989    }
990  end
991
992  def arithmeticMeanStats
993    stats {
994      | stat |
995      stat.arithmeticMean
996    }
997  end
998end
999
1000module Benchmark
1001  attr_accessor :benchmarkSuite
1002  attr_reader :name
1003
1004  def fullname
1005    benchmarkSuite.name + "/" + name
1006  end
1007
1008  def to_s
1009    fullname
1010  end
1011end
1012
1013class SunSpiderBenchmark
1014  include Benchmark
1015
1016  def initialize(name)
1017    @name = name
1018  end
1019
1020  def emitRunCode(plan)
1021    emitBenchRunCode(fullname, plan, nil, ensureFile("SunSpider-#{@name}", "#{SUNSPIDER_PATH}/#{@name}.js"))
1022  end
1023end
1024
1025class V8Benchmark
1026  include Benchmark
1027
1028  def initialize(name)
1029    @name = name
1030  end
1031
1032  def emitRunCode(plan)
1033    emitBenchRunCode(fullname, plan, nil, ensureFile("V8-#{@name}", "#{V8_PATH}/v8-#{@name}.js"))
1034  end
1035end
1036
1037class KrakenBenchmark
1038  include Benchmark
1039
1040  def initialize(name)
1041    @name = name
1042  end
1043
1044  def emitRunCode(plan)
1045    emitBenchRunCode(fullname, plan, ensureFile("KrakenData-#{@name}", "#{KRAKEN_PATH}/#{@name}-data.js"), ensureFile("Kraken-#{@name}", "#{KRAKEN_PATH}/#{@name}.js"))
1046  end
1047end
1048
1049class BenchmarkSuite
1050  def initialize(name, path, preferredMean)
1051    @name = name
1052    @path = path
1053    @preferredMean = preferredMean
1054    @benchmarks = []
1055  end
1056
1057  def name
1058    @name
1059  end
1060
1061  def to_s
1062    @name
1063  end
1064
1065  def path
1066    @path
1067  end
1068
1069  def add(benchmark)
1070    if not $benchmarkPattern or "#{@name}/#{benchmark.name}" =~ $benchmarkPattern
1071      benchmark.benchmarkSuite = self
1072      @benchmarks << benchmark
1073    end
1074  end
1075
1076  def benchmarks
1077    @benchmarks
1078  end
1079
1080  def benchmarkForName(name)
1081    result = @benchmarks.select{|v| v.name == name}
1082    raise unless result.length == 1
1083    result[0]
1084  end
1085
1086  def empty?
1087    @benchmarks.empty?
1088  end
1089
1090  def retain_if
1091    @benchmarks.delete_if {
1092      | benchmark |
1093      not yield benchmark
1094    }
1095  end
1096
1097  def preferredMean
1098    @preferredMean
1099  end
1100
1101  def computeMean(stat)
1102    stat.send @preferredMean
1103  end
1104end
1105
1106class BenchRunPlan
1107  def initialize(benchmark, vm, iteration)
1108    @benchmark = benchmark
1109    @vm = vm
1110    @iteration = iteration
1111  end
1112
1113  def benchmark
1114    @benchmark
1115  end
1116
1117  def suite
1118    @benchmark.benchmarkSuite
1119  end
1120
1121  def vm
1122    @vm
1123  end
1124
1125  def iteration
1126    @iteration
1127  end
1128
1129  def emitRunCode
1130    @benchmark.emitRunCode(self)
1131  end
1132end
1133
1134class BenchmarkOnVM
1135  def initialize(benchmark, suiteOnVM)
1136    @benchmark = benchmark
1137    @suiteOnVM = suiteOnVM
1138    @stats = Stats.new
1139  end
1140
1141  def to_s
1142    "#{@benchmark} on #{@suiteOnVM.vm}"
1143  end
1144
1145  def benchmark
1146    @benchmark
1147  end
1148
1149  def vm
1150    @suiteOnVM.vm
1151  end
1152
1153  def vmStats
1154    @suiteOnVM.vmStats
1155  end
1156
1157  def suite
1158    @benchmark.benchmarkSuite
1159  end
1160
1161  def suiteOnVM
1162    @suiteOnVM
1163  end
1164
1165  def stats
1166    @stats
1167  end
1168
1169  def parseResult(result)
1170    raise "VM mismatch; I've got #{vm} and they've got #{result.vm}" unless result.vm == vm
1171    raise unless result.benchmark == @benchmark
1172    @stats.add(result.time)
1173  end
1174end
1175
1176class SuiteOnVM < StatsAccumulator
1177  def initialize(vm, vmStats, suite)
1178    super()
1179    @vm = vm
1180    @vmStats = vmStats
1181    @suite = suite
1182
1183    raise unless @vm.is_a? VM
1184    raise unless @vmStats.is_a? StatsAccumulator
1185    raise unless @suite.is_a? BenchmarkSuite
1186  end
1187
1188  def to_s
1189    "#{@suite} on #{@vm}"
1190  end
1191
1192  def suite
1193    @suite
1194  end
1195
1196  def vm
1197    @vm
1198  end
1199
1200  def vmStats
1201    raise unless @vmStats
1202    @vmStats
1203  end
1204end
1205
1206class BenchPlan
1207  def initialize(benchmarkOnVM, iteration)
1208    @benchmarkOnVM = benchmarkOnVM
1209    @iteration = iteration
1210  end
1211
1212  def to_s
1213    "#{@benchmarkOnVM} \##{@iteration+1}"
1214  end
1215
1216  def benchmarkOnVM
1217    @benchmarkOnVM
1218  end
1219
1220  def benchmark
1221    @benchmarkOnVM.benchmark
1222  end
1223
1224  def suite
1225    @benchmarkOnVM.suite
1226  end
1227
1228  def vm
1229    @benchmarkOnVM.vm
1230  end
1231
1232  def iteration
1233    @iteration
1234  end
1235
1236  def parseResult(result)
1237    raise unless result.plan == self
1238    @benchmarkOnVM.parseResult(result)
1239    @benchmarkOnVM.vmStats.statsForIteration(@iteration, result.innerIndex).add(result.time)
1240    @benchmarkOnVM.suiteOnVM.statsForIteration(@iteration, result.innerIndex).add(result.time)
1241  end
1242end
1243
1244def lpad(str,chars)
1245  if str.length>chars
1246    str
1247  else
1248    "%#{chars}s"%(str)
1249  end
1250end
1251
1252def rpad(str,chars)
1253  while str.length<chars
1254    str+=" "
1255  end
1256  str
1257end
1258
1259def center(str,chars)
1260  while str.length<chars
1261    str+=" "
1262    if str.length<chars
1263      str=" "+str
1264    end
1265  end
1266  str
1267end
1268
1269def statsToStr(stats)
1270  if $inner*$outer == 1
1271    string = numToStr(stats.mean)
1272    raise unless string =~ /\./
1273    left = $~.pre_match
1274    right = $~.post_match
1275    lpad(left,12)+"."+rpad(right,9)
1276  else
1277    lpad(numToStr(stats.mean),11)+"+-"+rpad(numToStr(stats.confInt),9)
1278  end
1279end
1280
1281def plural(num)
1282  if num == 1
1283    ""
1284  else
1285    "s"
1286  end
1287end
1288
1289def wrap(str, columns)
1290  array = str.split
1291  result = ""
1292  curLine = array.shift
1293  array.each {
1294    | curStr |
1295    if (curLine + " " + curStr).size > columns
1296      result += curLine + "\n"
1297      curLine = curStr
1298    else
1299      curLine += " " + curStr
1300    end
1301  }
1302  result + curLine + "\n"
1303end
1304
1305def runAndGetResults
1306  results = nil
1307  Dir.chdir(BENCH_DATA_PATH) {
1308    IO.popen("sh ./runscript", "r") {
1309      | inp |
1310      results = inp.read
1311    }
1312    raise "Script did not complete correctly: #{$?}" unless $?.success?
1313  }
1314  raise unless results
1315  results
1316end
1317
1318def parseAndDisplayResults(results)
1319  vmStatses = []
1320  $vms.each {
1321    vmStatses << StatsAccumulator.new
1322  }
1323
1324  suitesOnVMs = []
1325  suitesOnVMsForSuite = {}
1326  $suites.each {
1327    | suite |
1328    suitesOnVMsForSuite[suite] = []
1329  }
1330  suitesOnVMsForVM = {}
1331  $vms.each {
1332    | vm |
1333    suitesOnVMsForVM[vm] = []
1334  }
1335
1336  benchmarksOnVMs = []
1337  benchmarksOnVMsForBenchmark = {}
1338  $benchmarks.each {
1339    | benchmark |
1340    benchmarksOnVMsForBenchmark[benchmark] = []
1341  }
1342
1343  $vms.each_with_index {
1344    | vm, vmIndex |
1345    vmStats = vmStatses[vmIndex]
1346    $suites.each {
1347      | suite |
1348      suiteOnVM = SuiteOnVM.new(vm, vmStats, suite)
1349      suitesOnVMs << suiteOnVM
1350      suitesOnVMsForSuite[suite] << suiteOnVM
1351      suitesOnVMsForVM[vm] << suiteOnVM
1352      suite.benchmarks.each {
1353        | benchmark |
1354        benchmarkOnVM = BenchmarkOnVM.new(benchmark, suiteOnVM)
1355        benchmarksOnVMs << benchmarkOnVM
1356        benchmarksOnVMsForBenchmark[benchmark] << benchmarkOnVM
1357      }
1358    }
1359  }
1360
1361  plans = []
1362  benchmarksOnVMs.each {
1363    | benchmarkOnVM |
1364    $outer.times {
1365      | iteration |
1366      plans << BenchPlan.new(benchmarkOnVM, iteration)
1367    }
1368  }
1369
1370  hostname = nil
1371  hwmodel = nil
1372  results.each_line {
1373    | line |
1374    line.chomp!
1375    if line =~ /HOSTNAME:([^.]+)/
1376      hostname = $1
1377    elsif line =~ /HARDWARE:hw\.model: /
1378      hwmodel = $~.post_match.chomp
1379    else
1380      result = ParsedResult.parse(plans, line.chomp)
1381      if result
1382        result.plan.parseResult(result)
1383      end
1384    end
1385  }
1386
1387  # Compute the geomean of the preferred means of results on a SuiteOnVM
1388  overallResults = []
1389  $vms.each {
1390    | vm |
1391    result = Stats.new
1392    $outer.times {
1393      | outerIndex |
1394      $inner.times {
1395        | innerIndex |
1396        curResult = Stats.new
1397        suitesOnVMsForVM[vm].each {
1398          | suiteOnVM |
1399          # For a given iteration, suite, and VM, compute the suite's preferred mean
1400          # over the data collected for all benchmarks in that suite. We'll have one
1401          # sample per benchmark. For example on V8 this will be the geomean of 1
1402          # sample for crypto, 1 sample for deltablue, and so on, and 1 sample for
1403          # splay.
1404          curResult.add(suiteOnVM.suite.computeMean(suiteOnVM.statsForIteration(outerIndex, innerIndex)))
1405        }
1406
1407        # curResult now holds 1 sample for each of the means computed in the above
1408        # loop. Compute the geomean over this, and store it.
1409        result.add(curResult.geometricMean)
1410      }
1411    }
1412
1413    # $overallResults will have a Stats for each VM. That Stats object will hold
1414    # $inner*$outer geomeans, allowing us to compute the arithmetic mean and
1415    # confidence interval of the geomeans of preferred means. Convoluted, but
1416    # useful and probably sound.
1417    overallResults << result
1418  }
1419
1420  if $verbosity >= 2
1421    benchmarksOnVMs.each {
1422      | benchmarkOnVM |
1423      $stderr.puts "#{benchmarkOnVM}: #{benchmarkOnVM.stats}"
1424    }
1425
1426    $vms.each_with_index {
1427      | vm, vmIndex |
1428      vmStats = vmStatses[vmIndex]
1429      $stderr.puts "#{vm} (arithmeticMean): #{vmStats.arithmeticMeanStats}"
1430      $stderr.puts "#{vm} (geometricMean): #{vmStats.geometricMeanStats}"
1431    }
1432  end
1433
1434  reportName =
1435    (if ($vms.collect {
1436           | vm |
1437           vm.nameKind
1438         }.index :auto)
1439       ""
1440     else
1441       $vms.collect {
1442         | vm |
1443         vm.to_s
1444       }.join("_") + "_"
1445     end) +
1446    ($suites.collect {
1447       | suite |
1448       suite.to_s
1449     }.join("")) + "_" +
1450    (if hostname
1451       hostname + "_"
1452     else
1453       ""
1454     end)+
1455    (begin
1456       time = Time.now
1457       "%04d%02d%02d_%02d%02d" %
1458         [ time.year, time.month, time.day,
1459           time.hour, time.min ]
1460     end) +
1461    "_benchReport.txt"
1462
1463  unless $brief
1464    puts "Generating benchmark report at #{reportName}"
1465  end
1466
1467  outp = $stdout
1468  begin
1469    outp = File.open(reportName,"w")
1470  rescue => e
1471    $stderr.puts "Error: could not save report to #{reportName}: #{e}"
1472    $stderr.puts
1473  end
1474
1475  def createVMsString
1476    result = ""
1477    result += "   " if $suites.size > 1
1478    result += rpad("", $benchpad)
1479    result += " "
1480    $vms.size.times {
1481      | index |
1482      if index != 0
1483        result += " "+NoChange.new(0).shortForm
1484      end
1485      result += lpad(center($vms[index].name, 9+9+2), 11+9+2)
1486    }
1487    result += "    "
1488    if $vms.size >= 3
1489      result += center("#{$vms[-1].name} v. #{$vms[0].name}",26)
1490    elsif $vms.size >= 2
1491      result += " "*26
1492    end
1493    result
1494  end
1495
1496  columns = [createVMsString.size, 78].max
1497
1498  outp.print "Benchmark report for "
1499  if $suites.size == 1
1500    outp.print $suites[0].to_s
1501  elsif $suites.size == 2
1502    outp.print "#{$suites[0]} and #{$suites[1]}"
1503  else
1504    outp.print "#{$suites[0..-2].join(', ')}, and #{$suites[-1]}"
1505  end
1506  if hostname
1507    outp.print " on #{hostname}"
1508  end
1509  if hwmodel
1510    outp.print " (#{hwmodel})"
1511  end
1512  outp.puts "."
1513  outp.puts
1514
1515  # This looks stupid; revisit later.
1516  if false
1517    $suites.each {
1518      | suite |
1519      outp.puts "#{suite} at #{suite.path}"
1520    }
1521
1522    outp.puts
1523  end
1524
1525  outp.puts "VMs tested:"
1526  $vms.each {
1527    | vm |
1528    outp.print "\"#{vm.name}\" at #{vm.origPath}"
1529    if vm.svnRevision
1530      outp.print " (r#{vm.svnRevision})"
1531    end
1532    outp.puts
1533  }
1534
1535  outp.puts
1536
1537  outp.puts wrap("Collected #{$outer*$inner} sample#{plural($outer*$inner)} per benchmark/VM, "+
1538                 "with #{$outer} VM invocation#{plural($outer)} per benchmark."+
1539                 (if $rerun > 1 then (" Ran #{$rerun} benchmark iterations, and measured the "+
1540                                      "total time of those iterations, for each sample.")
1541                  else "" end)+
1542                 (if $measureGC == true then (" No manual garbage collection invocations were "+
1543                                              "emitted.")
1544                  elsif $measureGC then (" Emitted a call to gc() between sample measurements for "+
1545                                         "all VMs except #{$measureGC}.")
1546                  else (" Emitted a call to gc() between sample measurements.") end)+
1547                 (if $warmup == 0 then (" Did not include any warm-up iterations; measurements "+
1548                                        "began with the very first iteration.")
1549                  else (" Used #{$warmup*$rerun} benchmark iteration#{plural($warmup*$rerun)} per VM "+
1550                        "invocation for warm-up.") end)+
1551                 (case $timeMode
1552                  when :preciseTime then (" Used the jsc-specific preciseTime() function to get "+
1553                                          "microsecond-level timing.")
1554                  when :date then (" Used the portable Date.now() method to get millisecond-"+
1555                                   "level timing.")
1556                  else raise end)+
1557                 " Reporting benchmark execution times with 95% confidence "+
1558                 "intervals in milliseconds.",
1559                 columns)
1560
1561  outp.puts
1562
1563  def printVMs(outp)
1564    outp.puts createVMsString
1565  end
1566
1567  def summaryStats(outp, accumulators, name, &proc)
1568    outp.print "   " if $suites.size > 1
1569    outp.print rpad(name, $benchpad)
1570    outp.print " "
1571    accumulators.size.times {
1572      | index |
1573      if index != 0
1574        outp.print " "+accumulators[index].stats(&proc).compareTo(accumulators[index-1].stats(&proc)).shortForm
1575      end
1576      outp.print statsToStr(accumulators[index].stats(&proc))
1577    }
1578    if accumulators.size>=2
1579      outp.print("    "+accumulators[-1].stats(&proc).compareTo(accumulators[0].stats(&proc)).longForm)
1580    end
1581    outp.puts
1582  end
1583
1584  def meanName(currentMean, preferredMean)
1585    result = "<#{currentMean}>"
1586    if "#{currentMean}Mean" == preferredMean.to_s
1587      result += " *"
1588    end
1589    result
1590  end
1591
1592  def allSummaryStats(outp, accumulators, preferredMean)
1593    summaryStats(outp, accumulators, meanName("arithmetic", preferredMean)) {
1594      | stat |
1595      stat.arithmeticMean
1596    }
1597
1598    summaryStats(outp, accumulators, meanName("geometric", preferredMean)) {
1599      | stat |
1600      stat.geometricMean
1601    }
1602
1603    summaryStats(outp, accumulators, meanName("harmonic", preferredMean)) {
1604      | stat |
1605      stat.harmonicMean
1606    }
1607  end
1608
1609  $suites.each {
1610    | suite |
1611    printVMs(outp)
1612    if $suites.size > 1
1613      outp.puts "#{suite.name}:"
1614    else
1615      outp.puts
1616    end
1617    suite.benchmarks.each {
1618      | benchmark |
1619      outp.print "   " if $suites.size > 1
1620      outp.print rpad(benchmark.name, $benchpad)
1621      outp.print " "
1622      myConfigs = benchmarksOnVMsForBenchmark[benchmark]
1623      myConfigs.size.times {
1624        | index |
1625        if index != 0
1626          outp.print " "+myConfigs[index].stats.compareTo(myConfigs[index-1].stats).shortForm
1627        end
1628        outp.print statsToStr(myConfigs[index].stats)
1629      }
1630      if $vms.size>=2
1631        outp.print("    "+myConfigs[-1].stats.compareTo(myConfigs[0].stats).to_s)
1632      end
1633      outp.puts
1634    }
1635    outp.puts
1636    allSummaryStats(outp, suitesOnVMsForSuite[suite], suite.preferredMean)
1637    outp.puts if $suites.size > 1
1638  }
1639
1640  if $suites.size > 1
1641    printVMs(outp)
1642    outp.puts "All benchmarks:"
1643    allSummaryStats(outp, vmStatses, nil)
1644
1645    outp.puts
1646    printVMs(outp)
1647    outp.puts "Geomean of preferred means:"
1648    outp.print "   "
1649    outp.print rpad("<scaled-result>", $benchpad)
1650    outp.print " "
1651    $vms.size.times {
1652      | index |
1653      if index != 0
1654        outp.print " "+overallResults[index].compareTo(overallResults[index-1]).shortForm
1655      end
1656      outp.print statsToStr(overallResults[index])
1657    }
1658    if overallResults.size>=2
1659      outp.print("    "+overallResults[-1].compareTo(overallResults[0]).longForm)
1660    end
1661    outp.puts
1662  end
1663  outp.puts
1664
1665  if outp != $stdout
1666    outp.close
1667  end
1668
1669  if outp != $stdout and not $brief
1670    puts
1671    File.open(reportName) {
1672      | inp |
1673      puts inp.read
1674    }
1675  end
1676
1677  if $brief
1678    puts(overallResults.collect{|stats| stats.mean}.join("\t"))
1679    puts(overallResults.collect{|stats| stats.confInt}.join("\t"))
1680  end
1681
1682
1683end
1684
1685begin
1686  $sawBenchOptions = false
1687
1688  def resetBenchOptionsIfNecessary
1689    unless $sawBenchOptions
1690      $includeSunSpider = false
1691      $includeV8 = false
1692      $includeKraken = false
1693      $sawBenchOptions = true
1694    end
1695  end
1696
1697  GetoptLong.new(['--rerun', GetoptLong::REQUIRED_ARGUMENT],
1698                 ['--inner', GetoptLong::REQUIRED_ARGUMENT],
1699                 ['--outer', GetoptLong::REQUIRED_ARGUMENT],
1700                 ['--warmup', GetoptLong::REQUIRED_ARGUMENT],
1701                 ['--timing-mode', GetoptLong::REQUIRED_ARGUMENT],
1702                 ['--sunspider-only', GetoptLong::NO_ARGUMENT],
1703                 ['--v8-only', GetoptLong::NO_ARGUMENT],
1704                 ['--kraken-only', GetoptLong::NO_ARGUMENT],
1705                 ['--exclude-sunspider', GetoptLong::NO_ARGUMENT],
1706                 ['--exclude-v8', GetoptLong::NO_ARGUMENT],
1707                 ['--exclude-kraken', GetoptLong::NO_ARGUMENT],
1708                 ['--sunspider', GetoptLong::NO_ARGUMENT],
1709                 ['--v8', GetoptLong::NO_ARGUMENT],
1710                 ['--kraken', GetoptLong::NO_ARGUMENT],
1711                 ['--benchmarks', GetoptLong::REQUIRED_ARGUMENT],
1712                 ['--measure-gc', GetoptLong::OPTIONAL_ARGUMENT],
1713                 ['--force-vm-kind', GetoptLong::REQUIRED_ARGUMENT],
1714                 ['--force-vm-copy', GetoptLong::NO_ARGUMENT],
1715                 ['--dont-copy-vms', GetoptLong::NO_ARGUMENT],
1716                 ['--verbose', '-v', GetoptLong::NO_ARGUMENT],
1717                 ['--brief', GetoptLong::NO_ARGUMENT],
1718                 ['--silent', GetoptLong::NO_ARGUMENT],
1719                 ['--remote', GetoptLong::REQUIRED_ARGUMENT],
1720                 ['--local', GetoptLong::NO_ARGUMENT],
1721                 ['--ssh-options', GetoptLong::REQUIRED_ARGUMENT],
1722                 ['--slave', GetoptLong::NO_ARGUMENT],
1723                 ['--prepare-only', GetoptLong::NO_ARGUMENT],
1724                 ['--analyze', GetoptLong::REQUIRED_ARGUMENT],
1725                 ['--vms', GetoptLong::REQUIRED_ARGUMENT],
1726                 ['--help', '-h', GetoptLong::NO_ARGUMENT]).each {
1727    | opt, arg |
1728    case opt
1729    when '--rerun'
1730      $rerun = intArg(opt,arg,1,nil)
1731    when '--inner'
1732      $inner = intArg(opt,arg,1,nil)
1733    when '--outer'
1734      $outer = intArg(opt,arg,1,nil)
1735    when '--warmup'
1736      $warmup = intArg(opt,arg,0,nil)
1737    when '--timing-mode'
1738      if arg.upcase == "PRECISETIME"
1739        $timeMode = :preciseTime
1740      elsif arg.upcase == "DATE"
1741        $timeMode = :date
1742      elsif arg.upcase == "AUTO"
1743        $timeMode = :auto
1744      else
1745        quickFail("Expected either 'preciseTime', 'date', or 'auto' for --time-mode, but got '#{arg}'.",
1746                  "Invalid argument for command-line option")
1747      end
1748    when '--force-vm-kind'
1749      if arg.upcase == "JSC"
1750        $forceVMKind = :jsc
1751      elsif arg.upcase == "DUMPRENDERTREE"
1752        $forceVMKind = :dumpRenderTree
1753      elsif arg.upcase == "AUTO"
1754        $forceVMKind = nil
1755      else
1756        quickFail("Expected either 'jsc' or 'DumpRenderTree' for --force-vm-kind, but got '#{arg}'.",
1757                  "Invalid argument for command-line option")
1758      end
1759    when '--force-vm-copy'
1760      $needToCopyVMs = true
1761    when '--dont-copy-vms'
1762      $dontCopyVMs = true
1763    when '--sunspider-only'
1764      $includeV8 = false
1765      $includeKraken = false
1766    when '--v8-only'
1767      $includeSunSpider = false
1768      $includeKraken = false
1769    when '--kraken-only'
1770      $includeSunSpider = false
1771      $includeV8 = false
1772    when '--exclude-sunspider'
1773      $includeSunSpider = false
1774    when '--exclude-v8'
1775      $includeV8 = false
1776    when '--exclude-kraken'
1777      $includeKraken = false
1778    when '--sunspider'
1779      resetBenchOptionsIfNecessary
1780      $includeSunSpider = true
1781    when '--v8'
1782      resetBenchOptionsIfNecessary
1783      $includeV8 = true
1784    when '--kraken'
1785      resetBenchOptionsIfNecessary
1786      $includeKraken = true
1787    when '--benchmarks'
1788      $benchmarkPattern = Regexp.new(arg)
1789    when '--measure-gc'
1790      if arg == ''
1791        $measureGC = true
1792      else
1793        $measureGC = arg
1794      end
1795    when '--verbose'
1796      $verbosity += 1
1797    when '--brief'
1798      $brief = true
1799    when '--silent'
1800      $silent = true
1801    when '--remote'
1802      $remoteHosts += arg.split(',')
1803      $needToCopyVMs = true
1804    when '--ssh-options'
1805      $sshOptions << arg
1806    when '--local'
1807      $alsoLocal = true
1808    when '--prepare-only'
1809      $run = false
1810    when '--analyze'
1811      $prepare = false
1812      $run = false
1813      $analyze << arg
1814    when '--help'
1815      usage
1816    else
1817      raise "bad option: #{opt}"
1818    end
1819  }
1820
1821  # If the --dont-copy-vms option was passed, it overrides the --force-vm-copy option.
1822  if $dontCopyVMs
1823    $needToCopyVMs = false
1824  end
1825
1826  SUNSPIDER = BenchmarkSuite.new("SunSpider", SUNSPIDER_PATH, :arithmeticMean)
1827  ["3d-cube", "3d-morph", "3d-raytrace", "access-binary-trees",
1828   "access-fannkuch", "access-nbody", "access-nsieve",
1829   "bitops-3bit-bits-in-byte", "bitops-bits-in-byte", "bitops-bitwise-and",
1830   "bitops-nsieve-bits", "controlflow-recursive", "crypto-aes",
1831   "crypto-md5", "crypto-sha1", "date-format-tofte", "date-format-xparb",
1832   "math-cordic", "math-partial-sums", "math-spectral-norm", "regexp-dna",
1833   "string-base64", "string-fasta", "string-tagcloud",
1834   "string-unpack-code", "string-validate-input"].each {
1835    | name |
1836    SUNSPIDER.add SunSpiderBenchmark.new(name)
1837  }
1838
1839  V8 = BenchmarkSuite.new("V8", V8_PATH, :geometricMean)
1840  ["crypto", "deltablue", "earley-boyer", "raytrace",
1841   "regexp", "richards", "splay"].each {
1842    | name |
1843    V8.add V8Benchmark.new(name)
1844  }
1845
1846  KRAKEN = BenchmarkSuite.new("Kraken", KRAKEN_PATH, :arithmeticMean)
1847  ["ai-astar", "audio-beat-detection", "audio-dft", "audio-fft",
1848   "audio-oscillator", "imaging-darkroom", "imaging-desaturate",
1849   "imaging-gaussian-blur", "json-parse-financial",
1850   "json-stringify-tinderbox", "stanford-crypto-aes",
1851   "stanford-crypto-ccm", "stanford-crypto-pbkdf2",
1852   "stanford-crypto-sha256-iterative"].each {
1853    | name |
1854    KRAKEN.add KrakenBenchmark.new(name)
1855  }
1856
1857  ARGV.each {
1858    | vm |
1859    if vm =~ /([a-zA-Z0-9_ ]+):/
1860      name = $1
1861      nameKind = :given
1862      vm = $~.post_match
1863    else
1864      name = "Conf\##{$vms.length+1}"
1865      nameKind = :auto
1866    end
1867    $stderr.puts "#{name}: #{vm}" if $verbosity >= 1
1868    $vms << VM.new(Pathname.new(vm).realpath, name, nameKind, nil)
1869  }
1870
1871  if $vms.empty?
1872    quickFail("Please specify at least on configuraiton on the command line.",
1873              "Insufficient arguments")
1874  end
1875
1876  $vms.each {
1877    | vm |
1878    if vm.vmType != :jsc and $timeMode != :date
1879      $timeMode = :date
1880      $stderr.puts "Warning: using Date.now() instead of preciseTime() because #{vm} doesn't support the latter."
1881    end
1882  }
1883
1884  if FileTest.exist? BENCH_DATA_PATH
1885    cmd = "rm -rf #{BENCH_DATA_PATH}"
1886    $stderr.puts ">> #{cmd}" if $verbosity >= 2
1887    raise unless system cmd
1888  end
1889
1890  Dir.mkdir BENCH_DATA_PATH
1891
1892  if $needToCopyVMs
1893    canCopyIntoBenchPath = true
1894    $vms.each {
1895      | vm |
1896      canCopyIntoBenchPath = false unless vm.canCopyIntoBenchPath
1897    }
1898
1899    if canCopyIntoBenchPath
1900      $vms.each {
1901        | vm |
1902        $stderr.puts "Copying #{vm} into #{BENCH_DATA_PATH}..."
1903        vm.copyIntoBenchPath
1904      }
1905      $stderr.puts "All VMs are in place."
1906    else
1907      $stderr.puts "Warning: don't know how to copy some VMs into #{BENCH_DATA_PATH}, so I won't do it."
1908    end
1909  end
1910
1911  if $measureGC and $measureGC != true
1912    found = false
1913    $vms.each {
1914      | vm |
1915      if vm.name == $measureGC
1916        found = true
1917      end
1918    }
1919    unless found
1920      $stderr.puts "Warning: --measure-gc option ignored because no VM is named #{$measureGC}"
1921    end
1922  end
1923
1924  if $outer*$inner == 1
1925    $stderr.puts "Warning: will only collect one sample per benchmark/VM.  Confidence interval calculation will fail."
1926  end
1927
1928  $stderr.puts "Using timeMode = #{$timeMode}." if $verbosity >= 1
1929
1930  $suites = []
1931
1932  if $includeSunSpider and not SUNSPIDER.empty?
1933    $suites << SUNSPIDER
1934  end
1935
1936  if $includeV8 and not V8.empty?
1937    $suites << V8
1938  end
1939
1940  if $includeKraken and not KRAKEN.empty?
1941    $suites << KRAKEN
1942  end
1943
1944  $benchmarks = []
1945  $suites.each {
1946    | suite |
1947    $benchmarks += suite.benchmarks
1948  }
1949
1950  $runPlans = []
1951  $vms.each {
1952    | vm |
1953    $benchmarks.each {
1954      | benchmark |
1955      $outer.times {
1956        | iteration |
1957        $runPlans << BenchRunPlan.new(benchmark, vm, iteration)
1958      }
1959    }
1960  }
1961
1962  $runPlans.shuffle!
1963
1964  $suitepad = $suites.collect {
1965    | suite |
1966    suite.to_s.size
1967  }.max + 1
1968
1969  $benchpad = ($benchmarks +
1970               ["<arithmetic> *", "<geometric> *", "<harmonic> *"]).collect {
1971    | benchmark |
1972    if benchmark.respond_to? :name
1973      benchmark.name.size
1974    else
1975      benchmark.size
1976    end
1977  }.max + 1
1978
1979  $vmpad = $vms.collect {
1980    | vm |
1981    vm.to_s.size
1982  }.max + 1
1983
1984  if $prepare
1985    File.open("#{BENCH_DATA_PATH}/runscript", "w") {
1986      | file |
1987      file.puts "echo \"HOSTNAME:\\c\""
1988      file.puts "hostname"
1989      file.puts "echo"
1990      file.puts "echo \"HARDWARE:\\c\""
1991      file.puts "/usr/sbin/sysctl hw.model"
1992      file.puts "echo"
1993      file.puts "set -e"
1994      $script = file
1995      $runPlans.each_with_index {
1996        | plan, idx |
1997        if $verbosity == 0 and not $silent
1998          text1 = lpad(idx.to_s,$runPlans.size.to_s.size)+"/"+$runPlans.size.to_s
1999          text2 = plan.benchmark.to_s+"/"+plan.vm.to_s
2000          file.puts("echo "+("\r#{text1} #{rpad(text2,$suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2")
2001          file.puts("echo "+("\r#{text1} #{text2}".inspect)[0..-2]+"\\c\" 1>&2")
2002        end
2003        plan.emitRunCode
2004      }
2005      if $verbosity == 0 and not $silent
2006        file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size} #{' '*($suitepad+1+$benchpad+1+$vmpad)}".inspect)[0..-2]+"\\c\" 1>&2")
2007        file.puts("echo "+("\r#{$runPlans.size}/#{$runPlans.size}".inspect)+" 1>&2")
2008      end
2009    }
2010  end
2011
2012  if $run
2013    unless $remoteHosts.empty?
2014      $stderr.puts "Packaging benchmarking directory for remote hosts..." if $verbosity==0
2015      Dir.chdir(TEMP_PATH) {
2016        cmd = "tar -czf payload.tar.gz benchdata"
2017        $stderr.puts ">> #{cmd}" if $verbosity>=2
2018        raise unless system(cmd)
2019      }
2020
2021      def grokHost(host)
2022        if host =~ /:([0-9]+)$/
2023          "-p " + $1 + " " + $~.pre_match.inspect
2024        else
2025          host.inspect
2026        end
2027      end
2028
2029      def sshRead(host, command)
2030        cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}"
2031        $stderr.puts ">> #{cmd}" if $verbosity>=2
2032        result = ""
2033        IO.popen(cmd, "r") {
2034          | inp |
2035          inp.each_line {
2036            | line |
2037            $stderr.puts "#{host}: #{line}" if $verbosity>=2
2038            result += line
2039          }
2040        }
2041        raise "#{$?}" unless $?.success?
2042        result
2043      end
2044
2045      def sshWrite(host, command, data)
2046        cmd = "ssh #{$sshOptions.collect{|x| x.inspect}.join(' ')} #{grokHost(host)} #{command.inspect}"
2047        $stderr.puts ">> #{cmd}" if $verbosity>=2
2048        IO.popen(cmd, "w") {
2049          | outp |
2050          outp.write(data)
2051        }
2052        raise "#{$?}" unless $?.success?
2053      end
2054
2055      $remoteHosts.each {
2056        | host |
2057        $stderr.puts "Sending benchmark payload to #{host}..." if $verbosity==0
2058
2059        remoteTempPath = JSON::parse(sshRead(host, "cat ~/.bencher"))["tempPath"]
2060        raise unless remoteTempPath
2061
2062        sshWrite(host, "cd #{remoteTempPath.inspect} && rm -rf benchdata && tar -xz", IO::read("#{TEMP_PATH}/payload.tar.gz"))
2063
2064        $stderr.puts "Running on #{host}..." if $verbosity==0
2065
2066        parseAndDisplayResults(sshRead(host, "cd #{(remoteTempPath+'/benchdata').inspect} && sh runscript"))
2067      }
2068    end
2069
2070    if not $remoteHosts.empty? and $alsoLocal
2071      $stderr.puts "Running locally..."
2072    end
2073
2074    if $remoteHosts.empty? or $alsoLocal
2075      parseAndDisplayResults(runAndGetResults)
2076    end
2077  end
2078
2079  $analyze.each_with_index {
2080    | filename, index |
2081    if index >= 1
2082      puts
2083    end
2084    parseAndDisplayResults(IO::read(filename))
2085  }
2086
2087  if $prepare and not $run and $analyze.empty?
2088    puts wrap("Benchmarking script and data are in #{BENCH_DATA_PATH}. You can run "+
2089              "the benchmarks and get the results by doing:", 78)
2090    puts
2091    puts "cd #{BENCH_DATA_PATH}"
2092    puts "sh runscript > results.txt"
2093    puts
2094    puts wrap("Then you can analyze the results by running bencher with the same arguments "+
2095              "as now, but replacing --prepare-only with --analyze results.txt.", 78)
2096  end
2097rescue => e
2098  fail(e)
2099end
2100
2101
2102