1ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Copyright 2014 the V8 project authors. All rights reserved. 2ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Use of this source code is governed by a BSD-style license that can be 3ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// found in the LICENSE file. 4ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 5ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 6ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Performance.now is used in latency benchmarks, the fallback is Date.now. 7ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgvar performance = performance || {}; 8ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgperformance.now = (function() { 9ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return performance.now || 10ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org performance.mozNow || 11ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org performance.msNow || 12ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org performance.oNow || 13ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org performance.webkitNow || 14ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org Date.now; 15ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org})(); 16ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 17ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Simple framework for running the benchmark suites and 18ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// computing a score based on the timing measurements. 19ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 20ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 21ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// A benchmark has a name (string) and a function that will be run to 22ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// do the performance measurement. The optional setup and tearDown 23ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// arguments are functions that will be invoked before and after 24ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// running the benchmark, but the running time of these functions will 25ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// not be accounted for in the benchmark score. 26ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgfunction Benchmark(name, doWarmup, doDeterministic, deterministicIterations, 27ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org run, setup, tearDown, rmsResult, minIterations) { 28ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.name = name; 29ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.doWarmup = doWarmup; 30ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.doDeterministic = doDeterministic; 31ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.deterministicIterations = deterministicIterations; 32ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.run = run; 33ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.Setup = setup ? setup : function() { }; 34ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.TearDown = tearDown ? tearDown : function() { }; 35ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.rmsResult = rmsResult ? rmsResult : null; 36ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.minIterations = minIterations ? minIterations : 32; 37ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 38ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 39ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 40ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Benchmark results hold the benchmark and the measured time used to 41ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// run the benchmark. The benchmark score is computed later once a 42ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// full benchmark suite has run to completion. If latency is set to 0 43ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// then there is no latency score for this benchmark. 44ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgfunction BenchmarkResult(benchmark, time, latency) { 45ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.benchmark = benchmark; 46ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.time = time; 47ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.latency = latency; 48ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 49ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 50ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 51ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Automatically convert results to numbers. Used by the geometric 52ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// mean computation. 53ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkResult.prototype.valueOf = function() { 54ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return this.time; 55ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 56ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 57ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 58ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Suites of benchmarks consist of a name and the set of benchmarks in 59ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// addition to the reference timing that the final score will be based 60ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// on. This way, all scores are relative to a reference run and higher 61ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// scores implies better performance. 62ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgfunction BenchmarkSuite(name, reference, benchmarks) { 63ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.name = name; 64ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.reference = reference; 65ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.benchmarks = benchmarks; 66ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org BenchmarkSuite.suites.push(this); 67ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 68ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 69ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 70ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Keep track of all declared benchmark suites. 71ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.suites = []; 72ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 73ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Scores are not comparable across versions. Bump the version if 74ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// you're making changes that will affect that scores, e.g. if you add 75ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// a new benchmark or change an existing one. 76ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.version = '1'; 77ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 78ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 79ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Defines global benchsuite running mode that overrides benchmark suite 80ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// behavior. Intended to be set by the benchmark driver. Undefined 81ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// values here allow a benchmark to define behaviour itself. 82ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.config = { 83ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org doWarmup: undefined, 84ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org doDeterministic: undefined 85ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org}; 86ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 87ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 88ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Override the alert function to throw an exception instead. 89ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgalert = function(s) { 90ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org throw "Alert called with argument: " + s; 91ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org}; 92ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 93ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 94ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// To make the benchmark results predictable, we replace Math.random 95ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// with a 100% deterministic alternative. 96ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.ResetRNG = function() { 97ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org Math.random = (function() { 98ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var seed = 49734321; 99ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return function() { 100ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org // Robert Jenkins' 32 bit integer hash function. 101ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org seed = ((seed + 0x7ed55d16) + (seed << 12)) & 0xffffffff; 102ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org seed = ((seed ^ 0xc761c23c) ^ (seed >>> 19)) & 0xffffffff; 103ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org seed = ((seed + 0x165667b1) + (seed << 5)) & 0xffffffff; 104ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org seed = ((seed + 0xd3a2646c) ^ (seed << 9)) & 0xffffffff; 105ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org seed = ((seed + 0xfd7046c5) + (seed << 3)) & 0xffffffff; 106ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org seed = ((seed ^ 0xb55a4f09) ^ (seed >>> 16)) & 0xffffffff; 107ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return (seed & 0xfffffff) / 0x10000000; 108ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org }; 109ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org })(); 110ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 111ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 112ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 113ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Runs all registered benchmark suites and optionally yields between 114ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// each individual benchmark to avoid running for too long in the 115ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// context of browsers. Once done, the final score is reported to the 116ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// runner. 117ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.RunSuites = function(runner, skipBenchmarks) { 118ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org skipBenchmarks = typeof skipBenchmarks === 'undefined' ? [] : skipBenchmarks; 119ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var continuation = null; 120ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var suites = BenchmarkSuite.suites; 121ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var length = suites.length; 122ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org BenchmarkSuite.scores = []; 123ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var index = 0; 124ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org function RunStep() { 125ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org while (continuation || index < length) { 126ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (continuation) { 127ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org continuation = continuation(); 128ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } else { 129ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var suite = suites[index++]; 130ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (runner.NotifyStart) runner.NotifyStart(suite.name); 131ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (skipBenchmarks.indexOf(suite.name) > -1) { 132ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org suite.NotifySkipped(runner); 133ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } else { 134ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org continuation = suite.RunStep(runner); 135ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 136ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 137ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (continuation && typeof window != 'undefined' && window.setTimeout) { 138ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org window.setTimeout(RunStep, 25); 139ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return; 140ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 141ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 142ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 143ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org // show final result 144ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (runner.NotifyScore) { 145ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var score = BenchmarkSuite.GeometricMean(BenchmarkSuite.scores); 146ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var formatted = BenchmarkSuite.FormatScore(100 * score); 147ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org runner.NotifyScore(formatted); 148ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 149ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 150ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org RunStep(); 151ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 152ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 153ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 154ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Counts the total number of registered benchmarks. Useful for 155ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// showing progress as a percentage. 156ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.CountBenchmarks = function() { 157ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var result = 0; 158ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var suites = BenchmarkSuite.suites; 159ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org for (var i = 0; i < suites.length; i++) { 160ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org result += suites[i].benchmarks.length; 161ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 162ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return result; 163ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 164ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 165ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 166ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Computes the geometric mean of a set of numbers. 167ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.GeometricMean = function(numbers) { 168ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var log = 0; 169ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org for (var i = 0; i < numbers.length; i++) { 170ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org log += Math.log(numbers[i]); 171ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 172ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return Math.pow(Math.E, log / numbers.length); 173ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 174ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 175ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 176ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Computes the geometric mean of a set of throughput time measurements. 177ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.GeometricMeanTime = function(measurements) { 178ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var log = 0; 179ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org for (var i = 0; i < measurements.length; i++) { 180ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org log += Math.log(measurements[i].time); 181ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 182ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return Math.pow(Math.E, log / measurements.length); 183ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 184ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 185ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 186ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Computes the geometric mean of a set of rms measurements. 187ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.GeometricMeanLatency = function(measurements) { 188ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var log = 0; 189ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var hasLatencyResult = false; 190ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org for (var i = 0; i < measurements.length; i++) { 191ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (measurements[i].latency != 0) { 192ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org log += Math.log(measurements[i].latency); 193ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org hasLatencyResult = true; 194ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 195ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 196ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (hasLatencyResult) { 197ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return Math.pow(Math.E, log / measurements.length); 198ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } else { 199ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return 0; 200ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 201ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 202ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 203ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 204ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Converts a score value to a string with at least three significant 205ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// digits. 206ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.FormatScore = function(value) { 207ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (value > 100) { 208ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return value.toFixed(0); 209ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } else { 210ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return value.toPrecision(3); 211ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 212ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 213ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 214ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Notifies the runner that we're done running a single benchmark in 215ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// the benchmark suite. This can be useful to report progress. 216ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.prototype.NotifyStep = function(result) { 217ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.results.push(result); 218ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (this.runner.NotifyStep) this.runner.NotifyStep(result.benchmark.name); 219ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 220ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 221ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 222ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Notifies the runner that we're done with running a suite and that 223ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// we have a result which can be reported to the user if needed. 224ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.prototype.NotifyResult = function() { 225ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var mean = BenchmarkSuite.GeometricMeanTime(this.results); 226ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var score = this.reference[0] / mean; 227ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org BenchmarkSuite.scores.push(score); 228ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (this.runner.NotifyResult) { 229ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var formatted = BenchmarkSuite.FormatScore(100 * score); 230ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.runner.NotifyResult(this.name, formatted); 231ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 232ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (this.reference.length == 2) { 233ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var meanLatency = BenchmarkSuite.GeometricMeanLatency(this.results); 234ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (meanLatency != 0) { 235ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var scoreLatency = this.reference[1] / meanLatency; 236ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org BenchmarkSuite.scores.push(scoreLatency); 237ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (this.runner.NotifyResult) { 238ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var formattedLatency = BenchmarkSuite.FormatScore(100 * scoreLatency) 239ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.runner.NotifyResult(this.name + "Latency", formattedLatency); 240ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 241ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 242ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 243ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 244ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 245ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 246ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.prototype.NotifySkipped = function(runner) { 247ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org BenchmarkSuite.scores.push(1); // push default reference score. 248ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (runner.NotifyResult) { 249ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org runner.NotifyResult(this.name, "Skipped"); 250ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 251ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 252ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 253ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 254ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Notifies the runner that running a benchmark resulted in an error. 255ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.prototype.NotifyError = function(error) { 256ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (this.runner.NotifyError) { 257ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.runner.NotifyError(this.name, error); 258ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 259ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (this.runner.NotifyStep) { 260ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.runner.NotifyStep(this.name); 261ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 262ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 263ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 264ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 265ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// Runs a single benchmark for at least a second and computes the 266ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// average time it takes to run a single iteration. 267ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.prototype.RunSingleBenchmark = function(benchmark, data) { 268ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var config = BenchmarkSuite.config; 269ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var doWarmup = config.doWarmup !== undefined 270ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ? config.doWarmup 271ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org : benchmark.doWarmup; 272ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var doDeterministic = config.doDeterministic !== undefined 273ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org ? config.doDeterministic 274ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org : benchmark.doDeterministic; 275ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 276ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org function Measure(data) { 277ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var elapsed = 0; 278ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var start = new Date(); 279ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 280ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org // Run either for 1 second or for the number of iterations specified 281ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org // by minIterations, depending on the config flag doDeterministic. 282ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org for (var i = 0; (doDeterministic ? 283ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org i<benchmark.deterministicIterations : elapsed < 1000); i++) { 284ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org benchmark.run(); 285ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org elapsed = new Date() - start; 286ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 287ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (data != null) { 288ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org data.runs += i; 289ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org data.elapsed += elapsed; 290ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 291ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 292ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 293ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org // Sets up data in order to skip or not the warmup phase. 294ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (!doWarmup && data == null) { 295ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org data = { runs: 0, elapsed: 0 }; 296ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 297ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 298ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (data == null) { 299ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org Measure(null); 300ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return { runs: 0, elapsed: 0 }; 301ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } else { 302ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org Measure(data); 303ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org // If we've run too few iterations, we continue for another second. 304ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (data.runs < benchmark.minIterations) return data; 305ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var usec = (data.elapsed * 1000) / data.runs; 306ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var rms = (benchmark.rmsResult != null) ? benchmark.rmsResult() : 0; 307ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.NotifyStep(new BenchmarkResult(benchmark, usec, rms)); 308ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return null; 309ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 310ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 311ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 312ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 313ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// This function starts running a suite, but stops between each 314ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// individual benchmark in the suite and returns a continuation 315ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// function which can be invoked to run the next benchmark. Once the 316ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org// last benchmark has been executed, null is returned. 317ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.orgBenchmarkSuite.prototype.RunStep = function(runner) { 318ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org BenchmarkSuite.ResetRNG(); 319ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.results = []; 320ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org this.runner = runner; 321ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var length = this.benchmarks.length; 322ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var index = 0; 323ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var suite = this; 324ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org var data; 325ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 326ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org // Run the setup, the actual benchmark, and the tear down in three 327ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org // separate steps to allow the framework to yield between any of the 328ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org // steps. 329ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 330ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org function RunNextSetup() { 331ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org if (index < length) { 332ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org try { 333ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org suite.benchmarks[index].Setup(); 334ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } catch (e) { 335ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org suite.NotifyError(e); 336ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return null; 337ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 338ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return RunNextBenchmark; 339ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 340ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org suite.NotifyResult(); 341ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return null; 342ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 343ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 344ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org function RunNextBenchmark() { 345ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org try { 346ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org data = suite.RunSingleBenchmark(suite.benchmarks[index], data); 347ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } catch (e) { 348ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org suite.NotifyError(e); 349ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return null; 350ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 351ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org // If data is null, we're done with this benchmark. 352ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return (data == null) ? RunNextTearDown : RunNextBenchmark(); 353ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 354ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 355ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org function RunNextTearDown() { 356ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org try { 357ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org suite.benchmarks[index++].TearDown(); 358ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } catch (e) { 359ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org suite.NotifyError(e); 360ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return null; 361ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 362ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return RunNextSetup; 363ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org } 364ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org 365ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org // Start out running the setup. 366ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org return RunNextSetup(); 367ada3a6017e603965f87fa34f6e2fa60379e8d697machenbach@chromium.org} 368