peacekeeper.py revision f2477e01787aa58f445919b809d89e252beef54f 
1# Copyright 2013 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5"""PeaceKeeper benchmark suite.
6
7Peacekeeper measures browser's performance by testing its JavaScript
8functionality. JavaScript is a widely used programming language used in the
9creation of modern websites to provide features such as animation, navigation,
10forms and other common requirements. By measuring a browser's ability to handle
11commonly used JavaScript functions Peacekeeper can evaluate its performance.
12Peacekeeper scores are measured in operations per second or rendered frames per
13second depending on the test. Final Score is computed by calculating geometric
14mean of individual tests scores.
15"""
16
17import os
18
19from metrics import statistics
20from telemetry import test
21from telemetry.page import page_measurement
22from telemetry.page import page_set
23from telemetry.value import merge_values
24
25class PeaceKeeperMeasurement(page_measurement.PageMeasurement):
26
27  def WillNavigateToPage(self, page, tab):
28    page.script_to_evaluate_on_commit = """
29        var __results = {};
30        var _done = false;
31        var __real_log = window.console.log;
32        var test_frame = null;
33        var benchmark = null;
34        window.console.log = function(msg) {
35          if (typeof(msg) == "string" && (msg.indexOf("benchmark")) == 0) {
36            test_frame = document.getElementById("testFrame");
37            benchmark =  test_frame.contentWindow.benchmark;
38            test_frame.contentWindow.onbeforeunload = {};
39            if ((msg.indexOf("Submit ok.")) != -1) {
40              _done = true;
41              __results["test"] = benchmark.testObjectName;
42              __results["score"] = benchmark.test.result;
43              if (typeof(benchmark.test.unit) != "undefined") {
44                __results["unit"] = benchmark.test.unit;
45              } else {
46                __results["unit"] = benchmark.test.isFps ? "fps" : "ops";
47              }
48            }
49          }
50          __real_log.apply(this, [msg]);
51        }
52        """
53
54  def MeasurePage(self, _, tab, results):
55    tab.WaitForJavaScriptExpression('_done', 600)
56    result = tab.EvaluateJavaScript('__results')
57
58    results.Add('Score', 'score', int(result['score']), result['test'],
59                'unimportant')
60
61  def DidRunTest(self, browser, results):
62    # Calculate geometric mean as the total for the combined tests.
63    combined = merge_values.MergeLikeValuesFromDifferentPages(
64        results.all_page_specific_values,
65        group_by_name_suffix=True)
66    combined_score = [x for x in combined if x.name == 'Score'][0]
67    total = statistics.GeometricMean(combined_score.values)
68    results.AddSummary('Score', 'score', total, 'Total')
69
70
71class PeaceKeeperBenchmark(test.Test):
72  """A base class for Peackeeper benchmarks."""
73  test = PeaceKeeperMeasurement
74
75  def CreatePageSet(self, options):
76    """Makes a PageSet for PeaceKeeper benchmarks."""
77    # Subclasses are expected to define a class member called query_param.
78    if not hasattr(self, 'test_param'):
79      raise NotImplementedError('test_param not in PeaceKeeper benchmark.')
80
81    # The docstring of benchmark classes may also be used as a description
82    # when 'run_benchmarks list' is run.
83    description = self.__doc__ or 'PeaceKeeper Benchmark'
84    test_urls = []
85    for test_name in self.test_param:
86      test_urls.append(
87        {"url": ("http://peacekeeper.futuremark.com/run.action?debug=true&"
88                 "repeat=false&forceSuiteName=%s&forceTestName=%s") %
89                 (self.tag, test_name)
90        })
91
92    page_set_dict = {
93        'description': description,
94        'archive_data_file': '../page_sets/data/peacekeeper_%s.json' % self.tag,
95        'make_javascript_deterministic': False,
96        'pages': test_urls,
97    }
98    return page_set.PageSet.FromDict(page_set_dict, os.path.abspath(__file__))
99
100
101class PeaceKeeperRender(PeaceKeeperBenchmark):
102  """PeaceKeeper rendering benchmark suite.
103
104  These tests measure your browser's ability to render and modify specific
105  elements used in typical web pages. Rendering tests manipulate the DOM tree in
106  real-time. The tests measure display updating speed (frames per seconds).
107  """
108  tag = 'render'
109  test_param = ['renderGrid01',
110                'renderGrid02',
111                'renderGrid03',
112                'renderPhysics'
113               ]
114
115
116class PeaceKeeperData(PeaceKeeperBenchmark):
117  """PeaceKeeper Data operations benchmark suite.
118
119  These tests measure your browser's ability to add, remove and modify data
120  stored in an array. The Data suite consists of two tests:
121  1. arrayCombined: This test uses all features of the JavaScript Array object.
122  This is a technical test that is not based on profiled data.
123  The source data are different sized arrays of numbers.
124  2. arrayWeighted: This test is similar to 'arrayCombined', but the load is
125  balanced based on profiled data. The source data is a list of all the
126  countries in the world.
127  """
128
129  tag = 'array'
130  test_param = ['arrayCombined01',
131                'arrayWeighted'
132               ]
133
134
135class PeaceKeeperDom(PeaceKeeperBenchmark):
136  """PeaceKeeper DOM operations benchmark suite.
137
138  These tests emulate the methods used to create typical dynamic webpages.
139  The DOM tests are based on development experience and the capabilities of the
140  jQuery framework.
141  1. domGetElements: This test uses native DOM methods getElementById and
142    getElementsByName. The elements are not modified.
143  2. domDynamicCreationCreateElement: A common use of DOM is to dynamically
144    create content with JavaScript, this test measures creating objects
145    individually and then appending them to DOM.
146  3. domDynamicCreationInnerHTML: This test is similarl to the previous one,
147    but uses the innerHTML-method.
148  4. domJQueryAttributeFilters: This test does a DOM query with jQuery.
149    It searches elements with specific attributes.
150  5. domJQueryBasicFilters: This test uses filters to query elements from DOM.
151  6. domJQueryBasics: This test queries elements from DOM with basic methods.
152    It is similar to domGetElements, but uses jQuery rather than native methods.
153  7. domJQueryContentFilters: Query elements based on content. This does string
154    searching and these methods are assumed to be time consuming.
155  8. domJQueryHierarchy: Query elements based on hierarchy, such as getting
156    sibling, parent or child nodes from a DOM tree.
157  9. domQueryselector: QuerySelector, which allows JavaScript to search elements
158    from the DOM tree directly without the need to iterate the whole tree
159    through domGetElements.
160  """
161
162  tag = 'dom'
163  test_param = ['domGetElements',
164                'domDynamicCreationCreateElement',
165                'domDynamicCreationInnerHTML',
166                'domJQueryAttributeFilters',
167                'domJQueryBasicFilters',
168                'domJQueryBasics',
169                'domJQueryContentFilters',
170                'domJQueryHierarchy',
171                'domQueryselector'
172               ]
173
174
175class PeaceKeeperTextParsing(PeaceKeeperBenchmark):
176  """PeaceKeeper Text Parsing benchmark suite.
177
178  These tests measure your browser's performance in typical text manipulations
179  such as using a profanity filter for chats, browser detection and form
180  validation.
181  1. stringChat: This test removes swearing from artificial chat messages.
182    Test measures looping and string replace-method.
183  2. stringDetectBrowser: This test uses string indexOf-method to detect browser
184    and operating system.
185  3. stringFilter: This test filters a list of movies with a given keyword.
186    The behaviour is known as filtering select or continuous filter. It's used
187    to give real time suggestions while a user is filling input fields.
188    The test uses simple regular expressions.
189  4. stringValidateForm: This test uses complex regular expressions to validate
190    user input.
191  5. stringWeighted: This is an artificial test. Methods used and their
192    intensities are chosen based on profiled data.
193  """
194
195  tag = 'string'
196  test_param = ['stringChat',
197                'stringDetectBrowser',
198                'stringFilter',
199                'stringWeighted',
200                'stringValidateForm'
201               ]
202
203
204class PeaceKeeperHTML5Canvas(PeaceKeeperBenchmark):
205  """PeaceKeeper HTML5 Canvas benchmark suite.
206
207  These tests use HTML5 Canvas, which is a web technology for drawing and
208  manipulating graphics without external plug-ins.
209  1. experimentalRipple01: Simulates a 'water ripple' effect by using HTML 5
210    Canvas. It measures the browser's ability to draw individual pixels.
211  2. experimentalRipple02: Same test as 'experimentalRipple01', but with a
212    larger canvas and thus a heavier workload.
213  """
214
215  tag = 'experimental'
216  test_param = ['experimentalRipple01',
217                'experimentalRipple02'
218               ]
219
220
221class PeaceKeeperHTML5Capabilities(PeaceKeeperBenchmark):
222  """PeaceKeeper HTML5 Capabilities benchmark suite.
223
224  These tests checks browser HTML5 capabilities support for WebGL, Video
225  foramts, simple 2D sprite based games and web worker.
226  This benchmark only tests HTML5 capability and thus is not calculate into the
227  overall score.
228  1. HTML5 - WebGL: WebGL allows full blown 3D graphics to be rendered in a
229    browser without the need for any external plug-ins.
230    a) webglSphere
231  2. HTML5 - Video: hese tests find out which HTML5 video formats are supposed
232    by your browser. Peacekeeper only checks if your browser is able to play a
233    specific format, no other valuation is done.
234    a) videoCodecH264
235    b) videoCodecTheora
236    c) videoCodecWebM
237    d) videoPosterSupport
238  3.HTML5 - Web Worker: These tests use HTML5 Web Worker, which allows
239    JavaScript to multhread - ie. the ability to perform multiple actions
240    concurrently.
241    a) workerContrast01
242    b) workerContrast02
243  4. HTML5 - Game: This test simulates a simple 2D, sprite-based game.
244    The test itself is the real game, and what is shown is a recorded play.
245    a) gamingSpitfire
246  """
247
248  tag = 'html5'
249  test_param = ['webglSphere',
250                'gamingSpitfire',
251                'videoCodecH264',
252                'videoCodecTheora',
253                'videoCodecWebM',
254                'videoPosterSupport',
255                'workerContrast01',
256                'workerContrast02'
257                ]
258