1#!/usr/bin/env python
2# Copyright 2015 the V8 project authors. All rights reserved.
3# Use of this source code is governed by a BSD-style license that can be
4# found in the LICENSE file.
5'''
6python %prog
7
8Convert a perf trybot JSON file into a pleasing HTML page. It can read
9from standard input or via the --filename option. Examples:
10
11  cat results.json | %prog --title "ia32 results"
12  %prog -f results.json -t "ia32 results" -o results.html
13'''
14
15import commands
16import json
17import math
18from optparse import OptionParser
19import os
20import shutil
21import sys
22import tempfile
23
24PERCENT_CONSIDERED_SIGNIFICANT = 0.5
25PROBABILITY_CONSIDERED_SIGNIFICANT = 0.02
26PROBABILITY_CONSIDERED_MEANINGLESS = 0.05
27
28
29def ComputeZ(baseline_avg, baseline_sigma, mean, n):
30  if baseline_sigma == 0:
31    return 1000.0;
32  return abs((mean - baseline_avg) / (baseline_sigma / math.sqrt(n)))
33
34
35# Values from http://www.fourmilab.ch/rpkp/experiments/analysis/zCalc.html
36def ComputeProbability(z):
37  if z > 2.575829: # p 0.005: two sided < 0.01
38    return 0
39  if z > 2.326348: # p 0.010
40    return 0.01
41  if z > 2.170091: # p 0.015
42    return 0.02
43  if z > 2.053749: # p 0.020
44    return 0.03
45  if z > 1.959964: # p 0.025: two sided < 0.05
46    return 0.04
47  if z > 1.880793: # p 0.030
48    return 0.05
49  if z > 1.811910: # p 0.035
50    return 0.06
51  if z > 1.750686: # p 0.040
52    return 0.07
53  if z > 1.695397: # p 0.045
54    return 0.08
55  if z > 1.644853: # p 0.050: two sided < 0.10
56    return 0.09
57  if z > 1.281551: # p 0.100: two sided < 0.20
58    return 0.10
59  return 0.20 # two sided p >= 0.20
60
61
62class Result:
63  def __init__(self, test_name, count, hasScoreUnits, result, sigma,
64               master_result, master_sigma):
65    self.result_ = float(result)
66    self.sigma_ = float(sigma)
67    self.master_result_ = float(master_result)
68    self.master_sigma_ = float(master_sigma)
69    self.significant_ = False
70    self.notable_ = 0
71    self.percentage_string_ = ""
72    # compute notability and significance.
73    if hasScoreUnits:
74      compare_num = 100*self.result_/self.master_result_ - 100
75    else:
76      compare_num = 100*self.master_result_/self.result_ - 100
77    if abs(compare_num) > 0.1:
78      self.percentage_string_ = "%3.1f" % (compare_num)
79      z = ComputeZ(self.master_result_, self.master_sigma_, self.result_, count)
80      p = ComputeProbability(z)
81      if p < PROBABILITY_CONSIDERED_SIGNIFICANT:
82        self.significant_ = True
83      if compare_num >= PERCENT_CONSIDERED_SIGNIFICANT:
84        self.notable_ = 1
85      elif compare_num <= -PERCENT_CONSIDERED_SIGNIFICANT:
86        self.notable_ = -1
87
88  def result(self):
89    return self.result_
90
91  def sigma(self):
92    return self.sigma_
93
94  def master_result(self):
95    return self.master_result_
96
97  def master_sigma(self):
98    return self.master_sigma_
99
100  def percentage_string(self):
101    return self.percentage_string_;
102
103  def isSignificant(self):
104    return self.significant_
105
106  def isNotablyPositive(self):
107    return self.notable_ > 0
108
109  def isNotablyNegative(self):
110    return self.notable_ < 0
111
112
113class Benchmark:
114  def __init__(self, name, data):
115    self.name_ = name
116    self.tests_ = {}
117    for test in data:
118      # strip off "<name>/" prefix, allowing for subsequent "/"s
119      test_name = test.split("/", 1)[1]
120      self.appendResult(test_name, data[test])
121
122  # tests is a dictionary of Results
123  def tests(self):
124    return self.tests_
125
126  def SortedTestKeys(self):
127    keys = self.tests_.keys()
128    keys.sort()
129    t = "Total"
130    if t in keys:
131      keys.remove(t)
132      keys.append(t)
133    return keys
134
135  def name(self):
136    return self.name_
137
138  def appendResult(self, test_name, test_data):
139    with_string = test_data["result with patch   "]
140    data = with_string.split()
141    master_string = test_data["result without patch"]
142    master_data = master_string.split()
143    runs = int(test_data["runs"])
144    units = test_data["units"]
145    hasScoreUnits = units == "score"
146    self.tests_[test_name] = Result(test_name,
147                                    runs,
148                                    hasScoreUnits,
149                                    data[0], data[2],
150                                    master_data[0], master_data[2])
151
152
153class BenchmarkRenderer:
154  def __init__(self, output_file):
155    self.print_output_ = []
156    self.output_file_ = output_file
157
158  def Print(self, str_data):
159    self.print_output_.append(str_data)
160
161  def FlushOutput(self):
162    string_data = "\n".join(self.print_output_)
163    print_output = []
164    if self.output_file_:
165      # create a file
166      with open(self.output_file_, "w") as text_file:
167        text_file.write(string_data)
168    else:
169      print(string_data)
170
171  def RenderOneBenchmark(self, benchmark):
172    self.Print("<h2>")
173    self.Print("<a name=\"" + benchmark.name() + "\">")
174    self.Print(benchmark.name() + "</a> <a href=\"#top\">(top)</a>")
175    self.Print("</h2>");
176    self.Print("<table class=\"benchmark\">")
177    self.Print("<thead>")
178    self.Print("  <th>Test</th>")
179    self.Print("  <th>Result</th>")
180    self.Print("  <th>Master</th>")
181    self.Print("  <th>%</th>")
182    self.Print("</thead>")
183    self.Print("<tbody>")
184    tests = benchmark.tests()
185    for test in benchmark.SortedTestKeys():
186      t = tests[test]
187      self.Print("  <tr>")
188      self.Print("    <td>" + test + "</td>")
189      self.Print("    <td>" + str(t.result()) + "</td>")
190      self.Print("    <td>" + str(t.master_result()) + "</td>")
191      t = tests[test]
192      res = t.percentage_string()
193      if t.isSignificant():
194        res = self.bold(res)
195      if t.isNotablyPositive():
196        res = self.green(res)
197      elif t.isNotablyNegative():
198        res = self.red(res)
199      self.Print("    <td>" + res + "</td>")
200      self.Print("  </tr>")
201    self.Print("</tbody>")
202    self.Print("</table>")
203
204  def ProcessJSONData(self, data, title):
205    self.Print("<h1>" + title + "</h1>")
206    self.Print("<ul>")
207    for benchmark in data:
208     if benchmark != "errors":
209       self.Print("<li><a href=\"#" + benchmark + "\">" + benchmark + "</a></li>")
210    self.Print("</ul>")
211    for benchmark in data:
212      if benchmark != "errors":
213        benchmark_object = Benchmark(benchmark, data[benchmark])
214        self.RenderOneBenchmark(benchmark_object)
215
216  def bold(self, data):
217    return "<b>" + data + "</b>"
218
219  def red(self, data):
220    return "<font color=\"red\">" + data + "</font>"
221
222
223  def green(self, data):
224    return "<font color=\"green\">" + data + "</font>"
225
226  def PrintHeader(self):
227    data = """<html>
228<head>
229<title>Output</title>
230<style type="text/css">
231/*
232Style inspired by Andy Ferra's gist at https://gist.github.com/andyferra/2554919
233*/
234body {
235  font-family: Helvetica, arial, sans-serif;
236  font-size: 14px;
237  line-height: 1.6;
238  padding-top: 10px;
239  padding-bottom: 10px;
240  background-color: white;
241  padding: 30px;
242}
243h1, h2, h3, h4, h5, h6 {
244  margin: 20px 0 10px;
245  padding: 0;
246  font-weight: bold;
247  -webkit-font-smoothing: antialiased;
248  cursor: text;
249  position: relative;
250}
251h1 {
252  font-size: 28px;
253  color: black;
254}
255
256h2 {
257  font-size: 24px;
258  border-bottom: 1px solid #cccccc;
259  color: black;
260}
261
262h3 {
263  font-size: 18px;
264}
265
266h4 {
267  font-size: 16px;
268}
269
270h5 {
271  font-size: 14px;
272}
273
274h6 {
275  color: #777777;
276  font-size: 14px;
277}
278
279p, blockquote, ul, ol, dl, li, table, pre {
280  margin: 15px 0;
281}
282
283li p.first {
284  display: inline-block;
285}
286
287ul, ol {
288  padding-left: 30px;
289}
290
291ul :first-child, ol :first-child {
292  margin-top: 0;
293}
294
295ul :last-child, ol :last-child {
296  margin-bottom: 0;
297}
298
299table {
300  padding: 0;
301}
302
303table tr {
304  border-top: 1px solid #cccccc;
305  background-color: white;
306  margin: 0;
307  padding: 0;
308}
309
310table tr:nth-child(2n) {
311  background-color: #f8f8f8;
312}
313
314table tr th {
315  font-weight: bold;
316  border: 1px solid #cccccc;
317  text-align: left;
318  margin: 0;
319  padding: 6px 13px;
320}
321table tr td {
322  border: 1px solid #cccccc;
323  text-align: left;
324  margin: 0;
325  padding: 6px 13px;
326}
327table tr th :first-child, table tr td :first-child {
328  margin-top: 0;
329}
330table tr th :last-child, table tr td :last-child {
331  margin-bottom: 0;
332}
333</style>
334</head>
335<body>
336"""
337    self.Print(data)
338
339  def PrintFooter(self):
340    data = """</body>
341</html>
342"""
343    self.Print(data)
344
345
346def Render(opts, args):
347  if opts.filename:
348    with open(opts.filename) as json_data:
349      data = json.load(json_data)
350  else:
351    # load data from stdin
352    data = json.load(sys.stdin)
353
354  if opts.title:
355    title = opts.title
356  elif opts.filename:
357    title = opts.filename
358  else:
359    title = "Benchmark results"
360  renderer = BenchmarkRenderer(opts.output)
361  renderer.PrintHeader()
362  renderer.ProcessJSONData(data, title)
363  renderer.PrintFooter()
364  renderer.FlushOutput()
365
366
367if __name__ == '__main__':
368  parser = OptionParser(usage=__doc__)
369  parser.add_option("-f", "--filename", dest="filename",
370                    help="Specifies the filename for the JSON results "
371                         "rather than reading from stdin.")
372  parser.add_option("-t", "--title", dest="title",
373                    help="Optional title of the web page.")
374  parser.add_option("-o", "--output", dest="output",
375                    help="Write html output to this file rather than stdout.")
376
377  (opts, args) = parser.parse_args()
378  Render(opts, args)
379