results_report.py revision df3d7d197f33c19e51ca688bdcc2d750e8ffb484 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7from utils.tabulator import *
8
9from column_chart import ColumnChart
10from results_organizer import ResultOrganizer
11from perf_table import PerfTable
12
13
14class ResultsReport(object):
15  MAX_COLOR_CODE = 255
16  PERF_ROWS = 5
17
18  def __init__(self, experiment):
19    self.experiment = experiment
20    self.benchmark_runs = experiment.benchmark_runs
21    self.labels = experiment.labels
22    self.benchmarks = experiment.benchmarks
23    self.baseline = self.labels[0]
24
25  def _SortByLabel(self, runs):
26    labels = {}
27    for benchmark_run in runs:
28      if benchmark_run.label_name not in labels:
29        labels[benchmark_run.label_name] = []
30      labels[benchmark_run.label_name].append(benchmark_run)
31    return labels
32
33  def GetFullTables(self, perf=False):
34    columns = [Column(RawResult(),
35                      Format()),
36               Column(MinResult(),
37                      Format()),
38               Column(MaxResult(),
39                      Format()),
40               Column(AmeanResult(),
41                      Format()),
42               Column(StdResult(),
43                      Format(), "StdDev"),
44               Column(CoeffVarResult(),
45                      CoeffVarFormat(), "StdDev/Mean"),
46               Column(GmeanRatioResult(),
47                      RatioFormat(), "GmeanSpeedup"),
48               Column(PValueResult(),
49                      PValueFormat(), "p-value")
50              ]
51    if not perf:
52      return self._GetTables(self.labels, self.benchmark_runs, columns,
53                             "full")
54    return self._GetPerfTables(self.labels, columns, "full")
55
56  def GetSummaryTables(self, perf=False):
57    columns = [Column(AmeanResult(),
58                      Format()),
59               Column(StdResult(),
60                      Format(), "StdDev"),
61               Column(CoeffVarResult(),
62                      CoeffVarFormat(), "StdDev/Mean"),
63               Column(GmeanRatioResult(),
64                      RatioFormat(), "GmeanSpeedup"),
65               Column(GmeanRatioResult(),
66                      ColorBoxFormat(), " "),
67               Column(PValueResult(),
68                      PValueFormat(), "p-value")
69              ]
70    if not perf:
71      return self._GetTables(self.labels, self.benchmark_runs, columns,
72                             "summary")
73    return self._GetPerfTables(self.labels, columns, "summary")
74
75  def _ParseColumn(self, columns, iteration):
76    new_column = []
77    for column in columns:
78      if column.result.__class__.__name__ != "RawResult":
79      #TODO(asharif): tabulator should support full table natively.
80        new_column.append(column)
81      else:
82        for i in range(iteration):
83          cc = Column(LiteralResult(i), Format(), str(i+1))
84          new_column.append(cc)
85    return new_column
86
87  def _AreAllRunsEmpty(self, runs):
88    for label in runs:
89      for dictionary in label:
90        if dictionary:
91          return False
92    return True
93
94  def _GetTableHeader(self, benchmark):
95    benchmark_info = ("Benchmark:  {0};  Iterations: {1}"
96                      .format(benchmark.name, benchmark.iterations))
97    cell = Cell()
98    cell.string_value = benchmark_info
99    cell.header = True
100    return  [[cell]]
101
102  def _GetTables(self, labels, benchmark_runs, columns, table_type):
103    tables = []
104    ro = ResultOrganizer(benchmark_runs, labels, self.benchmarks)
105    result = ro.result
106    label_name = ro.labels
107    for item in result:
108      runs = result[item]
109      for benchmark in self.benchmarks:
110        if benchmark.name == item:
111          break
112      ben_table = self._GetTableHeader(benchmark)
113
114      if  self._AreAllRunsEmpty(runs):
115        cell = Cell()
116        cell.string_value = ("This benchmark contains no result."
117                             " Is the benchmark name valid?")
118        cell_table = [[cell]]
119      else:
120        tg = TableGenerator(runs, label_name)
121        table = tg.GetTable()
122        parsed_columns = self._ParseColumn(columns, benchmark.iterations)
123        tf = TableFormatter(table, parsed_columns)
124        cell_table = tf.GetCellTable(table_type)
125      tables.append(ben_table)
126      tables.append(cell_table)
127    return tables
128
129  def _GetPerfTables(self, labels, columns, table_type):
130    tables = []
131    label_names = [label.name for label in labels]
132    p_table = PerfTable(self.experiment, label_names)
133
134    if not p_table.perf_data:
135      return tables
136
137    for benchmark in p_table.perf_data:
138      ben = None
139      for ben in self.benchmarks:
140        if ben.name == benchmark:
141          break
142
143      ben_table = self._GetTableHeader(ben)
144      tables.append(ben_table)
145      benchmark_data = p_table.perf_data[benchmark]
146      row_info = p_table.row_info[benchmark]
147      table = []
148      for event in benchmark_data:
149        tg = TableGenerator(benchmark_data[event], label_names,
150                            sort=TableGenerator.SORT_BY_VALUES_DESC)
151        table = tg.GetTable(max(self.PERF_ROWS, row_info[event]))
152        parsed_columns = self._ParseColumn(columns, ben.iterations)
153        tf = TableFormatter(table, parsed_columns)
154        tf.GenerateCellTable()
155        tf.AddColumnName()
156        tf.AddLabelName()
157        tf.AddHeader(str(event))
158        table = tf.GetCellTable(table_type, headers=False)
159        tables.append(table)
160    return tables
161
162  def PrintTables(self, tables, out_to):
163    output = ""
164    if not tables:
165      return output
166    for table in tables:
167      if out_to == "HTML":
168        tp = TablePrinter(table, TablePrinter.HTML)
169      elif out_to == "PLAIN":
170        tp = TablePrinter(table, TablePrinter.PLAIN)
171      elif out_to == "CONSOLE":
172        tp = TablePrinter(table, TablePrinter.CONSOLE)
173      elif out_to == "TSV":
174        tp = TablePrinter(table, TablePrinter.TSV)
175      elif out_to == "EMAIL":
176        tp = TablePrinter(table, TablePrinter.EMAIL)
177      else:
178        pass
179      output += tp.Print()
180    return output
181
182
183class TextResultsReport(ResultsReport):
184  TEXT = """
185===========================================
186Results report for: '%s'
187===========================================
188
189-------------------------------------------
190Summary
191-------------------------------------------
192%s
193
194
195Number re-images: %s
196
197-------------------------------------------
198Benchmark Run Status
199-------------------------------------------
200%s
201
202
203-------------------------------------------
204Perf Data
205-------------------------------------------
206%s
207
208
209
210Experiment File
211-------------------------------------------
212%s
213
214
215CPUInfo
216-------------------------------------------
217%s
218===========================================
219"""
220
221  def __init__(self, experiment, email=False):
222    super(TextResultsReport, self).__init__(experiment)
223    self.email = email
224
225  def GetStatusTable(self):
226    """Generate the status table by the tabulator."""
227    table = [["", ""]]
228    columns = [Column(LiteralResult(iteration=0), Format(), "Status"),
229               Column(LiteralResult(iteration=1), Format(), "Failing Reason")]
230
231    for benchmark_run in self.benchmark_runs:
232      status = [benchmark_run.name, [benchmark_run.timeline.GetLastEvent(),
233                                     benchmark_run.failure_reason]]
234      table.append(status)
235    tf = TableFormatter(table, columns)
236    cell_table = tf.GetCellTable("status")
237    return [cell_table]
238
239  def GetReport(self):
240    """Generate the report for email and console."""
241    status_table = self.GetStatusTable()
242    summary_table = self.GetSummaryTables()
243    full_table = self.GetFullTables()
244    perf_table = self.GetSummaryTables(perf=True)
245    if not perf_table:
246      perf_table = None
247    if not self.email:
248      return self.TEXT % (self.experiment.name,
249                          self.PrintTables(summary_table, "CONSOLE"),
250                          self.experiment.machine_manager.num_reimages,
251                          self.PrintTables(status_table, "CONSOLE"),
252                          self.PrintTables(perf_table, "CONSOLE"),
253                          self.experiment.experiment_file,
254                          self.experiment.machine_manager.GetAllCPUInfo(
255                              self.experiment.labels))
256
257    return self.TEXT % (self.experiment.name,
258                        self.PrintTables(summary_table, "EMAIL"),
259                        self.experiment.machine_manager.num_reimages,
260                        self.PrintTables(status_table, "EMAIL"),
261                        self.PrintTables(perf_table, "EMAIL"),
262                        self.experiment.experiment_file,
263                        self.experiment.machine_manager.GetAllCPUInfo(
264                            self.experiment.labels))
265
266
267class HTMLResultsReport(ResultsReport):
268
269  HTML = """
270<html>
271  <head>
272    <style type="text/css">
273
274body {
275  font-family: "Lucida Sans Unicode", "Lucida Grande", Sans-Serif;
276  font-size: 12px;
277}
278
279pre {
280  margin: 10px;
281  color: #039;
282  font-size: 14px;
283}
284
285.chart {
286  display: inline;
287}
288
289.hidden {
290  visibility: hidden;
291}
292
293.results-section {
294  border: 1px solid #b9c9fe;
295  margin: 10px;
296}
297
298.results-section-title {
299  background-color: #b9c9fe;
300  color: #039;
301  padding: 7px;
302  font-size: 14px;
303  width: 200px;
304}
305
306.results-section-content {
307  margin: 10px;
308  padding: 10px;
309  overflow:auto;
310}
311
312#box-table-a {
313  font-size: 12px;
314  width: 480px;
315  text-align: left;
316  border-collapse: collapse;
317}
318
319#box-table-a th {
320  padding: 6px;
321  background: #b9c9fe;
322  border-right: 1px solid #fff;
323  border-bottom: 1px solid #fff;
324  color: #039;
325  text-align: center;
326}
327
328#box-table-a td {
329  padding: 4px;
330  background: #e8edff;
331  border-bottom: 1px solid #fff;
332  border-right: 1px solid #fff;
333  color: #669;
334  border-top: 1px solid transparent;
335}
336
337#box-table-a tr:hover td {
338  background: #d0dafd;
339  color: #339;
340}
341
342    </style>
343    <script type='text/javascript' src='https://www.google.com/jsapi'></script>
344    <script type='text/javascript'>
345      google.load('visualization', '1', {packages:['corechart']});
346      google.setOnLoadCallback(init);
347      function init() {
348        switchTab('summary', 'html');
349        %s
350        switchTab('full', 'html');
351        drawTable();
352      }
353      function drawTable() {
354        %s
355      }
356      function switchTab(table, tab) {
357        document.getElementById(table + '-html').style.display = 'none';
358        document.getElementById(table + '-text').style.display = 'none';
359        document.getElementById(table + '-tsv').style.display = 'none';
360        document.getElementById(table + '-' + tab).style.display = 'block';
361      }
362    </script>
363  </head>
364
365  <body>
366    <div class='results-section'>
367      <div class='results-section-title'>Summary Table</div>
368      <div class='results-section-content'>
369        <div id='summary-html'>%s</div>
370        <div id='summary-text'><pre>%s</pre></div>
371        <div id='summary-tsv'><pre>%s</pre></div>
372      </div>
373      %s
374    </div>
375    %s
376    <div class='results-section'>
377      <div class='results-section-title'>Charts</div>
378      <div class='results-section-content'>%s</div>
379    </div>
380    <div class='results-section'>
381      <div class='results-section-title'>Full Table</div>
382      <div class='results-section-content'>
383        <div id='full-html'>%s</div>
384        <div id='full-text'><pre>%s</pre></div>
385        <div id='full-tsv'><pre>%s</pre></div>
386      </div>
387      %s
388    </div>
389    <div class='results-section'>
390      <div class='results-section-title'>Experiment File</div>
391      <div class='results-section-content'>
392        <pre>%s</pre>
393    </div>
394    </div>
395  </body>
396</html>
397"""
398
399  PERF_HTML = """
400    <div class='results-section'>
401      <div class='results-section-title'>Perf Table</div>
402      <div class='results-section-content'>
403        <div id='perf-html'>%s</div>
404        <div id='perf-text'><pre>%s</pre></div>
405        <div id='perf-tsv'><pre>%s</pre></div>
406      </div>
407      %s
408    </div>
409"""
410
411  def __init__(self, experiment):
412    super(HTMLResultsReport, self).__init__(experiment)
413
414  def _GetTabMenuHTML(self, table):
415    return """
416<div class='tab-menu'>
417  <a href="javascript:switchTab('%s', 'html')">HTML</a>
418  <a href="javascript:switchTab('%s', 'text')">Text</a>
419  <a href="javascript:switchTab('%s', 'tsv')">TSV</a>
420</div>""" % (table, table, table)
421
422  def GetReport(self):
423    chart_javascript = ""
424    charts = self._GetCharts(self.labels, self.benchmark_runs)
425    for chart in charts:
426      chart_javascript += chart.GetJavascript()
427    chart_divs = ""
428    for chart in charts:
429      chart_divs += chart.GetDiv()
430
431    summary_table = self.GetSummaryTables()
432    full_table = self.GetFullTables()
433    perf_table = self.GetSummaryTables(perf=True)
434    if perf_table:
435      perf_html = self.PERF_HTML % (
436          self.PrintTables(perf_table, "HTML"),
437          self.PrintTables(perf_table, "PLAIN"),
438          self.PrintTables(perf_table, "TSV"),
439          self._GetTabMenuHTML("perf")
440          )
441      perf_init = "switchTab('perf', 'html');"
442    else:
443      perf_html = ""
444      perf_init = ""
445
446    return self.HTML % (perf_init,
447                        chart_javascript,
448                        self.PrintTables(summary_table, "HTML"),
449                        self.PrintTables(summary_table, "PLAIN"),
450                        self.PrintTables(summary_table, "TSV"),
451                        self._GetTabMenuHTML("summary"),
452                        perf_html,
453                        chart_divs,
454                        self.PrintTables(full_table, "HTML"),
455                        self.PrintTables(full_table, "PLAIN"),
456                        self.PrintTables(full_table, "TSV"),
457                        self._GetTabMenuHTML("full"),
458                        self.experiment.experiment_file)
459
460  def _GetCharts(self, labels, benchmark_runs):
461    charts = []
462    ro = ResultOrganizer(benchmark_runs, labels)
463    result = ro.result
464    for item in result:
465      runs = result[item]
466      tg = TableGenerator(runs, ro.labels)
467      table = tg.GetTable()
468      columns = [Column(AmeanResult(),
469                        Format()),
470                 Column(MinResult(),
471                        Format()),
472                 Column(MaxResult(),
473                        Format())
474                ]
475      tf = TableFormatter(table, columns)
476      data_table = tf.GetCellTable("full")
477
478      for i in range(2, len(data_table)):
479        cur_row_data = data_table[i]
480        test_key = cur_row_data[0].string_value
481        title = "{0}: {1}".format(item, test_key.replace("/", ""))
482        chart = ColumnChart(title, 300, 200)
483        chart.AddColumn("Label", "string")
484        chart.AddColumn("Average", "number")
485        chart.AddColumn("Min", "number")
486        chart.AddColumn("Max", "number")
487        chart.AddSeries("Min", "line", "black")
488        chart.AddSeries("Max", "line", "black")
489        cur_index = 1
490        for label in ro.labels:
491          chart.AddRow([label, cur_row_data[cur_index].value,
492                        cur_row_data[cur_index + 1].value,
493                        cur_row_data[cur_index + 2].value])
494          if isinstance(cur_row_data[cur_index].value, str):
495            chart = None
496            break
497          cur_index += 3
498        if chart:
499          charts.append(chart)
500    return charts
501