results_report.py revision 04dc5dc8547dbfbe524cf35ac39537346ad749bb 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7from utils.tabulator import *
8
9from column_chart import ColumnChart
10from results_organizer import ResultOrganizer
11from perf_table import PerfTable
12
13
14class ResultsReport(object):
15  MAX_COLOR_CODE = 255
16  PERF_ROWS = 5
17
18  def __init__(self, experiment):
19    self.experiment = experiment
20    self.benchmark_runs = experiment.benchmark_runs
21    self.labels = experiment.labels
22    self.benchmarks = experiment.benchmarks
23    self.baseline = self.labels[0]
24
25  def _SortByLabel(self, runs):
26    labels = {}
27    for benchmark_run in runs:
28      if benchmark_run.label_name not in labels:
29        labels[benchmark_run.label_name] = []
30      labels[benchmark_run.label_name].append(benchmark_run)
31    return labels
32
33  def GetFullTables(self, perf=False):
34    columns = [Column(NonEmptyCountResult(),
35                      Format(),
36                      "Completed"),
37               Column(RawResult(),
38                      Format()),
39               Column(MinResult(),
40                      Format()),
41               Column(MaxResult(),
42                      Format()),
43               Column(AmeanResult(),
44                      Format()),
45               Column(StdResult(),
46                      Format(), "StdDev"),
47               Column(CoeffVarResult(),
48                      CoeffVarFormat(), "StdDev/Mean"),
49               Column(GmeanRatioResult(),
50                      RatioFormat(), "GmeanSpeedup"),
51               Column(PValueResult(),
52                      PValueFormat(), "p-value")
53              ]
54    if not perf:
55      return self._GetTables(self.labels, self.benchmark_runs, columns)
56    return self. _GetPerfTables(self.labels, columns)
57
58  def GetSummaryTables(self, perf=False):
59    columns = [Column(NonEmptyCountResult(),
60                      Format(),
61                      "Completed"),
62               Column(AmeanResult(),
63                      Format()),
64               Column(StdResult(),
65                      Format(), "StdDev"),
66               Column(CoeffVarResult(),
67                      CoeffVarFormat(), "StdDev/Mean"),
68               Column(GmeanRatioResult(),
69                      RatioFormat(), "GmeanSpeedup"),
70               Column(GmeanRatioResult(),
71                      ColorBoxFormat(), " "),
72               Column(PValueResult(),
73                      PValueFormat(), "p-value")
74              ]
75    if not perf:
76      return self._GetTables(self.labels, self.benchmark_runs, columns)
77    return self. _GetPerfTables(self.labels, columns)
78
79  def _ParseColumn(self, columns, iteration):
80    new_column = []
81    for column in columns:
82      if column.result.__class__.__name__ != "RawResult":
83      #TODO(asharif): tabulator should support full table natively.
84        new_column.append(column)
85      else:
86        for i in range(iteration):
87          cc = Column(LiteralResult(i), Format(), str(i+1))
88          new_column.append(cc)
89    return new_column
90
91  def _AreAllRunsEmpty(self, runs):
92    for label in runs:
93      for dictionary in label:
94        if dictionary:
95          return False
96    return True
97
98  def _GetTableHeader(self, benchmark):
99    benchmark_info = ("Benchmark:  {0};  Iterations: {1}"
100                      .format(benchmark.name, benchmark.iterations))
101    cell = Cell()
102    cell.string_value = benchmark_info
103    cell.header = True
104    return  [[cell]]
105
106  def _GetTables(self, labels, benchmark_runs, columns):
107    tables = []
108    ro = ResultOrganizer(benchmark_runs, labels, self.benchmarks)
109    result = ro.result
110    label_name = ro.labels
111    for item in result:
112      runs = result[item]
113      for benchmark in self.benchmarks:
114        if benchmark.name == item:
115          break
116      ben_table = self._GetTableHeader(benchmark)
117
118      if  self._AreAllRunsEmpty(runs):
119        cell = Cell()
120        cell.string_value = ("This benchmark contains no result."
121                             " Is the benchmark name valid?")
122        cell_table = [[cell]]
123      else:
124        tg = TableGenerator(runs, label_name)
125        table = tg.GetTable()
126        parsed_columns = self._ParseColumn(columns, benchmark.iterations)
127        tf = TableFormatter(table, parsed_columns)
128        cell_table = tf.GetCellTable()
129      tables.append(ben_table)
130      tables.append(cell_table)
131    return tables
132
133  def _GetPerfTables(self, labels, columns):
134    tables = []
135    label_names = [label.name for label in labels]
136    p_table = PerfTable(self.experiment, label_names)
137
138    if not p_table.perf_data:
139      return tables
140
141    for benchmark in p_table.perf_data:
142      ben = None
143      for ben in self.benchmarks:
144        if ben.name == benchmark:
145          break
146
147      ben_table = self._GetTableHeader(ben)
148      tables.append(ben_table)
149      benchmark_data = p_table.perf_data[benchmark]
150      row_info = p_table.row_info[benchmark]
151      table = []
152      for event in benchmark_data:
153        tg = TableGenerator(benchmark_data[event], label_names,
154                            sort=TableGenerator.SORT_BY_VALUES_DESC)
155        table = tg.GetTable(max(self.PERF_ROWS, row_info[event]))
156        parsed_columns = self._ParseColumn(columns, ben.iterations)
157        tf = TableFormatter(table, parsed_columns)
158        tf.GenerateCellTable()
159        tf.AddColumnName()
160        tf.AddLabelName()
161        tf.AddHeader(str(event))
162        table = tf.GetCellTable(headers=False)
163        tables.append(table)
164    return tables
165
166  def PrintTables(self, tables, out_to):
167    output = ""
168    if not tables:
169      return output
170    for table in tables:
171      if out_to == "HTML":
172        tp = TablePrinter(table, TablePrinter.HTML)
173      elif out_to == "PLAIN":
174        tp = TablePrinter(table, TablePrinter.PLAIN)
175      elif out_to == "CONSOLE":
176        tp = TablePrinter(table, TablePrinter.CONSOLE)
177      elif out_to == "TSV":
178        tp = TablePrinter(table, TablePrinter.TSV)
179      elif out_to == "EMAIL":
180        tp = TablePrinter(table, TablePrinter.EMAIL)
181      else:
182        pass
183      output += tp.Print()
184    return output
185
186
187class TextResultsReport(ResultsReport):
188  TEXT = """
189===========================================
190Results report for: '%s'
191===========================================
192
193-------------------------------------------
194Summary
195-------------------------------------------
196%s
197
198
199Number re-images: %s
200
201-------------------------------------------
202Benchmark Run Status
203-------------------------------------------
204%s
205
206
207-------------------------------------------
208Perf Data
209-------------------------------------------
210%s
211
212
213
214Experiment File
215-------------------------------------------
216%s
217
218
219CPUInfo
220-------------------------------------------
221%s
222===========================================
223"""
224
225  def __init__(self, experiment, email=False):
226    super(TextResultsReport, self).__init__(experiment)
227    self.email = email
228
229  def GetStatusTable(self):
230    """Generate the status table by the tabulator."""
231    table = [["", ""]]
232    columns = [Column(LiteralResult(iteration=0), Format(), "Status"),
233               Column(LiteralResult(iteration=1), Format(), "Failing Reason")]
234
235    for benchmark_run in self.benchmark_runs:
236      status = [benchmark_run.name, [benchmark_run.timeline.GetLastEvent(),
237                                     benchmark_run.failure_reason]]
238      table.append(status)
239    tf = TableFormatter(table, columns)
240    cell_table = tf.GetCellTable()
241    return [cell_table]
242
243  def GetReport(self):
244    """Generate the report for email and console."""
245    status_table = self.GetStatusTable()
246    summary_table = self.GetSummaryTables()
247    full_table = self.GetFullTables()
248    perf_table = self.GetSummaryTables(perf=True)
249    if not perf_table:
250      perf_table = None
251    if not self.email:
252      return self.TEXT % (self.experiment.name,
253                          self.PrintTables(summary_table, "CONSOLE"),
254                          self.experiment.machine_manager.num_reimages,
255                          self.PrintTables(status_table, "CONSOLE"),
256                          self.PrintTables(perf_table, "CONSOLE"),
257                          self.experiment.experiment_file,
258                          self.experiment.machine_manager.GetAllCPUInfo(
259                              self.experiment.labels))
260
261    return self.TEXT % (self.experiment.name,
262                        self.PrintTables(summary_table, "EMAIL"),
263                        self.experiment.machine_manager.num_reimages,
264                        self.PrintTables(status_table, "EMAIL"),
265                        self.PrintTables(perf_table, "EMAIL"),
266                        self.experiment.experiment_file,
267                        self.experiment.machine_manager.GetAllCPUInfo(
268                            self.experiment.labels))
269
270
271class HTMLResultsReport(ResultsReport):
272
273  HTML = """
274<html>
275  <head>
276    <style type="text/css">
277
278body {
279  font-family: "Lucida Sans Unicode", "Lucida Grande", Sans-Serif;
280  font-size: 12px;
281}
282
283pre {
284  margin: 10px;
285  color: #039;
286  font-size: 14px;
287}
288
289.chart {
290  display: inline;
291}
292
293.hidden {
294  visibility: hidden;
295}
296
297.results-section {
298  border: 1px solid #b9c9fe;
299  margin: 10px;
300}
301
302.results-section-title {
303  background-color: #b9c9fe;
304  color: #039;
305  padding: 7px;
306  font-size: 14px;
307  width: 200px;
308}
309
310.results-section-content {
311  margin: 10px;
312  padding: 10px;
313  overflow:auto;
314}
315
316#box-table-a {
317  font-size: 12px;
318  width: 480px;
319  text-align: left;
320  border-collapse: collapse;
321}
322
323#box-table-a th {
324  padding: 6px;
325  background: #b9c9fe;
326  border-right: 1px solid #fff;
327  border-bottom: 1px solid #fff;
328  color: #039;
329  text-align: center;
330}
331
332#box-table-a td {
333  padding: 4px;
334  background: #e8edff;
335  border-bottom: 1px solid #fff;
336  border-right: 1px solid #fff;
337  color: #669;
338  border-top: 1px solid transparent;
339}
340
341#box-table-a tr:hover td {
342  background: #d0dafd;
343  color: #339;
344}
345
346    </style>
347    <script type='text/javascript' src='https://www.google.com/jsapi'></script>
348    <script type='text/javascript'>
349      google.load('visualization', '1', {packages:['corechart']});
350      google.setOnLoadCallback(init);
351      function init() {
352        switchTab('summary', 'html');
353        %s
354        switchTab('full', 'html');
355        drawTable();
356      }
357      function drawTable() {
358        %s
359      }
360      function switchTab(table, tab) {
361        document.getElementById(table + '-html').style.display = 'none';
362        document.getElementById(table + '-text').style.display = 'none';
363        document.getElementById(table + '-tsv').style.display = 'none';
364        document.getElementById(table + '-' + tab).style.display = 'block';
365      }
366    </script>
367  </head>
368
369  <body>
370    <div class='results-section'>
371      <div class='results-section-title'>Summary Table</div>
372      <div class='results-section-content'>
373        <div id='summary-html'>%s</div>
374        <div id='summary-text'><pre>%s</pre></div>
375        <div id='summary-tsv'><pre>%s</pre></div>
376      </div>
377      %s
378    </div>
379    %s
380    <div class='results-section'>
381      <div class='results-section-title'>Charts</div>
382      <div class='results-section-content'>%s</div>
383    </div>
384    <div class='results-section'>
385      <div class='results-section-title'>Full Table</div>
386      <div class='results-section-content'>
387        <div id='full-html'>%s</div>
388        <div id='full-text'><pre>%s</pre></div>
389        <div id='full-tsv'><pre>%s</pre></div>
390      </div>
391      %s
392    </div>
393    <div class='results-section'>
394      <div class='results-section-title'>Experiment File</div>
395      <div class='results-section-content'>
396        <pre>%s</pre>
397    </div>
398    </div>
399  </body>
400</html>
401"""
402
403  PERF_HTML = """
404    <div class='results-section'>
405      <div class='results-section-title'>Perf Table</div>
406      <div class='results-section-content'>
407        <div id='perf-html'>%s</div>
408        <div id='perf-text'><pre>%s</pre></div>
409        <div id='perf-tsv'><pre>%s</pre></div>
410      </div>
411      %s
412    </div>
413"""
414
415  def __init__(self, experiment):
416    super(HTMLResultsReport, self).__init__(experiment)
417
418  def _GetTabMenuHTML(self, table):
419    return """
420<div class='tab-menu'>
421  <a href="javascript:switchTab('%s', 'html')">HTML</a>
422  <a href="javascript:switchTab('%s', 'text')">Text</a>
423  <a href="javascript:switchTab('%s', 'tsv')">TSV</a>
424</div>""" % (table, table, table)
425
426  def GetReport(self):
427    chart_javascript = ""
428    charts = self._GetCharts(self.labels, self.benchmark_runs)
429    for chart in charts:
430      chart_javascript += chart.GetJavascript()
431    chart_divs = ""
432    for chart in charts:
433      chart_divs += chart.GetDiv()
434
435    summary_table = self.GetSummaryTables()
436    full_table = self.GetFullTables()
437    perf_table = self.GetSummaryTables(perf=True)
438    if perf_table:
439      perf_html = self.PERF_HTML % (
440          self.PrintTables(perf_table, "HTML"),
441          self.PrintTables(perf_table, "PLAIN"),
442          self.PrintTables(perf_table, "TSV"),
443          self._GetTabMenuHTML("perf")
444          )
445      perf_init = "switchTab('perf', 'html');"
446    else:
447      perf_html = ""
448      perf_init = ""
449
450    return self.HTML % (perf_init,
451                        chart_javascript,
452                        self.PrintTables(summary_table, "HTML"),
453                        self.PrintTables(summary_table, "PLAIN"),
454                        self.PrintTables(summary_table, "TSV"),
455                        self._GetTabMenuHTML("summary"),
456                        perf_html,
457                        chart_divs,
458                        self.PrintTables(full_table, "HTML"),
459                        self.PrintTables(full_table, "PLAIN"),
460                        self.PrintTables(full_table, "TSV"),
461                        self._GetTabMenuHTML("full"),
462                        self.experiment.experiment_file)
463
464  def _GetCharts(self, labels, benchmark_runs):
465    charts = []
466    ro = ResultOrganizer(benchmark_runs, labels)
467    result = ro.result
468    for item in result:
469      runs = result[item]
470      tg = TableGenerator(runs, ro.labels)
471      table = tg.GetTable()
472      columns = [Column(AmeanResult(),
473                        Format()),
474                 Column(MinResult(),
475                        Format()),
476                 Column(MaxResult(),
477                        Format())
478                ]
479      tf = TableFormatter(table, columns)
480      data_table = tf.GetCellTable()
481
482      for i in range(2, len(data_table)):
483        cur_row_data = data_table[i]
484        test_key = cur_row_data[0].string_value
485        title = "{0}: {1}".format(item, test_key.replace("/", ""))
486        chart = ColumnChart(title, 300, 200)
487        chart.AddColumn("Label", "string")
488        chart.AddColumn("Average", "number")
489        chart.AddColumn("Min", "number")
490        chart.AddColumn("Max", "number")
491        chart.AddSeries("Min", "line", "black")
492        chart.AddSeries("Max", "line", "black")
493        cur_index = 1
494        for label in ro.labels:
495          chart.AddRow([label, cur_row_data[cur_index].value,
496                        cur_row_data[cur_index + 1].value,
497                        cur_row_data[cur_index + 2].value])
498          if isinstance(cur_row_data[cur_index].value, str):
499            chart = None
500            break
501          cur_index += 3
502        if chart:
503          charts.append(chart)
504    return charts
505