results_report.py revision 74b3d3234dc857ebee3af81faa3f8f420ac6ecce 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7from utils.tabulator import *
8
9from column_chart import ColumnChart
10from results_organizer import ResultOrganizer
11from perf_table import PerfTable
12
13
14class ResultsReport(object):
15  MAX_COLOR_CODE = 255
16  PERF_ROWS = 5
17
18  def __init__(self, experiment):
19    self.experiment = experiment
20    self.benchmark_runs = experiment.benchmark_runs
21    self.labels = experiment.labels
22    self.benchmarks = experiment.benchmarks
23    self.baseline = self.labels[0]
24
25  def _SortByLabel(self, runs):
26    labels = {}
27    for benchmark_run in runs:
28      if benchmark_run.label_name not in labels:
29        labels[benchmark_run.label_name] = []
30      labels[benchmark_run.label_name].append(benchmark_run)
31    return labels
32
33  def GetFullTables(self, perf=False):
34    columns = [Column(RawResult(),
35                      Format()),
36               Column(MinResult(),
37                      Format()),
38               Column(MaxResult(),
39                      Format()),
40               Column(AmeanResult(),
41                      Format()),
42               Column(StdResult(),
43                      Format(), "StdDev"),
44               Column(CoeffVarResult(),
45                      CoeffVarFormat(), "StdDev/Mean"),
46               Column(GmeanRatioResult(),
47                      RatioFormat(), "GmeanSpeedup"),
48               Column(PValueResult(),
49                      PValueFormat(), "p-value")
50              ]
51    if not perf:
52      return self._GetTables(self.labels, self.benchmark_runs, columns,
53                             "full")
54    return self._GetPerfTables(self.labels, columns, "full")
55
56  def GetSummaryTables(self, perf=False):
57    columns = [Column(AmeanResult(),
58                      Format()),
59               Column(StdResult(),
60                      Format(), "StdDev"),
61               Column(CoeffVarResult(),
62                      CoeffVarFormat(), "StdDev/Mean"),
63               Column(GmeanRatioResult(),
64                      RatioFormat(), "GmeanSpeedup"),
65               Column(PValueResult(),
66                      PValueFormat(), "p-value")
67              ]
68    if not perf:
69      return self._GetTables(self.labels, self.benchmark_runs, columns,
70                             "summary")
71    return self._GetPerfTables(self.labels, columns, "summary")
72
73  def _ParseColumn(self, columns, iteration):
74    new_column = []
75    for column in columns:
76      if column.result.__class__.__name__ != "RawResult":
77      #TODO(asharif): tabulator should support full table natively.
78        new_column.append(column)
79      else:
80        for i in range(iteration):
81          cc = Column(LiteralResult(i), Format(), str(i+1))
82          new_column.append(cc)
83    return new_column
84
85  def _AreAllRunsEmpty(self, runs):
86    for label in runs:
87      for dictionary in label:
88        if dictionary:
89          return False
90    return True
91
92  def _GetTableHeader(self, benchmark):
93    benchmark_info = ("Benchmark:  {0};  Iterations: {1}"
94                      .format(benchmark.name, benchmark.iterations))
95    cell = Cell()
96    cell.string_value = benchmark_info
97    cell.header = True
98    return  [[cell]]
99
100  def _FixFalsePositiveTests(self, result, table_type):
101    # Occasionally Telemetry tests will not fail but they will not return a
102    # result, either.  Look for those cases, and force them to be a fail.
103    # (This can happen if, for example, the test has been disabled.)
104    for k in result:
105      for run in result[k]:
106        run_dict = run[0]
107        if len(run_dict) != 1 or run_dict['retval'] != 0:
108          continue
109        run_dict['retval'] = 1
110        if table_type == 'summary':
111          print ("WARNING:  Test '%s' appears to have succeeded but returned"
112                 " no results." % k)
113
114  def _GetTables(self, labels, benchmark_runs, columns, table_type):
115    tables = []
116    ro = ResultOrganizer(benchmark_runs, labels, self.benchmarks)
117    result = ro.result
118    label_name = ro.labels
119    self._FixFalsePositiveTests(result, table_type)
120    for item in result:
121      runs = result[item]
122      for benchmark in self.benchmarks:
123        if benchmark.name == item:
124          break
125      ben_table = self._GetTableHeader(benchmark)
126
127      if  self._AreAllRunsEmpty(runs):
128        cell = Cell()
129        cell.string_value = ("This benchmark contains no result."
130                             " Is the benchmark name valid?")
131        cell_table = [[cell]]
132      else:
133        tg = TableGenerator(runs, label_name)
134        table = tg.GetTable()
135        parsed_columns = self._ParseColumn(columns, benchmark.iterations)
136        tf = TableFormatter(table, parsed_columns)
137        cell_table = tf.GetCellTable(table_type)
138      tables.append(ben_table)
139      tables.append(cell_table)
140    return tables
141
142  def _GetPerfTables(self, labels, columns, table_type):
143    tables = []
144    label_names = [label.name for label in labels]
145    p_table = PerfTable(self.experiment, label_names)
146
147    if not p_table.perf_data:
148      return tables
149
150    for benchmark in p_table.perf_data:
151      ben = None
152      for ben in self.benchmarks:
153        if ben.name == benchmark:
154          break
155
156      ben_table = self._GetTableHeader(ben)
157      tables.append(ben_table)
158      benchmark_data = p_table.perf_data[benchmark]
159      row_info = p_table.row_info[benchmark]
160      table = []
161      for event in benchmark_data:
162        tg = TableGenerator(benchmark_data[event], label_names,
163                            sort=TableGenerator.SORT_BY_VALUES_DESC)
164        table = tg.GetTable(max(self.PERF_ROWS, row_info[event]))
165        parsed_columns = self._ParseColumn(columns, ben.iterations)
166        tf = TableFormatter(table, parsed_columns)
167        tf.GenerateCellTable()
168        tf.AddColumnName()
169        tf.AddLabelName()
170        tf.AddHeader(str(event))
171        table = tf.GetCellTable(table_type, headers=False)
172        tables.append(table)
173    return tables
174
175  def PrintTables(self, tables, out_to):
176    output = ""
177    if not tables:
178      return output
179    for table in tables:
180      if out_to == "HTML":
181        tp = TablePrinter(table, TablePrinter.HTML)
182      elif out_to == "PLAIN":
183        tp = TablePrinter(table, TablePrinter.PLAIN)
184      elif out_to == "CONSOLE":
185        tp = TablePrinter(table, TablePrinter.CONSOLE)
186      elif out_to == "TSV":
187        tp = TablePrinter(table, TablePrinter.TSV)
188      elif out_to == "EMAIL":
189        tp = TablePrinter(table, TablePrinter.EMAIL)
190      else:
191        pass
192      output += tp.Print()
193    return output
194
195
196class TextResultsReport(ResultsReport):
197  TEXT = """
198===========================================
199Results report for: '%s'
200===========================================
201
202-------------------------------------------
203Summary
204-------------------------------------------
205%s
206
207
208Number re-images: %s
209
210-------------------------------------------
211Benchmark Run Status
212-------------------------------------------
213%s
214
215
216-------------------------------------------
217Perf Data
218-------------------------------------------
219%s
220
221
222
223Experiment File
224-------------------------------------------
225%s
226
227
228CPUInfo
229-------------------------------------------
230%s
231===========================================
232"""
233
234  def __init__(self, experiment, email=False):
235    super(TextResultsReport, self).__init__(experiment)
236    self.email = email
237
238  def GetStatusTable(self):
239    """Generate the status table by the tabulator."""
240    table = [["", ""]]
241    columns = [Column(LiteralResult(iteration=0), Format(), "Status"),
242               Column(LiteralResult(iteration=1), Format(), "Failing Reason")]
243
244    for benchmark_run in self.benchmark_runs:
245      status = [benchmark_run.name, [benchmark_run.timeline.GetLastEvent(),
246                                     benchmark_run.failure_reason]]
247      table.append(status)
248    tf = TableFormatter(table, columns)
249    cell_table = tf.GetCellTable("status")
250    return [cell_table]
251
252  def GetReport(self):
253    """Generate the report for email and console."""
254    status_table = self.GetStatusTable()
255    summary_table = self.GetSummaryTables()
256    full_table = self.GetFullTables()
257    perf_table = self.GetSummaryTables(perf=True)
258    if not perf_table:
259      perf_table = None
260    if not self.email:
261      return self.TEXT % (self.experiment.name,
262                          self.PrintTables(summary_table, "CONSOLE"),
263                          self.experiment.machine_manager.num_reimages,
264                          self.PrintTables(status_table, "CONSOLE"),
265                          self.PrintTables(perf_table, "CONSOLE"),
266                          self.experiment.experiment_file,
267                          self.experiment.machine_manager.GetAllCPUInfo(
268                              self.experiment.labels))
269
270    return self.TEXT % (self.experiment.name,
271                        self.PrintTables(summary_table, "EMAIL"),
272                        self.experiment.machine_manager.num_reimages,
273                        self.PrintTables(status_table, "EMAIL"),
274                        self.PrintTables(perf_table, "EMAIL"),
275                        self.experiment.experiment_file,
276                        self.experiment.machine_manager.GetAllCPUInfo(
277                            self.experiment.labels))
278
279
280class HTMLResultsReport(ResultsReport):
281
282  HTML = """
283<html>
284  <head>
285    <style type="text/css">
286
287body {
288  font-family: "Lucida Sans Unicode", "Lucida Grande", Sans-Serif;
289  font-size: 12px;
290}
291
292pre {
293  margin: 10px;
294  color: #039;
295  font-size: 14px;
296}
297
298.chart {
299  display: inline;
300}
301
302.hidden {
303  visibility: hidden;
304}
305
306.results-section {
307  border: 1px solid #b9c9fe;
308  margin: 10px;
309}
310
311.results-section-title {
312  background-color: #b9c9fe;
313  color: #039;
314  padding: 7px;
315  font-size: 14px;
316  width: 200px;
317}
318
319.results-section-content {
320  margin: 10px;
321  padding: 10px;
322  overflow:auto;
323}
324
325#box-table-a {
326  font-size: 12px;
327  width: 480px;
328  text-align: left;
329  border-collapse: collapse;
330}
331
332#box-table-a th {
333  padding: 6px;
334  background: #b9c9fe;
335  border-right: 1px solid #fff;
336  border-bottom: 1px solid #fff;
337  color: #039;
338  text-align: center;
339}
340
341#box-table-a td {
342  padding: 4px;
343  background: #e8edff;
344  border-bottom: 1px solid #fff;
345  border-right: 1px solid #fff;
346  color: #669;
347  border-top: 1px solid transparent;
348}
349
350#box-table-a tr:hover td {
351  background: #d0dafd;
352  color: #339;
353}
354
355    </style>
356    <script type='text/javascript' src='https://www.google.com/jsapi'></script>
357    <script type='text/javascript'>
358      google.load('visualization', '1', {packages:['corechart']});
359      google.setOnLoadCallback(init);
360      function init() {
361        switchTab('summary', 'html');
362        %s
363        switchTab('full', 'html');
364        drawTable();
365      }
366      function drawTable() {
367        %s
368      }
369      function switchTab(table, tab) {
370        document.getElementById(table + '-html').style.display = 'none';
371        document.getElementById(table + '-text').style.display = 'none';
372        document.getElementById(table + '-tsv').style.display = 'none';
373        document.getElementById(table + '-' + tab).style.display = 'block';
374      }
375    </script>
376  </head>
377
378  <body>
379    <div class='results-section'>
380      <div class='results-section-title'>Summary Table</div>
381      <div class='results-section-content'>
382        <div id='summary-html'>%s</div>
383        <div id='summary-text'><pre>%s</pre></div>
384        <div id='summary-tsv'><pre>%s</pre></div>
385      </div>
386      %s
387    </div>
388    %s
389    <div class='results-section'>
390      <div class='results-section-title'>Charts</div>
391      <div class='results-section-content'>%s</div>
392    </div>
393    <div class='results-section'>
394      <div class='results-section-title'>Full Table</div>
395      <div class='results-section-content'>
396        <div id='full-html'>%s</div>
397        <div id='full-text'><pre>%s</pre></div>
398        <div id='full-tsv'><pre>%s</pre></div>
399      </div>
400      %s
401    </div>
402    <div class='results-section'>
403      <div class='results-section-title'>Experiment File</div>
404      <div class='results-section-content'>
405        <pre>%s</pre>
406    </div>
407    </div>
408  </body>
409</html>
410"""
411
412  PERF_HTML = """
413    <div class='results-section'>
414      <div class='results-section-title'>Perf Table</div>
415      <div class='results-section-content'>
416        <div id='perf-html'>%s</div>
417        <div id='perf-text'><pre>%s</pre></div>
418        <div id='perf-tsv'><pre>%s</pre></div>
419      </div>
420      %s
421    </div>
422"""
423
424  def __init__(self, experiment):
425    super(HTMLResultsReport, self).__init__(experiment)
426
427  def _GetTabMenuHTML(self, table):
428    return """
429<div class='tab-menu'>
430  <a href="javascript:switchTab('%s', 'html')">HTML</a>
431  <a href="javascript:switchTab('%s', 'text')">Text</a>
432  <a href="javascript:switchTab('%s', 'tsv')">TSV</a>
433</div>""" % (table, table, table)
434
435  def GetReport(self):
436    chart_javascript = ""
437    charts = self._GetCharts(self.labels, self.benchmark_runs)
438    for chart in charts:
439      chart_javascript += chart.GetJavascript()
440    chart_divs = ""
441    for chart in charts:
442      chart_divs += chart.GetDiv()
443
444    summary_table = self.GetSummaryTables()
445    full_table = self.GetFullTables()
446    perf_table = self.GetSummaryTables(perf=True)
447    if perf_table:
448      perf_html = self.PERF_HTML % (
449          self.PrintTables(perf_table, "HTML"),
450          self.PrintTables(perf_table, "PLAIN"),
451          self.PrintTables(perf_table, "TSV"),
452          self._GetTabMenuHTML("perf")
453          )
454      perf_init = "switchTab('perf', 'html');"
455    else:
456      perf_html = ""
457      perf_init = ""
458
459    return self.HTML % (perf_init,
460                        chart_javascript,
461                        self.PrintTables(summary_table, "HTML"),
462                        self.PrintTables(summary_table, "PLAIN"),
463                        self.PrintTables(summary_table, "TSV"),
464                        self._GetTabMenuHTML("summary"),
465                        perf_html,
466                        chart_divs,
467                        self.PrintTables(full_table, "HTML"),
468                        self.PrintTables(full_table, "PLAIN"),
469                        self.PrintTables(full_table, "TSV"),
470                        self._GetTabMenuHTML("full"),
471                        self.experiment.experiment_file)
472
473  def _GetCharts(self, labels, benchmark_runs):
474    charts = []
475    ro = ResultOrganizer(benchmark_runs, labels)
476    result = ro.result
477    for item in result:
478      runs = result[item]
479      tg = TableGenerator(runs, ro.labels)
480      table = tg.GetTable()
481      columns = [Column(AmeanResult(),
482                        Format()),
483                 Column(MinResult(),
484                        Format()),
485                 Column(MaxResult(),
486                        Format())
487                ]
488      tf = TableFormatter(table, columns)
489      data_table = tf.GetCellTable("full")
490
491      for i in range(2, len(data_table)):
492        cur_row_data = data_table[i]
493        test_key = cur_row_data[0].string_value
494        title = "{0}: {1}".format(item, test_key.replace("/", ""))
495        chart = ColumnChart(title, 300, 200)
496        chart.AddColumn("Label", "string")
497        chart.AddColumn("Average", "number")
498        chart.AddColumn("Min", "number")
499        chart.AddColumn("Max", "number")
500        chart.AddSeries("Min", "line", "black")
501        chart.AddSeries("Max", "line", "black")
502        cur_index = 1
503        for label in ro.labels:
504          chart.AddRow([label, cur_row_data[cur_index].value,
505                        cur_row_data[cur_index + 1].value,
506                        cur_row_data[cur_index + 2].value])
507          if isinstance(cur_row_data[cur_index].value, str):
508            chart = None
509            break
510          cur_index += 3
511        if chart:
512          charts.append(chart)
513    return charts
514