results_report.py revision 4467f004e7f0854963bec90daff1879fbd9d2fec 
1#!/usr/bin/python
2
3# Copyright 2011 Google Inc. All Rights Reserved.
4
5from utils.tabulator import *
6
7from column_chart import ColumnChart
8from results_organizer import ResultOrganizer
9from perf_table import PerfTable
10
11
12class ResultsReport(object):
13  MAX_COLOR_CODE = 255
14
15  def __init__(self, experiment):
16    self.experiment = experiment
17    self.benchmark_runs = experiment.benchmark_runs
18    self.labels = experiment.labels
19    self.benchmarks = experiment.benchmarks
20    self.baseline = self.labels[0]
21
22  def _SortByLabel(self, runs):
23    labels = {}
24    for benchmark_run in runs:
25      if benchmark_run.label_name not in labels:
26        labels[benchmark_run.label_name] = []
27      labels[benchmark_run.label_name].append(benchmark_run)
28    return labels
29
30  def GetFullTables(self, perf=False):
31    columns = [Column(NonEmptyCountResult(),
32                      Format(),
33                      "Completed"),
34               Column(RawResult(),
35                      Format()),
36               Column(MinResult(),
37                      Format()),
38               Column(MaxResult(),
39                      Format()),
40               Column(AmeanResult(),
41                      Format()),
42               Column(StdResult(),
43                      Format())
44              ]
45    if not perf:
46      return self._GetTables(self.labels, self.benchmark_runs, columns)
47    return self. _GetPerfTables(self.labels, columns)
48
49  def GetSummaryTables(self, perf=False):
50    columns = [Column(NonEmptyCountResult(),
51                      Format(),
52                      "Completed"),
53               Column(AmeanResult(),
54                      Format()),
55               Column(StdResult(),
56                      Format(), "StdDev"),
57               Column(CoeffVarResult(),
58                      CoeffVarFormat(), "StdDev/Mean"),
59               Column(GmeanRatioResult(),
60                      RatioFormat(), "GmeanSpeedup"),
61               Column(GmeanRatioResult(),
62                      ColorBoxFormat(), " "),
63               Column(PValueResult(),
64                      PValueFormat(), "p-value")
65              ]
66    if not perf:
67      return self._GetTables(self.labels, self.benchmark_runs, columns)
68    return self. _GetPerfTables(self.labels, columns)
69
70  def _ParseColumn(self, columns, iteration):
71    new_column = []
72    for column in columns:
73      if column.result.__class__.__name__ != "RawResult":
74      #TODO(asharif): tabulator should support full table natively.
75        new_column.append(column)
76      else:
77        for i in range(iteration):
78          cc = Column(LiteralResult(i), Format(), str(i+1))
79          new_column.append(cc)
80    return new_column
81
82  def _AreAllRunsEmpty(self, runs):
83    for label in runs:
84      for dictionary in label:
85        if dictionary:
86          return False
87    return True
88
89  def _GetTableHeader(self, benchmark):
90    benchmark_info = ("Benchmark:  {0};  Iterations: {1}"
91                      .format(benchmark.name, benchmark.iterations))
92    cell = Cell()
93    cell.string_value = benchmark_info
94    cell.header = True
95    return  [[cell]]
96
97  def _GetTables(self, labels, benchmark_runs, columns):
98    tables = []
99    ro = ResultOrganizer(benchmark_runs, labels, self.benchmarks)
100    result = ro.result
101    label_name = ro.labels
102    for item in result:
103      runs = result[item]
104      for benchmark in self.benchmarks:
105        if benchmark.name == item:
106          break
107      ben_table = self._GetTableHeader(benchmark)
108
109      if  self._AreAllRunsEmpty(runs):
110        cell = Cell()
111        cell.string_value = ("This benchmark contains no result."
112                             " Is the benchmark name valid?")
113        cell_table = [[cell]]
114      else:
115        tg = TableGenerator(runs, label_name)
116        table = tg.GetTable()
117        parsed_columns = self._ParseColumn(columns, benchmark.iterations)
118        tf = TableFormatter(table, parsed_columns)
119        cell_table = tf.GetCellTable()
120      tables.append(ben_table)
121      tables.append(cell_table)
122    return tables
123
124  def _GetPerfTables(self, labels, columns):
125    tables = []
126    label_names = [label.name for label in labels]
127    p_table = PerfTable(self.experiment, label_names)
128
129    if not p_table.perf_data:
130      return tables
131
132    for benchmark in p_table.perf_data:
133      ben = None
134      for ben in self.benchmarks:
135        if ben.name == benchmark:
136          break
137
138      ben_table = self._GetTableHeader(ben)
139      tables.append(ben_table)
140      benchmark_data = p_table.perf_data[benchmark]
141      table = []
142      for event in benchmark_data:
143        tg = TableGenerator(benchmark_data[event], label_names)
144        table = tg.GetTable()
145        parsed_columns = self._ParseColumn(columns, ben.iterations)
146        tf = TableFormatter(table, parsed_columns)
147        tf.GenerateCellTable()
148        tf.AddColumnName()
149        tf.AddLabelName()
150        tf.AddHeader(str(event))
151        table = tf.GetCellTable(headers=False)
152        tables.append(table)
153    return tables
154
155  def PrintTables(self, tables, out_to):
156    output = ""
157    if not tables:
158      return output
159    for table in tables:
160      if out_to == "HTML":
161        tp = TablePrinter(table, TablePrinter.HTML)
162      elif out_to == "PLAIN":
163        tp = TablePrinter(table, TablePrinter.PLAIN)
164      elif out_to == "CONSOLE":
165        tp = TablePrinter(table, TablePrinter.CONSOLE)
166      elif out_to == "TSV":
167        tp = TablePrinter(table, TablePrinter.TSV)
168      elif out_to == "EMAIL":
169        tp = TablePrinter(table, TablePrinter.EMAIL)
170      else:
171        pass
172      output += tp.Print()
173    return output
174
175
176class TextResultsReport(ResultsReport):
177  TEXT = """
178===========================================
179Results report for: '%s'
180===========================================
181
182-------------------------------------------
183Summary
184-------------------------------------------
185%s
186
187
188Number re-images: %s
189
190-------------------------------------------
191Benchmark Run Status
192-------------------------------------------
193%s
194
195
196-------------------------------------------
197Perf Data
198-------------------------------------------
199%s
200
201
202
203Experiment File
204-------------------------------------------
205%s
206
207
208CPUInfo
209-------------------------------------------
210%s
211===========================================
212"""
213
214  def __init__(self, experiment, email=False):
215    super(TextResultsReport, self).__init__(experiment)
216    self.email = email
217
218  def GetStatusTable(self):
219    """Generate the status table by the tabulator."""
220    table = [["", ""]]
221    columns = [Column(LiteralResult(iteration=0), Format(), "Status"),
222               Column(LiteralResult(iteration=1), Format(), "Failing Reason")]
223
224    for benchmark_run in self.benchmark_runs:
225      status = [benchmark_run.name, [benchmark_run.timeline.GetLastEvent(),
226                                     benchmark_run.failure_reason]]
227      table.append(status)
228    tf = TableFormatter(table, columns)
229    cell_table = tf.GetCellTable()
230    return [cell_table]
231
232  def GetReport(self):
233    """Generate the report for email and console."""
234    status_table = self.GetStatusTable()
235    summary_table = self.GetSummaryTables()
236    full_table = self.GetFullTables()
237    perf_table = self.GetSummaryTables(perf=True)
238    if not perf_table:
239      perf_table = None
240    if not self.email:
241      return self.TEXT % (self.experiment.name,
242                          self.PrintTables(summary_table, "CONSOLE"),
243                          self.experiment.machine_manager.num_reimages,
244                          self.PrintTables(status_table, "CONSOLE"),
245                          self.PrintTables(perf_table, "CONSOLE"),
246                          self.experiment.experiment_file,
247                          self.experiment.machine_manager.GetAllCPUInfo(
248                              self.experiment.labels))
249
250    return self.TEXT % (self.experiment.name,
251                        self.PrintTables(summary_table, "EMAIL"),
252                        self.experiment.machine_manager.num_reimages,
253                        self.PrintTables(status_table, "EMAIL"),
254                        self.PrintTables(perf_table, "EMAIL"),
255                        self.experiment.experiment_file,
256                        self.experiment.machine_manager.GetAllCPUInfo(
257                            self.experiment.labels))
258
259
260class HTMLResultsReport(ResultsReport):
261
262  HTML = """
263<html>
264  <head>
265    <style type="text/css">
266
267body {
268  font-family: "Lucida Sans Unicode", "Lucida Grande", Sans-Serif;
269  font-size: 12px;
270}
271
272pre {
273  margin: 10px;
274  color: #039;
275  font-size: 14px;
276}
277
278.chart {
279  display: inline;
280}
281
282.hidden {
283  visibility: hidden;
284}
285
286.results-section {
287  border: 1px solid #b9c9fe;
288  margin: 10px;
289}
290
291.results-section-title {
292  background-color: #b9c9fe;
293  color: #039;
294  padding: 7px;
295  font-size: 14px;
296  width: 200px;
297}
298
299.results-section-content {
300  margin: 10px;
301  padding: 10px;
302  overflow:auto;
303}
304
305#box-table-a {
306  font-size: 12px;
307  width: 480px;
308  text-align: left;
309  border-collapse: collapse;
310}
311
312#box-table-a th {
313  padding: 6px;
314  background: #b9c9fe;
315  border-right: 1px solid #fff;
316  border-bottom: 1px solid #fff;
317  color: #039;
318  text-align: center;
319}
320
321#box-table-a td {
322  padding: 4px;
323  background: #e8edff;
324  border-bottom: 1px solid #fff;
325  border-right: 1px solid #fff;
326  color: #669;
327  border-top: 1px solid transparent;
328}
329
330#box-table-a tr:hover td {
331  background: #d0dafd;
332  color: #339;
333}
334
335    </style>
336    <script type='text/javascript' src='https://www.google.com/jsapi'></script>
337    <script type='text/javascript'>
338      google.load('visualization', '1', {packages:['corechart']});
339      google.setOnLoadCallback(init);
340      function init() {
341        switchTab('summary', 'html');
342        %s
343        switchTab('full', 'html');
344        drawTable();
345      }
346      function drawTable() {
347        %s
348      }
349      function switchTab(table, tab) {
350        document.getElementById(table + '-html').style.display = 'none';
351        document.getElementById(table + '-text').style.display = 'none';
352        document.getElementById(table + '-tsv').style.display = 'none';
353        document.getElementById(table + '-' + tab).style.display = 'block';
354      }
355    </script>
356  </head>
357
358  <body>
359    <div class='results-section'>
360      <div class='results-section-title'>Summary Table</div>
361      <div class='results-section-content'>
362        <div id='summary-html'>%s</div>
363        <div id='summary-text'><pre>%s</pre></div>
364        <div id='summary-tsv'><pre>%s</pre></div>
365      </div>
366      %s
367    </div>
368    %s
369    <div class='results-section'>
370      <div class='results-section-title'>Charts</div>
371      <div class='results-section-content'>%s</div>
372    </div>
373    <div class='results-section'>
374      <div class='results-section-title'>Full Table</div>
375      <div class='results-section-content'>
376        <div id='full-html'>%s</div>
377        <div id='full-text'><pre>%s</pre></div>
378        <div id='full-tsv'><pre>%s</pre></div>
379      </div>
380      %s
381    </div>
382    <div class='results-section'>
383      <div class='results-section-title'>Experiment File</div>
384      <div class='results-section-content'>
385        <pre>%s</pre>
386    </div>
387    </div>
388  </body>
389</html>
390"""
391
392  PERF_HTML = """
393    <div class='results-section'>
394      <div class='results-section-title'>Perf Table</div>
395      <div class='results-section-content'>
396        <div id='perf-html'>%s</div>
397        <div id='perf-text'><pre>%s</pre></div>
398        <div id='perf-tsv'><pre>%s</pre></div>
399      </div>
400      %s
401    </div>
402"""
403
404  def __init__(self, experiment):
405    super(HTMLResultsReport, self).__init__(experiment)
406
407  def _GetTabMenuHTML(self, table):
408    return """
409<div class='tab-menu'>
410  <a href="javascript:switchTab('%s', 'html')">HTML</a>
411  <a href="javascript:switchTab('%s', 'text')">Text</a>
412  <a href="javascript:switchTab('%s', 'tsv')">TSV</a>
413</div>""" % (table, table, table)
414
415  def GetReport(self):
416    chart_javascript = ""
417    charts = self._GetCharts(self.labels, self.benchmark_runs)
418    for chart in charts:
419      chart_javascript += chart.GetJavascript()
420    chart_divs = ""
421    for chart in charts:
422      chart_divs += chart.GetDiv()
423
424    summary_table = self.GetSummaryTables()
425    full_table = self.GetFullTables()
426    perf_table = self.GetSummaryTables(perf=True)
427    if perf_table:
428      perf_html = self.PERF_HTML % (
429          self.PrintTables(perf_table, "HTML"),
430          self.PrintTables(perf_table, "PLAIN"),
431          self.PrintTables(perf_table, "TSV"),
432          self._GetTabMenuHTML("perf")
433          )
434      perf_init = "switchTab('perf', 'html');"
435    else:
436      perf_html = ""
437      perf_init = ""
438
439    return self.HTML % (perf_init,
440                        chart_javascript,
441                        self.PrintTables(summary_table, "HTML"),
442                        self.PrintTables(summary_table, "PLAIN"),
443                        self.PrintTables(summary_table, "TSV"),
444                        self._GetTabMenuHTML("summary"),
445                        perf_html,
446                        chart_divs,
447                        self.PrintTables(full_table, "HTML"),
448                        self.PrintTables(full_table, "PLAIN"),
449                        self.PrintTables(full_table, "TSV"),
450                        self._GetTabMenuHTML("full"),
451                        self.experiment.experiment_file)
452
453  def _GetCharts(self, labels, benchmark_runs):
454    charts = []
455    ro = ResultOrganizer(benchmark_runs, labels)
456    result = ro.result
457    for item in result:
458      runs = result[item]
459      tg = TableGenerator(runs, ro.labels)
460      table = tg.GetTable()
461      columns = [Column(AmeanResult(),
462                        Format()),
463                 Column(MinResult(),
464                        Format()),
465                 Column(MaxResult(),
466                        Format())
467                ]
468      tf = TableFormatter(table, columns)
469      data_table = tf.GetCellTable()
470
471      for i in range(2, len(data_table)):
472        cur_row_data = data_table[i]
473        autotest_key = cur_row_data[0].string_value
474        title = "{0}: {1}".format(item, autotest_key.replace("/", ""))
475        chart = ColumnChart(title, 300, 200)
476        chart.AddColumn("Label", "string")
477        chart.AddColumn("Average", "number")
478        chart.AddColumn("Min", "number")
479        chart.AddColumn("Max", "number")
480        chart.AddSeries("Min", "line", "black")
481        chart.AddSeries("Max", "line", "black")
482        cur_index = 1
483        for label in ro.labels:
484          chart.AddRow([label, cur_row_data[cur_index].value,
485                        cur_row_data[cur_index + 1].value,
486                        cur_row_data[cur_index + 2].value])
487          if isinstance(cur_row_data[cur_index].value, str):
488            chart = None
489            break
490          cur_index += 3
491        if chart:
492          charts.append(chart)
493    return charts
494