results_report.py revision 0dcbc4b1714260820fd4b8d6536fbb05e139cc0f 
1#!/usr/bin/python
2
3# Copyright 2011 Google Inc. All Rights Reserved.
4
5from column_chart import ColumnChart
6from results_columns import IterationColumn
7from results_columns import IterationsCompleteColumn
8from results_columns import MaxColumn
9from results_columns import MeanColumn
10from results_columns import MinColumn
11from results_columns import RatioColumn
12from results_columns import StandardDeviationColumn
13from results_sorter import ResultSorter
14from table import Table
15
16
17class ResultsReport(object):
18  DELTA_COLUMN_NAME = "Change"
19
20  def __init__(self, experiment):
21    self.experiment = experiment
22    self.benchmark_runs = experiment.benchmark_runs
23    self.labels = experiment.labels
24    self.benchmarks = experiment.benchmarks
25    self.baseline = self.labels[0]
26
27  def _SortByLabel(self, runs):
28    labels = {}
29    for benchmark_run in runs:
30      if benchmark_run.label_name not in labels:
31        labels[benchmark_run.label_name] = []
32      labels[benchmark_run.label_name].append(benchmark_run)
33    return labels
34
35  def GetFullTable(self):
36    full_columns = []
37    max_iterations = 0
38    for benchmark in self.benchmarks:
39      if benchmark.iterations > max_iterations:
40        max_iterations = benchmark.iterations
41
42    for i in range(1, max_iterations + 1):
43      full_columns.append(IterationColumn(str(i), i))
44
45    full_columns.append(IterationsCompleteColumn("Completed"))
46    full_columns.append(MinColumn("Min"))
47    full_columns.append(MaxColumn("Max"))
48    full_columns.append(MeanColumn("Avg"))
49    full_columns.append(StandardDeviationColumn("Std Dev"))
50    full_columns.append(RatioColumn(self.DELTA_COLUMN_NAME))
51    return self._GetTable(self.labels, self.benchmarks, self.benchmark_runs,
52                          full_columns)
53
54  def GetSummaryTable(self):
55    summary_columns = [MeanColumn("Average"),
56                       RatioColumn(self.DELTA_COLUMN_NAME)]
57    return self._GetTable(self.labels, self.benchmarks, self.benchmark_runs,
58                          summary_columns)
59
60  def _GetTable(self, labels, benchmarks, benchmark_runs, columns):
61    table = Table("box-table-a")
62    label_headings = [Table.Cell("", hidden=True, colspan=2, header=True)]
63    for label in labels:
64      colspan = len(columns)
65      if label.name == self.baseline.name:
66        colspan -= 1
67      label_headings.append(Table.Cell(label.name, colspan=colspan,
68                                       header=True))
69
70    table.AddRow(label_headings)
71
72    column_headings = [Table.Cell("Autotest Key", header=True),
73                       Table.Cell("Iterations", header=True)]
74    for label in labels:
75      for column in columns:
76        if (label.name == self.baseline.name and
77            column.name == self.DELTA_COLUMN_NAME):
78          continue
79        column_headings.append(Table.Cell(column.name, header=True))
80
81    table.AddRow(column_headings)
82
83    sorter = ResultSorter(benchmark_runs)
84
85    for benchmark in benchmarks:
86      table.AddRow([Table.Cell(benchmark.name)])
87      autotest_keys = sorter.GetAutotestKeys(benchmark.name)
88      for autotest_key in autotest_keys:
89        row = [Table.Cell(autotest_key),
90               Table.Cell(benchmark.iterations)]
91        for label in labels:
92          for column in columns:
93            if (label.name == self.baseline.name and
94                column.name == self.DELTA_COLUMN_NAME):
95              continue
96            results = sorter.GetResults(benchmark.name,
97                                        autotest_key, label.name)
98            baseline_results = sorter.GetResults(benchmark.name,
99                                                 autotest_key,
100                                                 self.baseline.name)
101            value = column.Compute(results, baseline_results)
102            if isinstance(value, float):
103              value_string = "%.2f" % value
104            else:
105              value_string = value
106
107            row.append(Table.Cell(value_string))
108
109        table.AddRow(row)
110
111    return table
112
113
114class TextResultsReport(ResultsReport):
115  TEXT = """
116===========================================
117Results report for: '%s'
118===========================================
119
120-------------------------------------------
121Benchmark Run Status
122-------------------------------------------
123%s
124
125Number re-images: %s
126
127-------------------------------------------
128Summary
129-------------------------------------------
130%s
131
132-------------------------------------------
133Full Table
134-------------------------------------------
135%s
136
137-------------------------------------------
138Experiment File
139-------------------------------------------
140%s
141===========================================
142"""
143
144  def __init__(self, experiment):
145    super(TextResultsReport, self).__init__(experiment)
146
147  def GetStatusTable(self):
148    status_table = Table("status")
149    for benchmark_run in self.benchmark_runs:
150      status_table.AddRow([Table.Cell(benchmark_run.name),
151                           Table.Cell(benchmark_run.status),
152                           Table.Cell(benchmark_run.failure_reason)])
153    return status_table
154
155  def GetReport(self):
156    return self.TEXT % (self.experiment.name,
157                        self.GetStatusTable().ToText(),
158                        self.experiment.machine_manager.num_reimages,
159                        self.GetSummaryTable().ToText(30),
160                        self.GetFullTable().ToText(30),
161                        self.experiment.experiment_file)
162
163
164class HTMLResultsReport(ResultsReport):
165  HTML = """
166<html>
167  <head>
168    <style type="text/css">
169
170body {
171  font-family: "Lucida Sans Unicode", "Lucida Grande", Sans-Serif;
172  font-size: 12px;
173}
174
175pre {
176  margin: 10px;
177  color: #039;
178  font-size: 14px;
179}
180
181.chart {
182  display: inline;
183}
184
185.hidden {
186  visibility: hidden;
187}
188
189.results-section {
190  border: 1px solid #b9c9fe;
191  margin: 10px;
192}
193
194.results-section-title {
195  background-color: #b9c9fe;
196  color: #039;
197  padding: 7px;
198  font-size: 14px;
199  width: 200px;
200}
201
202.results-section-content {
203  margin: 10px;
204  padding: 10px;
205  overflow:auto;
206}
207
208#box-table-a {
209  font-size: 12px;
210  width: 480px;
211  text-align: left;
212  border-collapse: collapse;
213}
214
215#box-table-a th {
216  padding: 6px;
217  background: #b9c9fe;
218  border-right: 1px solid #fff;
219  border-bottom: 1px solid #fff;
220  color: #039;
221  text-align: center;
222}
223
224#box-table-a td {
225  padding: 4px;
226  background: #e8edff;
227  border-bottom: 1px solid #fff;
228  border-right: 1px solid #fff;
229  color: #669;
230  border-top: 1px solid transparent;
231}
232
233#box-table-a tr:hover td {
234  background: #d0dafd;
235  color: #339;
236}
237
238    </style>
239    <script type='text/javascript' src='https://www.google.com/jsapi'></script>
240    <script type='text/javascript'>
241      google.load('visualization', '1', {packages:['corechart']});
242      google.setOnLoadCallback(init);
243      function init() {
244        switchTab('summary', 'html');
245        switchTab('full', 'html');
246        drawTable();
247      }
248      function drawTable() {
249        %s
250      }
251      function switchTab(table, tab) {
252        document.getElementById(table + '-html').style.display = 'none';
253        document.getElementById(table + '-text').style.display = 'none';
254        document.getElementById(table + '-tsv').style.display = 'none';
255        document.getElementById(table + '-' + tab).style.display = 'block';
256      }
257    </script>
258  </head>
259
260  <body>
261    <div class='results-section'>
262      <div class='results-section-title'>Summary Table</div>
263      <div class='results-section-content'>
264        <div id='summary-html'>%s</div>
265        <div id='summary-text'><pre>%s</pre></div>
266        <div id='summary-tsv'><pre>%s</pre></div>
267      </div>
268      %s
269    </div>
270    <div class='results-section'>
271      <div class='results-section-title'>Charts</div>
272      <div class='results-section-content'>%s</div>
273    </div>
274    <div class='results-section'>
275      <div class='results-section-title'>Full Table</div>
276      <div class='results-section-content'>
277        <div id='full-html'>%s</div>
278        <div id='full-text'><pre>%s</pre></div>
279        <div id='full-tsv'><pre>%s</pre></div>
280      </div>
281      %s
282    </div>
283    <div class='results-section'>
284      <div class='results-section-title'>Experiment File</div>
285      <div class='results-section-content'>
286        <pre>%s</pre>
287    </div>
288    </div>
289  </body>
290</html>
291"""
292
293  def __init__(self, experiment):
294    super(HTMLResultsReport, self).__init__(experiment)
295
296  def _GetTabMenuHTML(self, table):
297    return """
298<div class='tab-menu'>
299  <a href="javascript:switchTab('%s', 'html')">HTML</a>
300  <a href="javascript:switchTab('%s', 'text')">Text</a>
301  <a href="javascript:switchTab('%s', 'tsv')">TSV</a>
302</div>""" % (table, table, table)
303
304  def GetReport(self):
305    chart_javascript = ""
306    charts = self._GetCharts(self.labels, self.benchmarks, self.benchmark_runs)
307    for chart in charts:
308      chart_javascript += chart.GetJavascript()
309    chart_divs = ""
310    for chart in charts:
311      chart_divs += chart.GetDiv()
312
313    summary_table = self.GetSummaryTable()
314    full_table = self.GetFullTable()
315    return self.HTML % (chart_javascript,
316                        summary_table.ToHTML(),
317                        summary_table.ToText(),
318                        summary_table.ToTSV(),
319                        self._GetTabMenuHTML("summary"),
320                        chart_divs,
321                        full_table.ToHTML(),
322                        full_table.ToText(),
323                        full_table.ToTSV(),
324                        self._GetTabMenuHTML("full"),
325                        self.experiment.experiment_file)
326
327  def _GetCharts(self, labels, benchmarks, benchmark_runs):
328    charts = []
329    sorter = ResultSorter(benchmark_runs)
330
331    for benchmark in benchmarks:
332      autotest_keys = sorter.GetAutotestKeys(benchmark.name)
333
334      for autotest_key in autotest_keys:
335        title = "%s: %s" % (benchmark.name, autotest_key.replace("/", " "))
336        chart = ColumnChart(title, 300, 200)
337        chart.AddColumn("Label", "string")
338        chart.AddColumn("Average", "number")
339        chart.AddColumn("Min", "number")
340        chart.AddColumn("Max", "number")
341        chart.AddSeries("Min", "line", "black")
342        chart.AddSeries("Max", "line", "black")
343
344        for label in labels:
345          res = sorter.GetResults(benchmark.name, autotest_key, label.name)
346          avg_val = MeanColumn("").Compute(res, None)
347          min_val = MinColumn("").Compute(res, None)
348          max_val = MaxColumn("").Compute(res, None)
349          chart.AddRow([label.name, avg_val, min_val, max_val])
350          if isinstance(avg_val, str):
351            chart = None
352            break
353
354        if chart:
355          charts.append(chart)
356    return charts
357
358