results_report.py revision ef4ca8a8cfe05ec09447896db42fb220d07834f8 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import datetime
8import json
9import os
10
11from utils.tabulator import *
12
13from update_telemetry_defaults import TelemetryDefaults
14from column_chart import ColumnChart
15from results_organizer import ResultOrganizer
16from perf_table import PerfTable
17
18
19class ResultsReport(object):
20  MAX_COLOR_CODE = 255
21  PERF_ROWS = 5
22
23  def __init__(self, experiment):
24    self.experiment = experiment
25    self.benchmark_runs = experiment.benchmark_runs
26    self.labels = experiment.labels
27    self.benchmarks = experiment.benchmarks
28    self.baseline = self.labels[0]
29
30  def _SortByLabel(self, runs):
31    labels = {}
32    for benchmark_run in runs:
33      if benchmark_run.label_name not in labels:
34        labels[benchmark_run.label_name] = []
35      labels[benchmark_run.label_name].append(benchmark_run)
36    return labels
37
38  def GetFullTables(self, perf=False):
39    columns = [Column(RawResult(),
40                      Format()),
41               Column(MinResult(),
42                      Format()),
43               Column(MaxResult(),
44                      Format()),
45               Column(AmeanResult(),
46                      Format()),
47               Column(StdResult(),
48                      Format(), "StdDev"),
49               Column(CoeffVarResult(),
50                      CoeffVarFormat(), "StdDev/Mean"),
51               Column(GmeanRatioResult(),
52                      RatioFormat(), "GmeanSpeedup"),
53               Column(PValueResult(),
54                      PValueFormat(), "p-value")
55              ]
56    if not perf:
57      return self._GetTables(self.labels, self.benchmark_runs, columns,
58                             "full")
59    return self._GetPerfTables(self.labels, columns, "full")
60
61  def GetSummaryTables(self, perf=False):
62    columns = [Column(AmeanResult(),
63                      Format()),
64               Column(StdResult(),
65                      Format(), "StdDev"),
66               Column(CoeffVarResult(),
67                      CoeffVarFormat(), "StdDev/Mean"),
68               Column(GmeanRatioResult(),
69                      RatioFormat(), "GmeanSpeedup"),
70               Column(PValueResult(),
71                      PValueFormat(), "p-value")
72              ]
73    if not perf:
74      return self._GetTables(self.labels, self.benchmark_runs, columns,
75                             "summary")
76    return self._GetPerfTables(self.labels, columns, "summary")
77
78  def _ParseColumn(self, columns, iteration):
79    new_column = []
80    for column in columns:
81      if column.result.__class__.__name__ != "RawResult":
82      #TODO(asharif): tabulator should support full table natively.
83        new_column.append(column)
84      else:
85        for i in range(iteration):
86          cc = Column(LiteralResult(i), Format(), str(i+1))
87          new_column.append(cc)
88    return new_column
89
90  def _AreAllRunsEmpty(self, runs):
91    for label in runs:
92      for dictionary in label:
93        if dictionary:
94          return False
95    return True
96
97  def _GetTableHeader(self, benchmark):
98    benchmark_info = ("Benchmark:  {0};  Iterations: {1}"
99                      .format(benchmark.name, benchmark.iterations))
100    cell = Cell()
101    cell.string_value = benchmark_info
102    cell.header = True
103    return  [[cell]]
104
105  def _FixFalsePositiveTests(self, result, table_type):
106    # Occasionally Telemetry tests will not fail but they will not return a
107    # result, either.  Look for those cases, and force them to be a fail.
108    # (This can happen if, for example, the test has been disabled.)
109    for k in result:
110      for run in result[k]:
111        run_dict = run[0]
112        if len(run_dict) != 1 or run_dict['retval'] != 0:
113          continue
114        run_dict['retval'] = 1
115        if table_type == 'summary':
116          print ("WARNING:  Test '%s' appears to have succeeded but returned"
117                 " no results." % k)
118
119  def _GetTables(self, labels, benchmark_runs, columns, table_type):
120    tables = []
121    ro = ResultOrganizer(benchmark_runs, labels, self.benchmarks)
122    result = ro.result
123    label_name = ro.labels
124    self._FixFalsePositiveTests(result, table_type)
125    for item in result:
126      runs = result[item]
127      for benchmark in self.benchmarks:
128        if benchmark.name == item:
129          break
130      ben_table = self._GetTableHeader(benchmark)
131
132      if  self._AreAllRunsEmpty(runs):
133        cell = Cell()
134        cell.string_value = ("This benchmark contains no result."
135                             " Is the benchmark name valid?")
136        cell_table = [[cell]]
137      else:
138        tg = TableGenerator(runs, label_name)
139        table = tg.GetTable()
140        parsed_columns = self._ParseColumn(columns, benchmark.iterations)
141        tf = TableFormatter(table, parsed_columns)
142        cell_table = tf.GetCellTable(table_type)
143      tables.append(ben_table)
144      tables.append(cell_table)
145    return tables
146
147  def _GetPerfTables(self, labels, columns, table_type):
148    tables = []
149    label_names = [label.name for label in labels]
150    p_table = PerfTable(self.experiment, label_names)
151
152    if not p_table.perf_data:
153      return tables
154
155    for benchmark in p_table.perf_data:
156      ben = None
157      for ben in self.benchmarks:
158        if ben.name == benchmark:
159          break
160
161      ben_table = self._GetTableHeader(ben)
162      tables.append(ben_table)
163      benchmark_data = p_table.perf_data[benchmark]
164      row_info = p_table.row_info[benchmark]
165      table = []
166      for event in benchmark_data:
167        tg = TableGenerator(benchmark_data[event], label_names,
168                            sort=TableGenerator.SORT_BY_VALUES_DESC)
169        table = tg.GetTable(max(self.PERF_ROWS, row_info[event]))
170        parsed_columns = self._ParseColumn(columns, ben.iterations)
171        tf = TableFormatter(table, parsed_columns)
172        tf.GenerateCellTable()
173        tf.AddColumnName()
174        tf.AddLabelName()
175        tf.AddHeader(str(event))
176        table = tf.GetCellTable(table_type, headers=False)
177        tables.append(table)
178    return tables
179
180  def PrintTables(self, tables, out_to):
181    output = ""
182    if not tables:
183      return output
184    for table in tables:
185      if out_to == "HTML":
186        tp = TablePrinter(table, TablePrinter.HTML)
187      elif out_to == "PLAIN":
188        tp = TablePrinter(table, TablePrinter.PLAIN)
189      elif out_to == "CONSOLE":
190        tp = TablePrinter(table, TablePrinter.CONSOLE)
191      elif out_to == "TSV":
192        tp = TablePrinter(table, TablePrinter.TSV)
193      elif out_to == "EMAIL":
194        tp = TablePrinter(table, TablePrinter.EMAIL)
195      else:
196        pass
197      output += tp.Print()
198    return output
199
200
201class TextResultsReport(ResultsReport):
202  TEXT = """
203===========================================
204Results report for: '%s'
205===========================================
206
207-------------------------------------------
208Summary
209-------------------------------------------
210%s
211
212
213Number re-images: %s
214
215-------------------------------------------
216Benchmark Run Status
217-------------------------------------------
218%s
219
220
221-------------------------------------------
222Perf Data
223-------------------------------------------
224%s
225
226
227
228Experiment File
229-------------------------------------------
230%s
231
232
233CPUInfo
234-------------------------------------------
235%s
236===========================================
237"""
238
239  def __init__(self, experiment, email=False):
240    super(TextResultsReport, self).__init__(experiment)
241    self.email = email
242
243  def GetStatusTable(self):
244    """Generate the status table by the tabulator."""
245    table = [["", ""]]
246    columns = [Column(LiteralResult(iteration=0), Format(), "Status"),
247               Column(LiteralResult(iteration=1), Format(), "Failing Reason")]
248
249    for benchmark_run in self.benchmark_runs:
250      status = [benchmark_run.name, [benchmark_run.timeline.GetLastEvent(),
251                                     benchmark_run.failure_reason]]
252      table.append(status)
253    tf = TableFormatter(table, columns)
254    cell_table = tf.GetCellTable("status")
255    return [cell_table]
256
257  def GetReport(self):
258    """Generate the report for email and console."""
259    status_table = self.GetStatusTable()
260    summary_table = self.GetSummaryTables()
261    full_table = self.GetFullTables()
262    perf_table = self.GetSummaryTables(perf=True)
263    if not perf_table:
264      perf_table = None
265    if not self.email:
266      return self.TEXT % (self.experiment.name,
267                          self.PrintTables(summary_table, "CONSOLE"),
268                          self.experiment.machine_manager.num_reimages,
269                          self.PrintTables(status_table, "CONSOLE"),
270                          self.PrintTables(perf_table, "CONSOLE"),
271                          self.experiment.experiment_file,
272                          self.experiment.machine_manager.GetAllCPUInfo(
273                              self.experiment.labels))
274
275    return self.TEXT % (self.experiment.name,
276                        self.PrintTables(summary_table, "EMAIL"),
277                        self.experiment.machine_manager.num_reimages,
278                        self.PrintTables(status_table, "EMAIL"),
279                        self.PrintTables(perf_table, "EMAIL"),
280                        self.experiment.experiment_file,
281                        self.experiment.machine_manager.GetAllCPUInfo(
282                            self.experiment.labels))
283
284
285class HTMLResultsReport(ResultsReport):
286
287  HTML = """
288<html>
289  <head>
290    <style type="text/css">
291
292body {
293  font-family: "Lucida Sans Unicode", "Lucida Grande", Sans-Serif;
294  font-size: 12px;
295}
296
297pre {
298  margin: 10px;
299  color: #039;
300  font-size: 14px;
301}
302
303.chart {
304  display: inline;
305}
306
307.hidden {
308  visibility: hidden;
309}
310
311.results-section {
312  border: 1px solid #b9c9fe;
313  margin: 10px;
314}
315
316.results-section-title {
317  background-color: #b9c9fe;
318  color: #039;
319  padding: 7px;
320  font-size: 14px;
321  width: 200px;
322}
323
324.results-section-content {
325  margin: 10px;
326  padding: 10px;
327  overflow:auto;
328}
329
330#box-table-a {
331  font-size: 12px;
332  width: 480px;
333  text-align: left;
334  border-collapse: collapse;
335}
336
337#box-table-a th {
338  padding: 6px;
339  background: #b9c9fe;
340  border-right: 1px solid #fff;
341  border-bottom: 1px solid #fff;
342  color: #039;
343  text-align: center;
344}
345
346#box-table-a td {
347  padding: 4px;
348  background: #e8edff;
349  border-bottom: 1px solid #fff;
350  border-right: 1px solid #fff;
351  color: #669;
352  border-top: 1px solid transparent;
353}
354
355#box-table-a tr:hover td {
356  background: #d0dafd;
357  color: #339;
358}
359
360    </style>
361    <script type='text/javascript' src='https://www.google.com/jsapi'></script>
362    <script type='text/javascript'>
363      google.load('visualization', '1', {packages:['corechart']});
364      google.setOnLoadCallback(init);
365      function init() {
366        switchTab('summary', 'html');
367        %s
368        switchTab('full', 'html');
369        drawTable();
370      }
371      function drawTable() {
372        %s
373      }
374      function switchTab(table, tab) {
375        document.getElementById(table + '-html').style.display = 'none';
376        document.getElementById(table + '-text').style.display = 'none';
377        document.getElementById(table + '-tsv').style.display = 'none';
378        document.getElementById(table + '-' + tab).style.display = 'block';
379      }
380    </script>
381  </head>
382
383  <body>
384    <div class='results-section'>
385      <div class='results-section-title'>Summary Table</div>
386      <div class='results-section-content'>
387        <div id='summary-html'>%s</div>
388        <div id='summary-text'><pre>%s</pre></div>
389        <div id='summary-tsv'><pre>%s</pre></div>
390      </div>
391      %s
392    </div>
393    %s
394    <div class='results-section'>
395      <div class='results-section-title'>Charts</div>
396      <div class='results-section-content'>%s</div>
397    </div>
398    <div class='results-section'>
399      <div class='results-section-title'>Full Table</div>
400      <div class='results-section-content'>
401        <div id='full-html'>%s</div>
402        <div id='full-text'><pre>%s</pre></div>
403        <div id='full-tsv'><pre>%s</pre></div>
404      </div>
405      %s
406    </div>
407    <div class='results-section'>
408      <div class='results-section-title'>Experiment File</div>
409      <div class='results-section-content'>
410        <pre>%s</pre>
411    </div>
412    </div>
413  </body>
414</html>
415"""
416
417  PERF_HTML = """
418    <div class='results-section'>
419      <div class='results-section-title'>Perf Table</div>
420      <div class='results-section-content'>
421        <div id='perf-html'>%s</div>
422        <div id='perf-text'><pre>%s</pre></div>
423        <div id='perf-tsv'><pre>%s</pre></div>
424      </div>
425      %s
426    </div>
427"""
428
429  def __init__(self, experiment):
430    super(HTMLResultsReport, self).__init__(experiment)
431
432  def _GetTabMenuHTML(self, table):
433    return """
434<div class='tab-menu'>
435  <a href="javascript:switchTab('%s', 'html')">HTML</a>
436  <a href="javascript:switchTab('%s', 'text')">Text</a>
437  <a href="javascript:switchTab('%s', 'tsv')">TSV</a>
438</div>""" % (table, table, table)
439
440  def GetReport(self):
441    chart_javascript = ""
442    charts = self._GetCharts(self.labels, self.benchmark_runs)
443    for chart in charts:
444      chart_javascript += chart.GetJavascript()
445    chart_divs = ""
446    for chart in charts:
447      chart_divs += chart.GetDiv()
448
449    summary_table = self.GetSummaryTables()
450    full_table = self.GetFullTables()
451    perf_table = self.GetSummaryTables(perf=True)
452    if perf_table:
453      perf_html = self.PERF_HTML % (
454          self.PrintTables(perf_table, "HTML"),
455          self.PrintTables(perf_table, "PLAIN"),
456          self.PrintTables(perf_table, "TSV"),
457          self._GetTabMenuHTML("perf")
458          )
459      perf_init = "switchTab('perf', 'html');"
460    else:
461      perf_html = ""
462      perf_init = ""
463
464    return self.HTML % (perf_init,
465                        chart_javascript,
466                        self.PrintTables(summary_table, "HTML"),
467                        self.PrintTables(summary_table, "PLAIN"),
468                        self.PrintTables(summary_table, "TSV"),
469                        self._GetTabMenuHTML("summary"),
470                        perf_html,
471                        chart_divs,
472                        self.PrintTables(full_table, "HTML"),
473                        self.PrintTables(full_table, "PLAIN"),
474                        self.PrintTables(full_table, "TSV"),
475                        self._GetTabMenuHTML("full"),
476                        self.experiment.experiment_file)
477
478  def _GetCharts(self, labels, benchmark_runs):
479    charts = []
480    ro = ResultOrganizer(benchmark_runs, labels)
481    result = ro.result
482    for item in result:
483      runs = result[item]
484      tg = TableGenerator(runs, ro.labels)
485      table = tg.GetTable()
486      columns = [Column(AmeanResult(),
487                        Format()),
488                 Column(MinResult(),
489                        Format()),
490                 Column(MaxResult(),
491                        Format())
492                ]
493      tf = TableFormatter(table, columns)
494      data_table = tf.GetCellTable("full")
495
496      for i in range(2, len(data_table)):
497        cur_row_data = data_table[i]
498        test_key = cur_row_data[0].string_value
499        title = "{0}: {1}".format(item, test_key.replace("/", ""))
500        chart = ColumnChart(title, 300, 200)
501        chart.AddColumn("Label", "string")
502        chart.AddColumn("Average", "number")
503        chart.AddColumn("Min", "number")
504        chart.AddColumn("Max", "number")
505        chart.AddSeries("Min", "line", "black")
506        chart.AddSeries("Max", "line", "black")
507        cur_index = 1
508        for label in ro.labels:
509          chart.AddRow([label, cur_row_data[cur_index].value,
510                        cur_row_data[cur_index + 1].value,
511                        cur_row_data[cur_index + 2].value])
512          if isinstance(cur_row_data[cur_index].value, str):
513            chart = None
514            break
515          cur_index += 3
516        if chart:
517          charts.append(chart)
518    return charts
519
520class JSONResultsReport(ResultsReport):
521
522  def __init__(self, experiment, date=None, time=None):
523    super(JSONResultsReport, self).__init__(experiment)
524    self.ro = ResultOrganizer(experiment.benchmark_runs,
525                              experiment.labels,
526                              experiment.benchmarks)
527    self.date = date
528    self.time = time
529    self.defaults = TelemetryDefaults()
530    if not self.date:
531      timestamp = datetime.datetime.strftime(datetime.datetime.now(),
532                                           "%Y-%m-%d %H:%M:%S")
533      date, time = timestamp.split(" ")
534      self.date = date
535      self.time = time
536
537
538  def _ParseChromeosImage(self, chromeos_image):
539    """Parse the chromeos_image string for the image and version.
540
541    The chromeos_image string will probably be in one of two formats:
542    1:  <path-to-chroot>/src/build/images/<board>/<ChromeOS-version>.<datetime>/chromiumos_test_image.bin
543    2: <path-to-chroot>/chroot/tmp/<buildbot-build>/<ChromeOS-version>/chromiumos_test_image.bin
544
545    We parse these strings to find the 'chromeos_version' to store in the
546    json archive (without the .datatime bit in the first case); and also
547    the 'chromeos_image', which would be all of the first case, but only the
548    part after '/chroot/tmp' in the second case.
549
550    Args:
551      chromeos_image:  String containing the path to the chromeos_image that
552        crosperf used for the test.
553
554    Returns:
555      version, image:  The results of parsing the input string, as explained
556        above.
557    """
558    version = ''
559    real_file = os.path.realpath(os.path.expanduser(chromeos_image))
560    pieces = real_file.split('/')
561    # Find the Chromeos Version, e.g. R45-2345.0.0.....
562    # chromeos_image should have been something like:
563    # <path>/<board-trybot-release>/<chromeos-version>/chromiumos_test_image.bin"
564    num_pieces = len(pieces)
565    if pieces[num_pieces-1] == "chromiumos_test_image.bin":
566      version = pieces[num_pieces-2]
567      # Find last '.' in the version and chop it off (removing the .datatime
568      # piece from local builds).
569      loc = version.rfind('.')
570      version = version[:loc]
571    # Find the chromeos image.  If it's somewhere in .../chroot/tmp/..., then
572    # it's an official image that got downloaded, so chop off the download path
573    # to make the official image name more clear.
574    loc = real_file.find('/chroot/tmp')
575    if loc != -1:
576      loc += len('/chroot/tmp')
577      real_file = real_file[loc:]
578    image = real_file
579    return version,image
580
581  def GetReport(self, results_dir):
582    self.defaults.ReadDefaultsFile()
583    final_results = []
584    board = self.experiment.labels[0].board
585    for test, test_results in self.ro.result.iteritems():
586      for i, label in enumerate(self.ro.labels):
587        label_results = test_results[i]
588        for j, iter_Results in enumerate(label_results):
589          iter_results = label_results[j]
590          json_results = dict()
591          json_results['date'] = self.date
592          json_results['time'] = self.time
593          json_results['board'] = board
594          for l in self.experiment.labels:
595            if l.name == label:
596              ver, img = self._ParseChromeosImage(l.chromeos_image)
597              json_results['chromeos_image'] = img
598              json_results['chromeos_version'] = ver
599              break
600          json_results['test_name'] = test
601          if iter_results['retval'] != 0:
602            json_results['pass'] = False
603          else:
604            json_results['pass'] = True
605            # Get overall results.
606            if test in self.defaults._defaults:
607              default_result_fields = self.defaults._defaults[test]
608              value = []
609              for f in default_result_fields:
610                item = (f, float(iter_results[f][0]))
611                value.append(item)
612              json_results['overall_result'] = value
613            # Get detailed results.
614            detail_results = dict()
615            for k in iter_results.keys():
616              if k != 'retval' and type(iter_results[k]) == list:
617                v_list = iter_results[k]
618                v = v_list[0]
619                detail_results[k] = float(v)
620            json_results['detailed_results'] = detail_results
621          final_results.append(json_results)
622
623    filename = "report_%s_%s_%s.json" % (board, self.date,
624                                         self.time.replace(':','.'))
625    fullname = os.path.join(results_dir, filename)
626    with open(fullname, "w") as fp:
627      json.dump(final_results, fp, indent=2)
628