results_report.py revision 9b852cfd9a602c5f80c8e621c696b796ce5177fd 
1#!/usr/bin/python
2
3# Copyright (c) 2013 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7import datetime
8import json
9import os
10
11from utils.tabulator import *
12
13from update_telemetry_defaults import TelemetryDefaults
14from column_chart import ColumnChart
15from results_organizer import ResultOrganizer
16from perf_table import PerfTable
17
18
19def ParseChromeosImage(chromeos_image):
20  """Parse the chromeos_image string for the image and version.
21
22  The chromeos_image string will probably be in one of two formats:
23  1: <path-to-chroot>/src/build/images/<board>/<ChromeOS-version>.<datetime>/ \
24     chromiumos_test_image.bin
25  2: <path-to-chroot>/chroot/tmp/<buildbot-build>/<ChromeOS-version>/ \
26      chromiumos_test_image.bin
27
28  We parse these strings to find the 'chromeos_version' to store in the
29  json archive (without the .datatime bit in the first case); and also
30  the 'chromeos_image', which would be all of the first case, but only the
31  part after '/chroot/tmp' in the second case.
32
33  Args:
34    chromeos_image:  String containing the path to the chromeos_image that
35      crosperf used for the test.
36
37  Returns:
38    version, image:  The results of parsing the input string, as explained
39      above.
40  """
41  version = ''
42  real_file = os.path.realpath(os.path.expanduser(chromeos_image))
43  pieces = real_file.split('/')
44  # Find the Chromeos Version, e.g. R45-2345.0.0.....
45  # chromeos_image should have been something like:
46  # <path>/<board-trybot-release>/<chromeos-version>/chromiumos_test_image.bin"
47  num_pieces = len(pieces)
48  if pieces[num_pieces-1] == "chromiumos_test_image.bin":
49    version = pieces[num_pieces-2]
50    # Find last '.' in the version and chop it off (removing the .datatime
51    # piece from local builds).
52    loc = version.rfind('.')
53    version = version[:loc]
54  # Find the chromeos image.  If it's somewhere in .../chroot/tmp/..., then
55  # it's an official image that got downloaded, so chop off the download path
56  # to make the official image name more clear.
57  loc = real_file.find('/chroot/tmp')
58  if loc != -1:
59    loc += len('/chroot/tmp')
60    real_file = real_file[loc:]
61  image = real_file
62  return version,image
63
64class ResultsReport(object):
65  MAX_COLOR_CODE = 255
66  PERF_ROWS = 5
67
68  def __init__(self, experiment):
69    self.experiment = experiment
70    self.benchmark_runs = experiment.benchmark_runs
71    self.labels = experiment.labels
72    self.benchmarks = experiment.benchmarks
73    self.baseline = self.labels[0]
74
75  def _SortByLabel(self, runs):
76    labels = {}
77    for benchmark_run in runs:
78      if benchmark_run.label_name not in labels:
79        labels[benchmark_run.label_name] = []
80      labels[benchmark_run.label_name].append(benchmark_run)
81    return labels
82
83  def GetFullTables(self, perf=False):
84    columns = [Column(RawResult(),
85                      Format()),
86               Column(MinResult(),
87                      Format()),
88               Column(MaxResult(),
89                      Format()),
90               Column(AmeanResult(),
91                      Format()),
92               Column(StdResult(),
93                      Format(), "StdDev"),
94               Column(CoeffVarResult(),
95                      CoeffVarFormat(), "StdDev/Mean"),
96               Column(GmeanRatioResult(),
97                      RatioFormat(), "GmeanSpeedup"),
98               Column(PValueResult(),
99                      PValueFormat(), "p-value")
100              ]
101    if not perf:
102      return self._GetTables(self.labels, self.benchmark_runs, columns,
103                             "full")
104    return self._GetPerfTables(self.labels, columns, "full")
105
106  def GetSummaryTables(self, perf=False):
107    columns = [Column(AmeanResult(),
108                      Format()),
109               Column(StdResult(),
110                      Format(), "StdDev"),
111               Column(CoeffVarResult(),
112                      CoeffVarFormat(), "StdDev/Mean"),
113               Column(GmeanRatioResult(),
114                      RatioFormat(), "GmeanSpeedup"),
115               Column(PValueResult(),
116                      PValueFormat(), "p-value")
117              ]
118    if not perf:
119      return self._GetTables(self.labels, self.benchmark_runs, columns,
120                             "summary")
121    return self._GetPerfTables(self.labels, columns, "summary")
122
123  def _ParseColumn(self, columns, iteration):
124    new_column = []
125    for column in columns:
126      if column.result.__class__.__name__ != "RawResult":
127      #TODO(asharif): tabulator should support full table natively.
128        new_column.append(column)
129      else:
130        for i in range(iteration):
131          cc = Column(LiteralResult(i), Format(), str(i+1))
132          new_column.append(cc)
133    return new_column
134
135  def _AreAllRunsEmpty(self, runs):
136    for label in runs:
137      for dictionary in label:
138        if dictionary:
139          return False
140    return True
141
142  def _GetTableHeader(self, benchmark):
143    benchmark_info = ("Benchmark:  {0};  Iterations: {1}"
144                      .format(benchmark.name, benchmark.iterations))
145    cell = Cell()
146    cell.string_value = benchmark_info
147    cell.header = True
148    return  [[cell]]
149
150  def _FixFalsePositiveTests(self, result, table_type):
151    # Occasionally Telemetry tests will not fail but they will not return a
152    # result, either.  Look for those cases, and force them to be a fail.
153    # (This can happen if, for example, the test has been disabled.)
154    for k in result:
155      for run in result[k]:
156        run_dict = run[0]
157        if len(run_dict) != 1 or run_dict['retval'] != 0:
158          continue
159        run_dict['retval'] = 1
160        if table_type == 'summary':
161          print ("WARNING:  Test '%s' appears to have succeeded but returned"
162                 " no results." % k)
163
164  def _GetTables(self, labels, benchmark_runs, columns, table_type):
165    tables = []
166    ro = ResultOrganizer(benchmark_runs, labels, self.benchmarks)
167    result = ro.result
168    label_name = ro.labels
169    self._FixFalsePositiveTests(result, table_type)
170    for item in result:
171      runs = result[item]
172      for benchmark in self.benchmarks:
173        if benchmark.name == item:
174          break
175      ben_table = self._GetTableHeader(benchmark)
176
177      if  self._AreAllRunsEmpty(runs):
178        cell = Cell()
179        cell.string_value = ("This benchmark contains no result."
180                             " Is the benchmark name valid?")
181        cell_table = [[cell]]
182      else:
183        tg = TableGenerator(runs, label_name)
184        table = tg.GetTable()
185        parsed_columns = self._ParseColumn(columns, benchmark.iterations)
186        tf = TableFormatter(table, parsed_columns)
187        cell_table = tf.GetCellTable(table_type)
188      tables.append(ben_table)
189      tables.append(cell_table)
190    return tables
191
192  def _GetPerfTables(self, labels, columns, table_type):
193    tables = []
194    label_names = [label.name for label in labels]
195    p_table = PerfTable(self.experiment, label_names)
196
197    if not p_table.perf_data:
198      return tables
199
200    for benchmark in p_table.perf_data:
201      ben = None
202      for ben in self.benchmarks:
203        if ben.name == benchmark:
204          break
205
206      ben_table = self._GetTableHeader(ben)
207      tables.append(ben_table)
208      benchmark_data = p_table.perf_data[benchmark]
209      row_info = p_table.row_info[benchmark]
210      table = []
211      for event in benchmark_data:
212        tg = TableGenerator(benchmark_data[event], label_names,
213                            sort=TableGenerator.SORT_BY_VALUES_DESC)
214        table = tg.GetTable(max(self.PERF_ROWS, row_info[event]))
215        parsed_columns = self._ParseColumn(columns, ben.iterations)
216        tf = TableFormatter(table, parsed_columns)
217        tf.GenerateCellTable()
218        tf.AddColumnName()
219        tf.AddLabelName()
220        tf.AddHeader(str(event))
221        table = tf.GetCellTable(table_type, headers=False)
222        tables.append(table)
223    return tables
224
225  def PrintTables(self, tables, out_to):
226    output = ""
227    if not tables:
228      return output
229    for table in tables:
230      if out_to == "HTML":
231        tp = TablePrinter(table, TablePrinter.HTML)
232      elif out_to == "PLAIN":
233        tp = TablePrinter(table, TablePrinter.PLAIN)
234      elif out_to == "CONSOLE":
235        tp = TablePrinter(table, TablePrinter.CONSOLE)
236      elif out_to == "TSV":
237        tp = TablePrinter(table, TablePrinter.TSV)
238      elif out_to == "EMAIL":
239        tp = TablePrinter(table, TablePrinter.EMAIL)
240      else:
241        pass
242      output += tp.Print()
243    return output
244
245
246class TextResultsReport(ResultsReport):
247  TEXT = """
248===========================================
249Results report for: '%s'
250===========================================
251
252-------------------------------------------
253Summary
254-------------------------------------------
255%s
256
257
258Number re-images: %s
259
260-------------------------------------------
261Benchmark Run Status
262-------------------------------------------
263%s
264
265
266-------------------------------------------
267Perf Data
268-------------------------------------------
269%s
270
271
272
273Experiment File
274-------------------------------------------
275%s
276
277
278CPUInfo
279-------------------------------------------
280%s
281===========================================
282"""
283
284  def __init__(self, experiment, email=False):
285    super(TextResultsReport, self).__init__(experiment)
286    self.email = email
287
288  def GetStatusTable(self):
289    """Generate the status table by the tabulator."""
290    table = [["", ""]]
291    columns = [Column(LiteralResult(iteration=0), Format(), "Status"),
292               Column(LiteralResult(iteration=1), Format(), "Failing Reason")]
293
294    for benchmark_run in self.benchmark_runs:
295      status = [benchmark_run.name, [benchmark_run.timeline.GetLastEvent(),
296                                     benchmark_run.failure_reason]]
297      table.append(status)
298    tf = TableFormatter(table, columns)
299    cell_table = tf.GetCellTable("status")
300    return [cell_table]
301
302  def GetReport(self):
303    """Generate the report for email and console."""
304    status_table = self.GetStatusTable()
305    summary_table = self.GetSummaryTables()
306    full_table = self.GetFullTables()
307    perf_table = self.GetSummaryTables(perf=True)
308    if not perf_table:
309      perf_table = None
310    if not self.email:
311      return self.TEXT % (self.experiment.name,
312                          self.PrintTables(summary_table, "CONSOLE"),
313                          self.experiment.machine_manager.num_reimages,
314                          self.PrintTables(status_table, "CONSOLE"),
315                          self.PrintTables(perf_table, "CONSOLE"),
316                          self.experiment.experiment_file,
317                          self.experiment.machine_manager.GetAllCPUInfo(
318                              self.experiment.labels))
319
320    return self.TEXT % (self.experiment.name,
321                        self.PrintTables(summary_table, "EMAIL"),
322                        self.experiment.machine_manager.num_reimages,
323                        self.PrintTables(status_table, "EMAIL"),
324                        self.PrintTables(perf_table, "EMAIL"),
325                        self.experiment.experiment_file,
326                        self.experiment.machine_manager.GetAllCPUInfo(
327                            self.experiment.labels))
328
329
330class HTMLResultsReport(ResultsReport):
331
332  HTML = """
333<html>
334  <head>
335    <style type="text/css">
336
337body {
338  font-family: "Lucida Sans Unicode", "Lucida Grande", Sans-Serif;
339  font-size: 12px;
340}
341
342pre {
343  margin: 10px;
344  color: #039;
345  font-size: 14px;
346}
347
348.chart {
349  display: inline;
350}
351
352.hidden {
353  visibility: hidden;
354}
355
356.results-section {
357  border: 1px solid #b9c9fe;
358  margin: 10px;
359}
360
361.results-section-title {
362  background-color: #b9c9fe;
363  color: #039;
364  padding: 7px;
365  font-size: 14px;
366  width: 200px;
367}
368
369.results-section-content {
370  margin: 10px;
371  padding: 10px;
372  overflow:auto;
373}
374
375#box-table-a {
376  font-size: 12px;
377  width: 480px;
378  text-align: left;
379  border-collapse: collapse;
380}
381
382#box-table-a th {
383  padding: 6px;
384  background: #b9c9fe;
385  border-right: 1px solid #fff;
386  border-bottom: 1px solid #fff;
387  color: #039;
388  text-align: center;
389}
390
391#box-table-a td {
392  padding: 4px;
393  background: #e8edff;
394  border-bottom: 1px solid #fff;
395  border-right: 1px solid #fff;
396  color: #669;
397  border-top: 1px solid transparent;
398}
399
400#box-table-a tr:hover td {
401  background: #d0dafd;
402  color: #339;
403}
404
405    </style>
406    <script type='text/javascript' src='https://www.google.com/jsapi'></script>
407    <script type='text/javascript'>
408      google.load('visualization', '1', {packages:['corechart']});
409      google.setOnLoadCallback(init);
410      function init() {
411        switchTab('summary', 'html');
412        %s
413        switchTab('full', 'html');
414        drawTable();
415      }
416      function drawTable() {
417        %s
418      }
419      function switchTab(table, tab) {
420        document.getElementById(table + '-html').style.display = 'none';
421        document.getElementById(table + '-text').style.display = 'none';
422        document.getElementById(table + '-tsv').style.display = 'none';
423        document.getElementById(table + '-' + tab).style.display = 'block';
424      }
425    </script>
426  </head>
427
428  <body>
429    <div class='results-section'>
430      <div class='results-section-title'>Summary Table</div>
431      <div class='results-section-content'>
432        <div id='summary-html'>%s</div>
433        <div id='summary-text'><pre>%s</pre></div>
434        <div id='summary-tsv'><pre>%s</pre></div>
435      </div>
436      %s
437    </div>
438    %s
439    <div class='results-section'>
440      <div class='results-section-title'>Charts</div>
441      <div class='results-section-content'>%s</div>
442    </div>
443    <div class='results-section'>
444      <div class='results-section-title'>Full Table</div>
445      <div class='results-section-content'>
446        <div id='full-html'>%s</div>
447        <div id='full-text'><pre>%s</pre></div>
448        <div id='full-tsv'><pre>%s</pre></div>
449      </div>
450      %s
451    </div>
452    <div class='results-section'>
453      <div class='results-section-title'>Experiment File</div>
454      <div class='results-section-content'>
455        <pre>%s</pre>
456    </div>
457    </div>
458  </body>
459</html>
460"""
461
462  PERF_HTML = """
463    <div class='results-section'>
464      <div class='results-section-title'>Perf Table</div>
465      <div class='results-section-content'>
466        <div id='perf-html'>%s</div>
467        <div id='perf-text'><pre>%s</pre></div>
468        <div id='perf-tsv'><pre>%s</pre></div>
469      </div>
470      %s
471    </div>
472"""
473
474  def __init__(self, experiment):
475    super(HTMLResultsReport, self).__init__(experiment)
476
477  def _GetTabMenuHTML(self, table):
478    return """
479<div class='tab-menu'>
480  <a href="javascript:switchTab('%s', 'html')">HTML</a>
481  <a href="javascript:switchTab('%s', 'text')">Text</a>
482  <a href="javascript:switchTab('%s', 'tsv')">TSV</a>
483</div>""" % (table, table, table)
484
485  def GetReport(self):
486    chart_javascript = ""
487    charts = self._GetCharts(self.labels, self.benchmark_runs)
488    for chart in charts:
489      chart_javascript += chart.GetJavascript()
490    chart_divs = ""
491    for chart in charts:
492      chart_divs += chart.GetDiv()
493
494    summary_table = self.GetSummaryTables()
495    full_table = self.GetFullTables()
496    perf_table = self.GetSummaryTables(perf=True)
497    if perf_table:
498      perf_html = self.PERF_HTML % (
499          self.PrintTables(perf_table, "HTML"),
500          self.PrintTables(perf_table, "PLAIN"),
501          self.PrintTables(perf_table, "TSV"),
502          self._GetTabMenuHTML("perf")
503          )
504      perf_init = "switchTab('perf', 'html');"
505    else:
506      perf_html = ""
507      perf_init = ""
508
509    return self.HTML % (perf_init,
510                        chart_javascript,
511                        self.PrintTables(summary_table, "HTML"),
512                        self.PrintTables(summary_table, "PLAIN"),
513                        self.PrintTables(summary_table, "TSV"),
514                        self._GetTabMenuHTML("summary"),
515                        perf_html,
516                        chart_divs,
517                        self.PrintTables(full_table, "HTML"),
518                        self.PrintTables(full_table, "PLAIN"),
519                        self.PrintTables(full_table, "TSV"),
520                        self._GetTabMenuHTML("full"),
521                        self.experiment.experiment_file)
522
523  def _GetCharts(self, labels, benchmark_runs):
524    charts = []
525    ro = ResultOrganizer(benchmark_runs, labels)
526    result = ro.result
527    for item in result:
528      runs = result[item]
529      tg = TableGenerator(runs, ro.labels)
530      table = tg.GetTable()
531      columns = [Column(AmeanResult(),
532                        Format()),
533                 Column(MinResult(),
534                        Format()),
535                 Column(MaxResult(),
536                        Format())
537                ]
538      tf = TableFormatter(table, columns)
539      data_table = tf.GetCellTable("full")
540
541      for i in range(2, len(data_table)):
542        cur_row_data = data_table[i]
543        test_key = cur_row_data[0].string_value
544        title = "{0}: {1}".format(item, test_key.replace("/", ""))
545        chart = ColumnChart(title, 300, 200)
546        chart.AddColumn("Label", "string")
547        chart.AddColumn("Average", "number")
548        chart.AddColumn("Min", "number")
549        chart.AddColumn("Max", "number")
550        chart.AddSeries("Min", "line", "black")
551        chart.AddSeries("Max", "line", "black")
552        cur_index = 1
553        for label in ro.labels:
554          chart.AddRow([label, cur_row_data[cur_index].value,
555                        cur_row_data[cur_index + 1].value,
556                        cur_row_data[cur_index + 2].value])
557          if isinstance(cur_row_data[cur_index].value, str):
558            chart = None
559            break
560          cur_index += 3
561        if chart:
562          charts.append(chart)
563    return charts
564
565class JSONResultsReport(ResultsReport):
566
567  def __init__(self, experiment, date=None, time=None):
568    super(JSONResultsReport, self).__init__(experiment)
569    self.ro = ResultOrganizer(experiment.benchmark_runs,
570                              experiment.labels,
571                              experiment.benchmarks,
572                              json_report=True)
573    self.date = date
574    self.time = time
575    self.defaults = TelemetryDefaults()
576    if not self.date:
577      timestamp = datetime.datetime.strftime(datetime.datetime.now(),
578                                           "%Y-%m-%d %H:%M:%S")
579      date, time = timestamp.split(" ")
580      self.date = date
581      self.time = time
582
583  def GetReport(self, results_dir):
584    self.defaults.ReadDefaultsFile()
585    final_results = []
586    board = self.experiment.labels[0].board
587    for test, test_results in self.ro.result.iteritems():
588      for i, label in enumerate(self.ro.labels):
589        label_results = test_results[i]
590        for j, iter_Results in enumerate(label_results):
591          iter_results = label_results[j]
592          json_results = dict()
593          json_results['date'] = self.date
594          json_results['time'] = self.time
595          json_results['board'] = board
596          json_results['label'] = label
597          common_checksum = ''
598          common_string = ''
599          for l in self.experiment.labels:
600            if l.name == label:
601              ver, img = ParseChromeosImage(l.chromeos_image)
602              json_results['chromeos_image'] = img
603              json_results['chromeos_version'] = ver
604              common_checksum = \
605                self.experiment.machine_manager.machine_checksum[l.name]
606              common_string = \
607                self.experiment.machine_manager.machine_checksum_string[l.name]
608              break
609          json_results['test_name'] = test
610          if not iter_results or iter_results['retval'] != 0:
611            json_results['pass'] = False
612          else:
613            json_results['pass'] = True
614            # Get overall results.
615            if test in self.defaults._defaults:
616              default_result_fields = self.defaults._defaults[test]
617              value = []
618              for f in default_result_fields:
619                v = iter_results[f]
620                if type(v) == list:
621                    v = v[0]
622                item = (f, float(v))
623                value.append(item)
624              json_results['overall_result'] = value
625            # Get detailed results.
626            detail_results = dict()
627            for k in iter_results.keys():
628              if k != 'retval':
629                v = iter_results[k]
630                if type(v) == list:
631                  v = v[0]
632                if v != 'PASS':
633                  if k.find('machine') == -1:
634                    detail_results[k] = float(v)
635                  else:
636                    json_results[k] = v
637            if 'machine_checksum' not in json_results.keys():
638              json_results['machine_checksum'] = common_checksum
639            if 'machine_string' not in json_results.keys():
640              json_results['machine_string'] = common_string
641            json_results['detailed_results'] = detail_results
642          final_results.append(json_results)
643
644    filename = "report_%s_%s_%s.json" % (board, self.date,
645                                         self.time.replace(':','.'))
646    fullname = os.path.join(results_dir, filename)
647    with open(fullname, "w") as fp:
648      json.dump(final_results, fp, indent=2)
649