1# Copyright 2014 The Chromium Authors. All rights reserved.
2# Use of this source code is governed by a BSD-style license that can be
3# found in the LICENSE file.
4
5import optparse
6import os
7import sys
8
9from telemetry.core import util
10from telemetry.results import buildbot_output_formatter
11from telemetry.results import chart_json_output_formatter
12from telemetry.results import csv_output_formatter
13from telemetry.results import gtest_progress_reporter
14from telemetry.results import html_output_formatter
15from telemetry.results import json_output_formatter
16from telemetry.results import page_test_results
17from telemetry.results import progress_reporter
18
19# Allowed output formats. The default is the first item in the list.
20_OUTPUT_FORMAT_CHOICES = ('html', 'buildbot', 'block', 'csv', 'gtest', 'json',
21    'chartjson', 'none')
22
23
24def AddResultsOptions(parser):
25  group = optparse.OptionGroup(parser, 'Results options')
26  group.add_option('--chartjson', action='store_true',
27                   help='Output Chart JSON. Ignores --output-format.')
28  group.add_option('--output-format', action='append', dest='output_formats',
29                    choices=_OUTPUT_FORMAT_CHOICES, default=[],
30                    help='Output format. Defaults to "%%default". '
31                    'Can be %s.' % ', '.join(_OUTPUT_FORMAT_CHOICES))
32  group.add_option('-o', '--output',
33                    dest='output_file',
34                    default=None,
35                    help='Redirects output to a file. Defaults to stdout.')
36  group.add_option('--output-trace-tag',
37                    default='',
38                    help='Append a tag to the key of each result trace.')
39  group.add_option('--reset-results', action='store_true',
40                    help='Delete all stored results.')
41  group.add_option('--upload-results', action='store_true',
42                    help='Upload the results to cloud storage.')
43  group.add_option('--results-label',
44                    default=None,
45                    help='Optional label to use for the results of a run .')
46  group.add_option('--suppress_gtest_report',
47                   default=False,
48                   help='Whether to suppress GTest progress report.')
49  parser.add_option_group(group)
50
51
52def _GetOutputStream(output_format, output_file):
53  assert output_format in _OUTPUT_FORMAT_CHOICES, 'Must specify a valid format.'
54  assert output_format not in ('gtest', 'none'), (
55      'Cannot set stream for \'gtest\' or \'none\' output formats.')
56
57  if output_file is None:
58    if output_format != 'html' and output_format != 'json':
59      return sys.stdout
60    output_file = os.path.join(util.GetBaseDir(), 'results.' + output_format)
61
62  output_file = os.path.expanduser(output_file)
63  open(output_file, 'a').close()  # Create file if it doesn't exist.
64  return open(output_file, 'r+')
65
66
67def _GetProgressReporter(output_skipped_tests_summary, suppress_gtest_report):
68  if suppress_gtest_report:
69    return progress_reporter.ProgressReporter()
70
71  return gtest_progress_reporter.GTestProgressReporter(
72      sys.stdout, output_skipped_tests_summary=output_skipped_tests_summary)
73
74
75def CreateResults(benchmark_metadata, options):
76  """
77  Args:
78    options: Contains the options specified in AddResultsOptions.
79  """
80  if not options.output_formats:
81    options.output_formats = [_OUTPUT_FORMAT_CHOICES[0]]
82
83  # TODO(chrishenry): It doesn't make sense to have a single output_file flag
84  # with multiple output formatters. We should explore other possible options:
85  #   - Have an output_file per output formatter
86  #   - Have --output-dir instead of --output-file
87  if len(options.output_formats) != 1 and options.output_file:
88    raise Exception('Cannot specify output_file flag with multiple output '
89                    'formats.')
90
91  output_formatters = []
92  for output_format in options.output_formats:
93    if output_format == 'none' or output_format == "gtest" or options.chartjson:
94      continue
95
96    output_stream = _GetOutputStream(output_format, options.output_file)
97    if output_format == 'csv':
98      output_formatters.append(csv_output_formatter.CsvOutputFormatter(
99          output_stream))
100    elif output_format == 'buildbot':
101      output_formatters.append(
102          buildbot_output_formatter.BuildbotOutputFormatter(
103              output_stream, trace_tag=options.output_trace_tag))
104    elif output_format == 'html':
105      # TODO(chrishenry): We show buildbot output so that users can grep
106      # through the results easily without needing to open the html
107      # file.  Another option for this is to output the results directly
108      # in gtest-style results (via some sort of progress reporter),
109      # as we plan to enable gtest-style output for all output formatters.
110      output_formatters.append(
111          buildbot_output_formatter.BuildbotOutputFormatter(
112              sys.stdout, trace_tag=options.output_trace_tag))
113      output_formatters.append(html_output_formatter.HtmlOutputFormatter(
114          output_stream, benchmark_metadata, options.reset_results,
115          options.upload_results, options.browser_type,
116          options.results_label, trace_tag=options.output_trace_tag))
117    elif output_format == 'json':
118      output_formatters.append(json_output_formatter.JsonOutputFormatter(
119          output_stream, benchmark_metadata))
120    elif output_format == 'chartjson':
121      output_formatters.append(
122          chart_json_output_formatter.ChartJsonOutputFormatter(
123              output_stream, benchmark_metadata))
124    else:
125      # Should never be reached. The parser enforces the choices.
126      raise Exception('Invalid --output-format "%s". Valid choices are: %s'
127                      % (output_format, ', '.join(_OUTPUT_FORMAT_CHOICES)))
128
129  # TODO(chrishenry): This is here to not change the output of
130  # gtest. Let's try enabling skipped tests summary for gtest test
131  # results too (in a separate patch), and see if we break anything.
132  output_skipped_tests_summary = 'gtest' in options.output_formats
133
134  reporter = _GetProgressReporter(output_skipped_tests_summary,
135                                  options.suppress_gtest_report)
136  return page_test_results.PageTestResults(
137      output_formatters=output_formatters, progress_reporter=reporter)
138