generate_report.py revision 4bf85d17e38c3f36f64c761a0743591d296f41a5 
1#!/usr/bin/env python2
2#
3# Copyright 2016 The Chromium OS Authors. All rights reserved.
4# Use of this source code is governed by a BSD-style license that can be
5# found in the LICENSE file.
6
7"""Given a specially-formatted JSON object, generates results report(s).
8
9The JSON object should look like:
10{"data": BenchmarkData, "platforms": BenchmarkPlatforms}
11
12BenchmarkPlatforms is a [str], each of which names a platform the benchmark
13  was run on (e.g. peppy, shamu, ...). Note that the order of this list is
14  related with the order of items in BenchmarkData.
15
16BenchmarkData is a {str: [PlatformData]}. The str is the name of the benchmark,
17and a PlatformData is a set of data for a given platform. There must be one
18PlatformData for each benchmark, for each element in BenchmarkPlatforms.
19
20A PlatformData is a [{str: float}], where each str names a metric we recorded,
21and the float is the value for that metric. Each element is considered to be
22the metrics collected from an independent run of this benchmark. NOTE: Each
23PlatformData is expected to have a "retval" key, with the return value of
24the benchmark. If the benchmark is successful, said return value should be 0.
25Otherwise, this will break some of our JSON functionality.
26
27Putting it all together, a JSON object will end up looking like:
28  { "platforms": ["peppy", "peppy-new-crosstool"],
29    "data": {
30      "bench_draw_line": [
31        [{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0},
32         {"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}],
33        [{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0},
34         {"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}]
35      ]
36    }
37  }
38
39Which says that we ran a benchmark on platforms named peppy, and
40  peppy-new-crosstool.
41We ran one benchmark, named bench_draw_line.
42It was run twice on each platform.
43Peppy's runs took 1.321ms and 1.920ms, while peppy-new-crosstool's took 1.221ms
44  and 1.423ms. None of the runs failed to complete.
45"""
46
47from __future__ import division
48from __future__ import print_function
49
50import argparse
51import functools
52import json
53import os
54import sys
55import traceback
56
57from results_report import BenchmarkResults
58from results_report import HTMLResultsReport
59from results_report import JSONResultsReport
60from results_report import TextResultsReport
61
62
63def CountBenchmarks(benchmark_runs):
64  """Counts the number of iterations for each benchmark in benchmark_runs."""
65  # Example input for benchmark_runs:
66  # {"bench": [[run1, run2, run3], [run1, run2, run3, run4]]}
67  def _MaxLen(results):
68    return 0 if not results else max(len(r) for r in results)
69  return [(name, _MaxLen(results))
70          for name, results in benchmark_runs.iteritems()]
71
72
73def CutResultsInPlace(results, max_keys=50, complain_on_update=True):
74  """Limits the given benchmark results to max_keys keys in-place.
75
76  This takes the `data` field from the benchmark input, and mutates each
77  benchmark run to contain `max_keys` elements (ignoring special elements, like
78  "retval"). At the moment, it just selects the first `max_keys` keyvals,
79  alphabetically.
80
81  If complain_on_update is true, this will print a message noting that a
82  truncation occurred.
83
84  This returns the `results` object that was passed in, for convenience.
85
86  e.g.
87  >>> benchmark_data = {
88  ...   "bench_draw_line": [
89  ...     [{"time (ms)": 1.321, "memory (mb)": 128.1, "retval": 0},
90  ...      {"time (ms)": 1.920, "memory (mb)": 128.4, "retval": 0}],
91  ...     [{"time (ms)": 1.221, "memory (mb)": 124.3, "retval": 0},
92  ...      {"time (ms)": 1.423, "memory (mb)": 123.9, "retval": 0}]
93  ...   ]
94  ... }
95  >>> CutResultsInPlace(benchmark_data, max_keys=1, complain_on_update=False)
96  {
97    'bench_draw_line': [
98      [{'memory (mb)': 128.1, 'retval': 0},
99       {'memory (mb)': 128.4, 'retval': 0}],
100      [{'memory (mb)': 124.3, 'retval': 0},
101       {'memory (mb)': 123.9, 'retval': 0}]
102    ]
103  }
104  """
105  actually_updated = False
106  for bench_results in results.itervalues():
107    for platform_results in bench_results:
108      for i, result in enumerate(platform_results):
109        # Keep the keys that come earliest when sorted alphabetically.
110        # Forcing alphabetical order is arbitrary, but necessary; otherwise,
111        # the keyvals we'd emit would depend on our iteration order through a
112        # map.
113        removable_keys = sorted(k for k in result if k != 'retval')
114        retained_keys = removable_keys[:max_keys]
115        platform_results[i] = {k: result[k] for k in retained_keys}
116        # retval needs to be passed through all of the time.
117        retval = result.get('retval')
118        if retval is not None:
119          platform_results[i]['retval'] = retval
120        actually_updated = actually_updated or \
121          len(retained_keys) != len(removable_keys)
122
123  if actually_updated and complain_on_update:
124    print("Warning: Some benchmark keyvals have been truncated.",
125          file=sys.stderr)
126  return results
127
128
129def _ConvertToASCII(obj):
130  """Convert an object loaded from JSON to ASCII; JSON gives us unicode."""
131
132  # Using something like `object_hook` is insufficient, since it only fires on
133  # actual JSON objects. `encoding` fails, too, since the default decoder always
134  # uses unicode() to decode strings.
135  if isinstance(obj, unicode):
136    return str(obj)
137  if isinstance(obj, dict):
138    return {_ConvertToASCII(k): _ConvertToASCII(v) for k, v in obj.iteritems()}
139  if isinstance(obj, list):
140    return [_ConvertToASCII(v) for v in obj]
141  return obj
142
143
144def _PositiveInt(s):
145  i = int(s)
146  if i < 0:
147    raise argparse.ArgumentTypeError('%d is not a positive integer.' % (i, ))
148  return i
149
150
151def _AccumulateActions(args):
152  """Given program arguments, determines what actions we want to run.
153
154  Returns [(ResultsReportCtor, str)], where ResultsReportCtor can construct a
155  ResultsReport, and the str is the file extension for the given report.
156  """
157  results = []
158  # The order of these is arbitrary.
159  if args.json:
160    results.append((JSONResultsReport, 'json'))
161  if args.text:
162    results.append((TextResultsReport, 'txt'))
163  if args.email:
164    email_ctor = functools.partial(TextResultsReport, email=True)
165    results.append((email_ctor, 'email'))
166  # We emit HTML if nothing else was specified.
167  if args.html or not results:
168    results.append((HTMLResultsReport, 'html'))
169  return results
170
171
172# Note: get_contents is a function, because it may be expensive (generating some
173# HTML reports takes O(seconds) on my machine, depending on the size of the
174# input data).
175def WriteFile(output_prefix, extension, get_contents, overwrite, verbose):
176  """Writes `contents` to a file named "${output_prefix}.${extension}".
177
178  get_contents should be a zero-args function that returns a string (of the
179  contents to write).
180  If output_prefix == '-', this writes to stdout.
181  If overwrite is False, this will not overwrite files.
182  """
183  if output_prefix == '-':
184    if verbose:
185      print('Writing %s report to stdout' % (extension, ), file=sys.stderr)
186    sys.stdout.write(get_contents())
187    return
188
189  file_name = '%s.%s' % (output_prefix, extension)
190  if not overwrite and os.path.exists(file_name):
191    raise IOError('Refusing to write %s -- it already exists' % (file_name, ))
192
193  with open(file_name, 'w') as out_file:
194    if verbose:
195      print('Writing %s report to %s' % (extension, file_name), file=sys.stderr)
196    out_file.write(get_contents())
197
198
199def RunActions(actions, benchmark_results, output_prefix, overwrite, verbose):
200  """Runs `actions`, returning True if all succeeded."""
201  failed = False
202
203  report_ctor = None # Make the linter happy
204  for report_ctor, extension in actions:
205    try:
206      get_contents = lambda: report_ctor(benchmark_results).GetReport()
207      WriteFile(output_prefix, extension, get_contents, overwrite, verbose)
208    except Exception:
209      # Complain and move along; we may have more actions that might complete
210      # successfully.
211      failed = True
212      traceback.print_exc()
213  return not failed
214
215
216def PickInputFile(input_name):
217  """Given program arguments, returns file to read for benchmark input."""
218  return sys.stdin if input_name == '-' else open(input_name)
219
220
221def _NoPerfReport(_label_name, _benchmark_name, _benchmark_iteration):
222  return {}
223
224
225def _ParseArgs(argv):
226  parser = argparse.ArgumentParser(description='Turns JSON into results '
227                                   'report(s).')
228  parser.add_argument('-v', '--verbose', action='store_true',
229                      help='Be a tiny bit more verbose.')
230  parser.add_argument('-f', '--force', action='store_true',
231                      help='Overwrite existing results files.')
232  parser.add_argument('-o', '--output', default='report', type=str,
233                      help='Prefix of the output filename (default: report). '
234                      '- means stdout.')
235  parser.add_argument('-i', '--input', required=True, type=str,
236                      help='Where to read the JSON from. - means stdin.')
237  parser.add_argument('-l', '--statistic-limit', default=0, type=_PositiveInt,
238                      help='The maximum number of benchmark statistics to '
239                      'display from a single run. 0 implies unlimited.')
240  parser.add_argument('--json', action='store_true',
241                      help='Output a JSON report.')
242  parser.add_argument('--text', action='store_true',
243                      help='Output a text report.')
244  parser.add_argument('--email', action='store_true',
245                      help='Output a text report suitable for email.')
246  parser.add_argument('--html', action='store_true',
247                      help='Output an HTML report (this is the default if no '
248                      'other output format is specified).')
249  return parser.parse_args(argv)
250
251
252def Main(argv):
253  args = _ParseArgs(argv)
254  # JSON likes to load UTF-8; our results reporter *really* doesn't like
255  # UTF-8.
256  with PickInputFile(args.input) as in_file:
257    raw_results = _ConvertToASCII(json.load(in_file))
258
259  platform_names = raw_results['platforms']
260  results = raw_results['data']
261  if args.statistic_limit:
262    results = CutResultsInPlace(results, max_keys=args.statistic_limit)
263  benches = CountBenchmarks(results)
264  # In crosperf, a label is essentially a platform+configuration. So, a name of
265  # a label and a name of a platform are equivalent for our purposes.
266  bench_results = BenchmarkResults(label_names=platform_names,
267                                   benchmark_names_and_iterations=benches,
268                                   run_keyvals=results,
269                                   read_perf_report=_NoPerfReport)
270  actions = _AccumulateActions(args)
271  ok = RunActions(actions, bench_results, args.output, args.force,
272                  args.verbose)
273  return 0 if ok else 1
274
275
276if __name__ == '__main__':
277  sys.exit(Main(sys.argv[1:]))
278