1// Copyright 2015 Google Inc. All rights reserved.
2//
3// Licensed under the Apache License, Version 2.0 (the "License");
4// you may not use this file except in compliance with the License.
5// You may obtain a copy of the License at
6//
7//     http://www.apache.org/licenses/LICENSE-2.0
8//
9// Unless required by applicable law or agreed to in writing, software
10// distributed under the License is distributed on an "AS IS" BASIS,
11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
12// See the License for the specific language governing permissions and
13// limitations under the License.
14#ifndef BENCHMARK_REPORTER_H_
15#define BENCHMARK_REPORTER_H_
16
17#include <cassert>
18#include <iosfwd>
19#include <string>
20#include <utility>
21#include <vector>
22
23#include "benchmark_api.h"  // For forward declaration of BenchmarkReporter
24
25namespace benchmark {
26
27// Interface for custom benchmark result printers.
28// By default, benchmark reports are printed to stdout. However an application
29// can control the destination of the reports by calling
30// RunSpecifiedBenchmarks and passing it a custom reporter object.
31// The reporter object must implement the following interface.
32class BenchmarkReporter {
33 public:
34  struct Context {
35    int num_cpus;
36    double mhz_per_cpu;
37    bool cpu_scaling_enabled;
38
39    // The number of chars in the longest benchmark name.
40    size_t name_field_width;
41  };
42
43  struct Run {
44    Run()
45        : error_occurred(false),
46          iterations(1),
47          time_unit(kNanosecond),
48          real_accumulated_time(0),
49          cpu_accumulated_time(0),
50          bytes_per_second(0),
51          items_per_second(0),
52          max_heapbytes_used(0),
53          complexity(oNone),
54          complexity_lambda(),
55          complexity_n(0),
56          report_big_o(false),
57          report_rms(false) {}
58
59    std::string benchmark_name;
60    std::string report_label;  // Empty if not set by benchmark.
61    bool error_occurred;
62    std::string error_message;
63
64    int64_t iterations;
65    TimeUnit time_unit;
66    double real_accumulated_time;
67    double cpu_accumulated_time;
68
69    // Return a value representing the real time per iteration in the unit
70    // specified by 'time_unit'.
71    // NOTE: If 'iterations' is zero the returned value represents the
72    // accumulated time.
73    double GetAdjustedRealTime() const;
74
75    // Return a value representing the cpu time per iteration in the unit
76    // specified by 'time_unit'.
77    // NOTE: If 'iterations' is zero the returned value represents the
78    // accumulated time.
79    double GetAdjustedCPUTime() const;
80
81    // Zero if not set by benchmark.
82    double bytes_per_second;
83    double items_per_second;
84
85    // This is set to 0.0 if memory tracing is not enabled.
86    double max_heapbytes_used;
87
88    // Keep track of arguments to compute asymptotic complexity
89    BigO complexity;
90    BigOFunc* complexity_lambda;
91    int complexity_n;
92
93    // Inform print function whether the current run is a complexity report
94    bool report_big_o;
95    bool report_rms;
96  };
97
98  // Construct a BenchmarkReporter with the output stream set to 'std::cout'
99  // and the error stream set to 'std::cerr'
100  BenchmarkReporter();
101
102  // Called once for every suite of benchmarks run.
103  // The parameter "context" contains information that the
104  // reporter may wish to use when generating its report, for example the
105  // platform under which the benchmarks are running. The benchmark run is
106  // never started if this function returns false, allowing the reporter
107  // to skip runs based on the context information.
108  virtual bool ReportContext(const Context& context) = 0;
109
110  // Called once for each group of benchmark runs, gives information about
111  // cpu-time and heap memory usage during the benchmark run. If the group
112  // of runs contained more than two entries then 'report' contains additional
113  // elements representing the mean and standard deviation of those runs.
114  // Additionally if this group of runs was the last in a family of benchmarks
115  // 'reports' contains additional entries representing the asymptotic
116  // complexity and RMS of that benchmark family.
117  virtual void ReportRuns(const std::vector<Run>& report) = 0;
118
119  // Called once and only once after ever group of benchmarks is run and
120  // reported.
121  virtual void Finalize() {}
122
123  // REQUIRES: The object referenced by 'out' is valid for the lifetime
124  // of the reporter.
125  void SetOutputStream(std::ostream* out) {
126    assert(out);
127    output_stream_ = out;
128  }
129
130  // REQUIRES: The object referenced by 'err' is valid for the lifetime
131  // of the reporter.
132  void SetErrorStream(std::ostream* err) {
133    assert(err);
134    error_stream_ = err;
135  }
136
137  std::ostream& GetOutputStream() const { return *output_stream_; }
138
139  std::ostream& GetErrorStream() const { return *error_stream_; }
140
141  virtual ~BenchmarkReporter();
142
143  // Write a human readable string to 'out' representing the specified
144  // 'context'.
145  // REQUIRES: 'out' is non-null.
146  static void PrintBasicContext(std::ostream* out, Context const& context);
147
148 private:
149  std::ostream* output_stream_;
150  std::ostream* error_stream_;
151};
152
153// Simple reporter that outputs benchmark data to the console. This is the
154// default reporter used by RunSpecifiedBenchmarks().
155class ConsoleReporter : public BenchmarkReporter {
156 public:
157  enum OutputOptions { OO_None, OO_Color };
158  explicit ConsoleReporter(OutputOptions color_output = OO_Color)
159      : name_field_width_(0), color_output_(color_output == OO_Color) {}
160
161  virtual bool ReportContext(const Context& context);
162  virtual void ReportRuns(const std::vector<Run>& reports);
163
164 protected:
165  virtual void PrintRunData(const Run& report);
166  size_t name_field_width_;
167
168 private:
169  bool color_output_;
170};
171
172class JSONReporter : public BenchmarkReporter {
173 public:
174  JSONReporter() : first_report_(true) {}
175  virtual bool ReportContext(const Context& context);
176  virtual void ReportRuns(const std::vector<Run>& reports);
177  virtual void Finalize();
178
179 private:
180  void PrintRunData(const Run& report);
181
182  bool first_report_;
183};
184
185class CSVReporter : public BenchmarkReporter {
186 public:
187  virtual bool ReportContext(const Context& context);
188  virtual void ReportRuns(const std::vector<Run>& reports);
189
190 private:
191  void PrintRunData(const Run& report);
192};
193
194inline const char* GetTimeUnitString(TimeUnit unit) {
195  switch (unit) {
196    case kMillisecond:
197      return "ms";
198    case kMicrosecond:
199      return "us";
200    case kNanosecond:
201    default:
202      return "ns";
203  }
204}
205
206inline double GetTimeUnitMultiplier(TimeUnit unit) {
207  switch (unit) {
208    case kMillisecond:
209      return 1e3;
210    case kMicrosecond:
211      return 1e6;
212    case kNanosecond:
213    default:
214      return 1e9;
215  }
216}
217
218}  // end namespace benchmark
219#endif  // BENCHMARK_REPORTER_H_
220