1// Copyright 2015 Google Inc. All rights reserved. 2// 3// Licensed under the Apache License, Version 2.0 (the "License"); 4// you may not use this file except in compliance with the License. 5// You may obtain a copy of the License at 6// 7// http://www.apache.org/licenses/LICENSE-2.0 8// 9// Unless required by applicable law or agreed to in writing, software 10// distributed under the License is distributed on an "AS IS" BASIS, 11// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12// See the License for the specific language governing permissions and 13// limitations under the License. 14#ifndef BENCHMARK_REPORTER_H_ 15#define BENCHMARK_REPORTER_H_ 16 17#include <cassert> 18#include <iosfwd> 19#include <string> 20#include <utility> 21#include <vector> 22 23#include "benchmark_api.h" // For forward declaration of BenchmarkReporter 24 25namespace benchmark { 26 27// Interface for custom benchmark result printers. 28// By default, benchmark reports are printed to stdout. However an application 29// can control the destination of the reports by calling 30// RunSpecifiedBenchmarks and passing it a custom reporter object. 31// The reporter object must implement the following interface. 32class BenchmarkReporter { 33 public: 34 struct Context { 35 int num_cpus; 36 double mhz_per_cpu; 37 bool cpu_scaling_enabled; 38 39 // The number of chars in the longest benchmark name. 40 size_t name_field_width; 41 }; 42 43 struct Run { 44 Run() 45 : error_occurred(false), 46 iterations(1), 47 time_unit(kNanosecond), 48 real_accumulated_time(0), 49 cpu_accumulated_time(0), 50 bytes_per_second(0), 51 items_per_second(0), 52 max_heapbytes_used(0), 53 complexity(oNone), 54 complexity_lambda(), 55 complexity_n(0), 56 report_big_o(false), 57 report_rms(false) {} 58 59 std::string benchmark_name; 60 std::string report_label; // Empty if not set by benchmark. 61 bool error_occurred; 62 std::string error_message; 63 64 int64_t iterations; 65 TimeUnit time_unit; 66 double real_accumulated_time; 67 double cpu_accumulated_time; 68 69 // Return a value representing the real time per iteration in the unit 70 // specified by 'time_unit'. 71 // NOTE: If 'iterations' is zero the returned value represents the 72 // accumulated time. 73 double GetAdjustedRealTime() const; 74 75 // Return a value representing the cpu time per iteration in the unit 76 // specified by 'time_unit'. 77 // NOTE: If 'iterations' is zero the returned value represents the 78 // accumulated time. 79 double GetAdjustedCPUTime() const; 80 81 // Zero if not set by benchmark. 82 double bytes_per_second; 83 double items_per_second; 84 85 // This is set to 0.0 if memory tracing is not enabled. 86 double max_heapbytes_used; 87 88 // Keep track of arguments to compute asymptotic complexity 89 BigO complexity; 90 BigOFunc* complexity_lambda; 91 int complexity_n; 92 93 // Inform print function whether the current run is a complexity report 94 bool report_big_o; 95 bool report_rms; 96 }; 97 98 // Construct a BenchmarkReporter with the output stream set to 'std::cout' 99 // and the error stream set to 'std::cerr' 100 BenchmarkReporter(); 101 102 // Called once for every suite of benchmarks run. 103 // The parameter "context" contains information that the 104 // reporter may wish to use when generating its report, for example the 105 // platform under which the benchmarks are running. The benchmark run is 106 // never started if this function returns false, allowing the reporter 107 // to skip runs based on the context information. 108 virtual bool ReportContext(const Context& context) = 0; 109 110 // Called once for each group of benchmark runs, gives information about 111 // cpu-time and heap memory usage during the benchmark run. If the group 112 // of runs contained more than two entries then 'report' contains additional 113 // elements representing the mean and standard deviation of those runs. 114 // Additionally if this group of runs was the last in a family of benchmarks 115 // 'reports' contains additional entries representing the asymptotic 116 // complexity and RMS of that benchmark family. 117 virtual void ReportRuns(const std::vector<Run>& report) = 0; 118 119 // Called once and only once after ever group of benchmarks is run and 120 // reported. 121 virtual void Finalize() {} 122 123 // REQUIRES: The object referenced by 'out' is valid for the lifetime 124 // of the reporter. 125 void SetOutputStream(std::ostream* out) { 126 assert(out); 127 output_stream_ = out; 128 } 129 130 // REQUIRES: The object referenced by 'err' is valid for the lifetime 131 // of the reporter. 132 void SetErrorStream(std::ostream* err) { 133 assert(err); 134 error_stream_ = err; 135 } 136 137 std::ostream& GetOutputStream() const { return *output_stream_; } 138 139 std::ostream& GetErrorStream() const { return *error_stream_; } 140 141 virtual ~BenchmarkReporter(); 142 143 // Write a human readable string to 'out' representing the specified 144 // 'context'. 145 // REQUIRES: 'out' is non-null. 146 static void PrintBasicContext(std::ostream* out, Context const& context); 147 148 private: 149 std::ostream* output_stream_; 150 std::ostream* error_stream_; 151}; 152 153// Simple reporter that outputs benchmark data to the console. This is the 154// default reporter used by RunSpecifiedBenchmarks(). 155class ConsoleReporter : public BenchmarkReporter { 156 public: 157 enum OutputOptions { OO_None, OO_Color }; 158 explicit ConsoleReporter(OutputOptions color_output = OO_Color) 159 : name_field_width_(0), color_output_(color_output == OO_Color) {} 160 161 virtual bool ReportContext(const Context& context); 162 virtual void ReportRuns(const std::vector<Run>& reports); 163 164 protected: 165 virtual void PrintRunData(const Run& report); 166 size_t name_field_width_; 167 168 private: 169 bool color_output_; 170}; 171 172class JSONReporter : public BenchmarkReporter { 173 public: 174 JSONReporter() : first_report_(true) {} 175 virtual bool ReportContext(const Context& context); 176 virtual void ReportRuns(const std::vector<Run>& reports); 177 virtual void Finalize(); 178 179 private: 180 void PrintRunData(const Run& report); 181 182 bool first_report_; 183}; 184 185class CSVReporter : public BenchmarkReporter { 186 public: 187 virtual bool ReportContext(const Context& context); 188 virtual void ReportRuns(const std::vector<Run>& reports); 189 190 private: 191 void PrintRunData(const Run& report); 192}; 193 194inline const char* GetTimeUnitString(TimeUnit unit) { 195 switch (unit) { 196 case kMillisecond: 197 return "ms"; 198 case kMicrosecond: 199 return "us"; 200 case kNanosecond: 201 default: 202 return "ns"; 203 } 204} 205 206inline double GetTimeUnitMultiplier(TimeUnit unit) { 207 switch (unit) { 208 case kMillisecond: 209 return 1e3; 210 case kMicrosecond: 211 return 1e6; 212 case kNanosecond: 213 default: 214 return 1e9; 215 } 216} 217 218} // end namespace benchmark 219#endif // BENCHMARK_REPORTER_H_ 220