reporter.h revision 867f9145a0a45f8b993cec8b48309c19391acaa0
14242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// Copyright 2015 Google Inc. All rights reserved. 24242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// 34242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// Licensed under the Apache License, Version 2.0 (the "License"); 44242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// you may not use this file except in compliance with the License. 54242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// You may obtain a copy of the License at 64242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// 74242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// http://www.apache.org/licenses/LICENSE-2.0 84242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// 94242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// Unless required by applicable law or agreed to in writing, software 104242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// distributed under the License is distributed on an "AS IS" BASIS, 114242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 124242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// See the License for the specific language governing permissions and 134242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// limitations under the License. 144242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier#ifndef BENCHMARK_REPORTER_H_ 154242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier#define BENCHMARK_REPORTER_H_ 164242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 175686bf1b38f8aa713267097d7c1944140f71b5d3Eric#include <cassert> 185686bf1b38f8aa713267097d7c1944140f71b5d3Eric#include <iosfwd> 194242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier#include <string> 204242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier#include <utility> 214242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier#include <vector> 224242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 238e72846da4bde9cdc8162ac99ce4168c65154f7dEric Fiselier#include "benchmark_api.h" // For forward declaration of BenchmarkReporter 24df904a1980b148139e566028dad27e3bea828afdEric Fiselier 254242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiseliernamespace benchmark { 264242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 274242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// Interface for custom benchmark result printers. 284242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// By default, benchmark reports are printed to stdout. However an application 294242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// can control the destination of the reports by calling 304242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// RunSpecifiedBenchmarks and passing it a custom reporter object. 314242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// The reporter object must implement the following interface. 324242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselierclass BenchmarkReporter { 334242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier public: 344242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier struct Context { 354242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier int num_cpus; 364242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier double mhz_per_cpu; 374242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier bool cpu_scaling_enabled; 384242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 394242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier // The number of chars in the longest benchmark name. 404242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier size_t name_field_width; 414242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier }; 424242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 434242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier struct Run { 444242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier Run() : 4543017f8b1510a50855e6b0ea145a534d3d754068Eric Fiselier error_occurred(false), 464242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier iterations(1), 477c69b36078b5773fbd6b09b539a30400138607a7Kai Wolf time_unit(kNanosecond), 484242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier real_accumulated_time(0), 494242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier cpu_accumulated_time(0), 504242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier bytes_per_second(0), 514242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier items_per_second(0), 52b73dc22944cb933289bbdbf5bb6616dbfc50168fIsmael max_heapbytes_used(0), 53ac05c045335d3e32ec75e3aae930ecc1c6533212Ismael complexity(oNone), 5436a9ae197f220df65ef7ed1a21015a8fb4ef1eceIsmael complexity_n(0), 555f9823bd92b2a24da06fac7b43f6658ec20cc901Ismael report_big_o(false), 562e5c397b4829503a5cb023ac67d2a1f13ebda3aaIsmael report_rms(false) {} 574242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 584242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier std::string benchmark_name; 594242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier std::string report_label; // Empty if not set by benchmark. 6043017f8b1510a50855e6b0ea145a534d3d754068Eric Fiselier bool error_occurred; 6143017f8b1510a50855e6b0ea145a534d3d754068Eric Fiselier std::string error_message; 6243017f8b1510a50855e6b0ea145a534d3d754068Eric Fiselier 632cf277b6f65d882cc8d3010fc1c91deb5edb32baJean-Louis Leroy int64_t iterations; 647c69b36078b5773fbd6b09b539a30400138607a7Kai Wolf TimeUnit time_unit; 654242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier double real_accumulated_time; 664242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier double cpu_accumulated_time; 674242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 681b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Return a value representing the real time per iteration in the unit 691b263fe6d906bb0854b84247f1b395bbacd3b88eEric // specified by 'time_unit'. 701b263fe6d906bb0854b84247f1b395bbacd3b88eEric // NOTE: If 'iterations' is zero the returned value represents the 711b263fe6d906bb0854b84247f1b395bbacd3b88eEric // accumulated time. 721b263fe6d906bb0854b84247f1b395bbacd3b88eEric double GetAdjustedRealTime() const; 731b263fe6d906bb0854b84247f1b395bbacd3b88eEric 741b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Return a value representing the cpu time per iteration in the unit 751b263fe6d906bb0854b84247f1b395bbacd3b88eEric // specified by 'time_unit'. 761b263fe6d906bb0854b84247f1b395bbacd3b88eEric // NOTE: If 'iterations' is zero the returned value represents the 771b263fe6d906bb0854b84247f1b395bbacd3b88eEric // accumulated time. 781b263fe6d906bb0854b84247f1b395bbacd3b88eEric double GetAdjustedCPUTime() const; 791b263fe6d906bb0854b84247f1b395bbacd3b88eEric 804242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier // Zero if not set by benchmark. 814242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier double bytes_per_second; 824242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier double items_per_second; 834242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 844242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier // This is set to 0.0 if memory tracing is not enabled. 854242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier double max_heapbytes_used; 86867f9145a0a45f8b993cec8b48309c19391acaa0Ismael 87b73dc22944cb933289bbdbf5bb6616dbfc50168fIsmael // Keep track of arguments to compute asymptotic complexity 88867f9145a0a45f8b993cec8b48309c19391acaa0Ismael BigO complexity; 89867f9145a0a45f8b993cec8b48309c19391acaa0Ismael BigOFunc* complexity_lambda; 90867f9145a0a45f8b993cec8b48309c19391acaa0Ismael size_t complexity_n; 91867f9145a0a45f8b993cec8b48309c19391acaa0Ismael 92290bd60289ef571875415cf82be805f9a446c6a9Ismael // Inform print function whether the current run is a complexity report 935f9823bd92b2a24da06fac7b43f6658ec20cc901Ismael bool report_big_o; 942e5c397b4829503a5cb023ac67d2a1f13ebda3aaIsmael bool report_rms; 954242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier }; 964242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 975686bf1b38f8aa713267097d7c1944140f71b5d3Eric // Construct a BenchmarkReporter with the output stream set to 'std::cout' 985686bf1b38f8aa713267097d7c1944140f71b5d3Eric // and the error stream set to 'std::cerr' 995686bf1b38f8aa713267097d7c1944140f71b5d3Eric BenchmarkReporter(); 1005686bf1b38f8aa713267097d7c1944140f71b5d3Eric 1014242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier // Called once for every suite of benchmarks run. 1024242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier // The parameter "context" contains information that the 1034242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier // reporter may wish to use when generating its report, for example the 1044242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier // platform under which the benchmarks are running. The benchmark run is 1054242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier // never started if this function returns false, allowing the reporter 1064242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier // to skip runs based on the context information. 10720f1c0e2a8e692076dd7a5586f73ab8e2d726c12Eric Fiselier virtual bool ReportContext(const Context& context) = 0; 1084242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 1094242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier // Called once for each group of benchmark runs, gives information about 1101b263fe6d906bb0854b84247f1b395bbacd3b88eEric // cpu-time and heap memory usage during the benchmark run. If the group 1111b263fe6d906bb0854b84247f1b395bbacd3b88eEric // of runs contained more than two entries then 'report' contains additional 1121b263fe6d906bb0854b84247f1b395bbacd3b88eEric // elements representing the mean and standard deviation of those runs. 1131b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Additionally if this group of runs was the last in a family of benchmarks 1141b263fe6d906bb0854b84247f1b395bbacd3b88eEric // 'reports' contains additional entries representing the asymptotic 1151b263fe6d906bb0854b84247f1b395bbacd3b88eEric // complexity and RMS of that benchmark family. 11620f1c0e2a8e692076dd7a5586f73ab8e2d726c12Eric Fiselier virtual void ReportRuns(const std::vector<Run>& report) = 0; 117867f9145a0a45f8b993cec8b48309c19391acaa0Ismael 11820f1c0e2a8e692076dd7a5586f73ab8e2d726c12Eric Fiselier // Called once and only once after ever group of benchmarks is run and 11920f1c0e2a8e692076dd7a5586f73ab8e2d726c12Eric Fiselier // reported. 1201b263fe6d906bb0854b84247f1b395bbacd3b88eEric virtual void Finalize() {} 1214242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 1225686bf1b38f8aa713267097d7c1944140f71b5d3Eric // REQUIRES: The object referenced by 'out' is valid for the lifetime 1235686bf1b38f8aa713267097d7c1944140f71b5d3Eric // of the reporter. 1245686bf1b38f8aa713267097d7c1944140f71b5d3Eric void SetOutputStream(std::ostream* out) { 1255686bf1b38f8aa713267097d7c1944140f71b5d3Eric assert(out); 1265686bf1b38f8aa713267097d7c1944140f71b5d3Eric output_stream_ = out; 1275686bf1b38f8aa713267097d7c1944140f71b5d3Eric } 1285686bf1b38f8aa713267097d7c1944140f71b5d3Eric 1295686bf1b38f8aa713267097d7c1944140f71b5d3Eric // REQUIRES: The object referenced by 'err' is valid for the lifetime 1305686bf1b38f8aa713267097d7c1944140f71b5d3Eric // of the reporter. 1315686bf1b38f8aa713267097d7c1944140f71b5d3Eric void SetErrorStream(std::ostream* err) { 1325686bf1b38f8aa713267097d7c1944140f71b5d3Eric assert(err); 1335686bf1b38f8aa713267097d7c1944140f71b5d3Eric error_stream_ = err; 1345686bf1b38f8aa713267097d7c1944140f71b5d3Eric } 1355686bf1b38f8aa713267097d7c1944140f71b5d3Eric 1365686bf1b38f8aa713267097d7c1944140f71b5d3Eric std::ostream& GetOutputStream() const { 1375686bf1b38f8aa713267097d7c1944140f71b5d3Eric return *output_stream_; 1385686bf1b38f8aa713267097d7c1944140f71b5d3Eric } 1395686bf1b38f8aa713267097d7c1944140f71b5d3Eric 1405686bf1b38f8aa713267097d7c1944140f71b5d3Eric std::ostream& GetErrorStream() const { 1415686bf1b38f8aa713267097d7c1944140f71b5d3Eric return *error_stream_; 1425686bf1b38f8aa713267097d7c1944140f71b5d3Eric } 1435686bf1b38f8aa713267097d7c1944140f71b5d3Eric 1444242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier virtual ~BenchmarkReporter(); 1451b263fe6d906bb0854b84247f1b395bbacd3b88eEric 1461b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Write a human readable string to 'out' representing the specified 1471b263fe6d906bb0854b84247f1b395bbacd3b88eEric // 'context'. 1481b263fe6d906bb0854b84247f1b395bbacd3b88eEric // REQUIRES: 'out' is non-null. 1491b263fe6d906bb0854b84247f1b395bbacd3b88eEric static void PrintBasicContext(std::ostream* out, Context const& context); 1505686bf1b38f8aa713267097d7c1944140f71b5d3Eric 1515686bf1b38f8aa713267097d7c1944140f71b5d3Ericprivate: 1525686bf1b38f8aa713267097d7c1944140f71b5d3Eric std::ostream* output_stream_; 1535686bf1b38f8aa713267097d7c1944140f71b5d3Eric std::ostream* error_stream_; 1544242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier}; 1554242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 1564242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// Simple reporter that outputs benchmark data to the console. This is the 1574242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier// default reporter used by RunSpecifiedBenchmarks(). 1584242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselierclass ConsoleReporter : public BenchmarkReporter { 1594242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier public: 16020f1c0e2a8e692076dd7a5586f73ab8e2d726c12Eric Fiselier virtual bool ReportContext(const Context& context); 16120f1c0e2a8e692076dd7a5586f73ab8e2d726c12Eric Fiselier virtual void ReportRuns(const std::vector<Run>& reports); 1627c69b36078b5773fbd6b09b539a30400138607a7Kai Wolf 163867f9145a0a45f8b993cec8b48309c19391acaa0Ismael protected: 16420f1c0e2a8e692076dd7a5586f73ab8e2d726c12Eric Fiselier virtual void PrintRunData(const Run& report); 16520f1c0e2a8e692076dd7a5586f73ab8e2d726c12Eric Fiselier 16620f1c0e2a8e692076dd7a5586f73ab8e2d726c12Eric Fiselier size_t name_field_width_; 1674242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier}; 1684242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier 169f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselierclass JSONReporter : public BenchmarkReporter { 170f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselierpublic: 171f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselier JSONReporter() : first_report_(true) {} 172f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselier virtual bool ReportContext(const Context& context); 173f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselier virtual void ReportRuns(const std::vector<Run>& reports); 174f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselier virtual void Finalize(); 175f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselier 176f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselierprivate: 177f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselier void PrintRunData(const Run& report); 178f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselier 179f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselier bool first_report_; 180f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselier}; 181f65da9d58165c6ab626d6a55b9d47968510c72aeEric Fiselier 18271c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamonclass CSVReporter : public BenchmarkReporter { 18371c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamonpublic: 18471c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon virtual bool ReportContext(const Context& context); 18571c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon virtual void ReportRuns(const std::vector<Run>& reports); 18671c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon 18771c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamonprivate: 18871c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon void PrintRunData(const Run& report); 18971c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon}; 19071c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon 19102230445e065456371cea2a18d5fee6a33b0c1d9Eric Fiselierinline const char* GetTimeUnitString(TimeUnit unit) { 19202230445e065456371cea2a18d5fee6a33b0c1d9Eric Fiselier switch (unit) { 193867f9145a0a45f8b993cec8b48309c19391acaa0Ismael case kMillisecond: 194867f9145a0a45f8b993cec8b48309c19391acaa0Ismael return "ms"; 195867f9145a0a45f8b993cec8b48309c19391acaa0Ismael case kMicrosecond: 196867f9145a0a45f8b993cec8b48309c19391acaa0Ismael return "us"; 197867f9145a0a45f8b993cec8b48309c19391acaa0Ismael case kNanosecond: 198867f9145a0a45f8b993cec8b48309c19391acaa0Ismael default: 199867f9145a0a45f8b993cec8b48309c19391acaa0Ismael return "ns"; 20002230445e065456371cea2a18d5fee6a33b0c1d9Eric Fiselier } 20102230445e065456371cea2a18d5fee6a33b0c1d9Eric Fiselier} 20202230445e065456371cea2a18d5fee6a33b0c1d9Eric Fiselier 20302230445e065456371cea2a18d5fee6a33b0c1d9Eric Fiselierinline double GetTimeUnitMultiplier(TimeUnit unit) { 204867f9145a0a45f8b993cec8b48309c19391acaa0Ismael switch (unit) { 205867f9145a0a45f8b993cec8b48309c19391acaa0Ismael case kMillisecond: 206867f9145a0a45f8b993cec8b48309c19391acaa0Ismael return 1e3; 207867f9145a0a45f8b993cec8b48309c19391acaa0Ismael case kMicrosecond: 208867f9145a0a45f8b993cec8b48309c19391acaa0Ismael return 1e6; 209867f9145a0a45f8b993cec8b48309c19391acaa0Ismael case kNanosecond: 210867f9145a0a45f8b993cec8b48309c19391acaa0Ismael default: 211867f9145a0a45f8b993cec8b48309c19391acaa0Ismael return 1e9; 21202230445e065456371cea2a18d5fee6a33b0c1d9Eric Fiselier } 21302230445e065456371cea2a18d5fee6a33b0c1d9Eric Fiselier} 21402230445e065456371cea2a18d5fee6a33b0c1d9Eric Fiselier 2154242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier} // end namespace benchmark 2164242f2f1d73d7800d95cfc2af0f3a3513dad2dc5Eric Fiselier#endif // BENCHMARK_REPORTER_H_ 217