171c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// Copyright 2015 Google Inc. All rights reserved. 271c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// 371c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// Licensed under the Apache License, Version 2.0 (the "License"); 471c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// you may not use this file except in compliance with the License. 571c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// You may obtain a copy of the License at 671c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// 771c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// http://www.apache.org/licenses/LICENSE-2.0 871c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// 971c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// Unless required by applicable law or agreed to in writing, software 1071c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// distributed under the License is distributed on an "AS IS" BASIS, 1171c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 1271c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// See the License for the specific language governing permissions and 1371c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon// limitations under the License. 1471c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon 1571c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon#include "benchmark/reporter.h" 16867f9145a0a45f8b993cec8b48309c19391acaa0Ismael#include "complexity.h" 1771c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon 18525858e68797d053ab4c859528164974978162baEric Fiselier#include <algorithm> 1922cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael#include <cstdint> 2071c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon#include <iostream> 2171c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon#include <string> 221203b3cbe47ad772291fe520efb2a029687229edKai Wolf#include <tuple> 2371c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon#include <vector> 2471c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon 2571c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon#include "string_util.h" 26cba945e37dd8f336c7c8f5367f3c7d9498d5e09bEric#include "timers.h" 27a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag#include "check.h" 2871c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon 29680a399a9387c4290442919681cb5d420a751232Dominic Hamon// File format reference: http://edoceo.com/utilitas/csv-file-format. 30680a399a9387c4290442919681cb5d420a751232Dominic Hamon 3171c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamonnamespace benchmark { 3271c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon 331003a70e5fe80967fe465da54c2f924944ad9fa6Eric Fiseliernamespace { 341003a70e5fe80967fe465da54c2f924944ad9fa6Eric Fiselierstd::vector<std::string> elements = { 35332f677b8bec401641a2743ab5d741c13cc6811dDominic Hamon "name", "iterations", "real_time", "cpu_time", 36332f677b8bec401641a2743ab5d741c13cc6811dDominic Hamon "time_unit", "bytes_per_second", "items_per_second", "label", 37332f677b8bec401641a2743ab5d741c13cc6811dDominic Hamon "error_occurred", "error_message"}; 381003a70e5fe80967fe465da54c2f924944ad9fa6Eric Fiselier} 391003a70e5fe80967fe465da54c2f924944ad9fa6Eric Fiselier 4071c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamonbool CSVReporter::ReportContext(const Context& context) { 411b263fe6d906bb0854b84247f1b395bbacd3b88eEric PrintBasicContext(&GetErrorStream(), context); 42a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag return true; 43a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag} 4471c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon 45a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmagvoid CSVReporter::ReportRuns(const std::vector<Run> & reports) { 461b263fe6d906bb0854b84247f1b395bbacd3b88eEric std::ostream& Out = GetOutputStream(); 47a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag 48a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag if (!printed_header_) { 49a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag // save the names of all the user counters 50a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag for (const auto& run : reports) { 51a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag for (const auto& cnt : run.counters) { 52a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag user_counter_names_.insert(cnt.first); 53a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag } 54a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag } 55a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag 56a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag // print the header 57a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag for (auto B = elements.begin(); B != elements.end();) { 58a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag Out << *B++; 59a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag if (B != elements.end()) Out << ","; 60a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag } 61a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag for (auto B = user_counter_names_.begin(); B != user_counter_names_.end();) { 62a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag Out << ",\"" << *B++ << "\""; 63a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag } 64a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag Out << "\n"; 65a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag 66a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag printed_header_ = true; 67a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag } else { 68a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag // check that all the current counters are saved in the name set 69a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag for (const auto& run : reports) { 70a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag for (const auto& cnt : run.counters) { 71a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag CHECK(user_counter_names_.find(cnt.first) != user_counter_names_.end()) 72a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag << "All counters must be present in each run. " 73a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag << "Counter named \"" << cnt.first 74a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag << "\" was not in a run after being added to the header"; 75a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag } 76a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag } 77a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag } 78a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag 79a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag // print results for each run 80a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag for (const auto& run : reports) { 81a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag PrintRunData(run); 821003a70e5fe80967fe465da54c2f924944ad9fa6Eric Fiselier } 8371c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon 84b73dc22944cb933289bbdbf5bb6616dbfc50168fIsmael} 85b73dc22944cb933289bbdbf5bb6616dbfc50168fIsmael 86a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmagvoid CSVReporter::PrintRunData(const Run & run) { 875686bf1b38f8aa713267097d7c1944140f71b5d3Eric std::ostream& Out = GetOutputStream(); 88525858e68797d053ab4c859528164974978162baEric Fiselier 89525858e68797d053ab4c859528164974978162baEric Fiselier // Field with embedded double-quote characters must be doubled and the field 90525858e68797d053ab4c859528164974978162baEric Fiselier // delimited with double-quotes. 91525858e68797d053ab4c859528164974978162baEric Fiselier std::string name = run.benchmark_name; 92525858e68797d053ab4c859528164974978162baEric Fiselier ReplaceAll(&name, "\"", "\"\""); 935686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << '"' << name << "\","; 94525858e68797d053ab4c859528164974978162baEric Fiselier if (run.error_occurred) { 955686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << std::string(elements.size() - 3, ','); 965686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << "true,"; 97525858e68797d053ab4c859528164974978162baEric Fiselier std::string msg = run.error_message; 98525858e68797d053ab4c859528164974978162baEric Fiselier ReplaceAll(&msg, "\"", "\"\""); 995686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << '"' << msg << "\"\n"; 1001003a70e5fe80967fe465da54c2f924944ad9fa6Eric Fiselier return; 101525858e68797d053ab4c859528164974978162baEric Fiselier } 102525858e68797d053ab4c859528164974978162baEric Fiselier 103266b3bd635a37b28d6e92125c615d3e17f5022eaIsmael // Do not print iteration on bigO and RMS report 10422cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael if (!run.report_big_o && !run.report_rms) { 1055686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << run.iterations; 1062440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon } 1075686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << ","; 1082440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon 1091b263fe6d906bb0854b84247f1b395bbacd3b88eEric Out << run.GetAdjustedRealTime() << ","; 1101b263fe6d906bb0854b84247f1b395bbacd3b88eEric Out << run.GetAdjustedCPUTime() << ","; 1112440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon 112867f9145a0a45f8b993cec8b48309c19391acaa0Ismael // Do not print timeLabel on bigO and RMS report 11322cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael if (run.report_big_o) { 114867f9145a0a45f8b993cec8b48309c19391acaa0Ismael Out << GetBigOString(run.complexity); 11522cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael } else if (!run.report_rms) { 1161b263fe6d906bb0854b84247f1b395bbacd3b88eEric Out << GetTimeUnitString(run.time_unit); 1172440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon } 1185686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << ","; 11971c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon 12071c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon if (run.bytes_per_second > 0.0) { 1215686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << run.bytes_per_second; 12271c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon } 1235686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << ","; 12471c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon if (run.items_per_second > 0.0) { 1255686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << run.items_per_second; 12671c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon } 1275686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << ","; 12871c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon if (!run.report_label.empty()) { 129680a399a9387c4290442919681cb5d420a751232Dominic Hamon // Field with embedded double-quote characters must be doubled and the field 130680a399a9387c4290442919681cb5d420a751232Dominic Hamon // delimited with double-quotes. 131680a399a9387c4290442919681cb5d420a751232Dominic Hamon std::string label = run.report_label; 132680a399a9387c4290442919681cb5d420a751232Dominic Hamon ReplaceAll(&label, "\"", "\"\""); 1335686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << "\"" << label << "\""; 13471c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon } 13522cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael Out << ",,"; // for error_occurred and error_message 136a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag 137a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag // Print user counters 138a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag for (const auto &ucn : user_counter_names_) { 139a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag auto it = run.counters.find(ucn); 140ea019f3cd8cf8433416f1c42d5e0a7e18fcbcc6fJoao Paulo Magalhaes if(it == run.counters.end()) { 141ea019f3cd8cf8433416f1c42d5e0a7e18fcbcc6fJoao Paulo Magalhaes Out << ","; 142160770fd08f8da8167edf66f55a047af1bc6291fJoao Paulo Magalhaes } else { 143ea019f3cd8cf8433416f1c42d5e0a7e18fcbcc6fJoao Paulo Magalhaes Out << "," << it->second; 144ea019f3cd8cf8433416f1c42d5e0a7e18fcbcc6fJoao Paulo Magalhaes } 145a9a66c85bbfda1d744c267c5e5aa073ef3d1c1d5jpmag } 1465686bf1b38f8aa713267097d7c1944140f71b5d3Eric Out << '\n'; 14771c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon} 14871c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon 14971c41cde57d7c741fca05b07585966129cf2ac5bDominic Hamon} // end namespace benchmark 150