1872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// Copyright 2016 Ismael Jimenez Martinez. All rights reserved. 2872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// 3872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// Licensed under the Apache License, Version 2.0 (the "License"); 4872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// you may not use this file except in compliance with the License. 5872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// You may obtain a copy of the License at 6872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// 7872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// http://www.apache.org/licenses/LICENSE-2.0 8872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// 9872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// Unless required by applicable law or agreed to in writing, software 10872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// distributed under the License is distributed on an "AS IS" BASIS, 11872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 12872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// See the License for the specific language governing permissions and 13872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// limitations under the License. 14872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael 15872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// Source project : https://github.com/ismaelJimenez/cpp.leastsq 16290bd60289ef571875415cf82be805f9a446c6a9Ismael// Adapted to be used with google benchmark 17872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael 18c1c7d33279b463088550986fe6f311a3ad2faa2eIsmael#include "benchmark/benchmark_api.h" 19c1c7d33279b463088550986fe6f311a3ad2faa2eIsmael 2022cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael#include <algorithm> 2122cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael#include <cmath> 22ac05c045335d3e32ec75e3aae930ecc1c6533212Ismael#include "check.h" 2322cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael#include "complexity.h" 241b263fe6d906bb0854b84247f1b395bbacd3b88eEric#include "stat.h" 25872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael 262f61f8aee0bc8b09429fce8b7d2718f805ed18acIsmaelnamespace benchmark { 27867f9145a0a45f8b993cec8b48309c19391acaa0Ismael 28872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael// Internal function to calculate the different scalability forms 29867f9145a0a45f8b993cec8b48309c19391acaa0IsmaelBigOFunc* FittingCurve(BigO complexity) { 30087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael switch (complexity) { 3122cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case oN: 32240ba4e64eb46e1f5acbafaadae34c2b2ca701ebIsmael return [](int n) -> double { return n; }; 3322cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case oNSquared: 34c04f703ab499058c62d6e3c4e05c11d3cb1e8781Eric Fiselier return [](int n) -> double { return std::pow(n, 2); }; 3522cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case oNCubed: 36c04f703ab499058c62d6e3c4e05c11d3cb1e8781Eric Fiselier return [](int n) -> double { return std::pow(n, 3); }; 3722cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case oLogN: 383fdd76bd14ff122c6881d7f15ec5cb2629241e7aIsmael return [](int n) { return std::log2(n); }; 3922cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case oNLogN: 403fdd76bd14ff122c6881d7f15ec5cb2629241e7aIsmael return [](int n) { return n * std::log2(n); }; 4122cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case o1: 4222cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael default: 43240ba4e64eb46e1f5acbafaadae34c2b2ca701ebIsmael return [](int) { return 1.0; }; 44087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael } 45087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael} 46087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael 47171588561112744263caa5847847e76e9bbde562Ismael// Function to return an string for the calculated complexity 482f61f8aee0bc8b09429fce8b7d2718f805ed18acIsmaelstd::string GetBigOString(BigO complexity) { 49d577987fd76595cb52602bd75b2866886e95b0f2Ismael switch (complexity) { 5022cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case oN: 5122cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael return "N"; 5222cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case oNSquared: 5322cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael return "N^2"; 5422cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case oNCubed: 5522cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael return "N^3"; 5622cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case oLogN: 5722cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael return "lgN"; 5822cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case oNLogN: 5922cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael return "NlgN"; 6022cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael case o1: 6122cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael return "(1)"; 6222cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael default: 6322cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael return "f(N)"; 64d577987fd76595cb52602bd75b2866886e95b0f2Ismael } 65872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael} 66872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael 6722cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael// Find the coefficient for the high-order term in the running time, by 6822cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael// minimizing the sum of squares of relative error, for the fitting curve 69171588561112744263caa5847847e76e9bbde562Ismael// given by the lambda expresion. 70087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael// - n : Vector containing the size of the benchmark tests. 71087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael// - time : Vector containing the times for the benchmark tests. 72087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael// - fitting_curve : lambda expresion (e.g. [](int n) {return n; };). 73171588561112744263caa5847847e76e9bbde562Ismael 742440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon// For a deeper explanation on the algorithm logic, look the README file at 752440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon// http://github.com/ismaelJimenez/Minimal-Cpp-Least-Squared-Fit 76872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael 77867f9145a0a45f8b993cec8b48309c19391acaa0IsmaelLeastSq MinimalLeastSq(const std::vector<int>& n, 78867f9145a0a45f8b993cec8b48309c19391acaa0Ismael const std::vector<double>& time, 79867f9145a0a45f8b993cec8b48309c19391acaa0Ismael BigOFunc* fitting_curve) { 8037ab858e4b245a49805b01358655fab069474a7cIsmael double sigma_gn = 0.0; 8137ab858e4b245a49805b01358655fab069474a7cIsmael double sigma_gn_squared = 0.0; 8237ab858e4b245a49805b01358655fab069474a7cIsmael double sigma_time = 0.0; 8337ab858e4b245a49805b01358655fab069474a7cIsmael double sigma_time_gn = 0.0; 84d577987fd76595cb52602bd75b2866886e95b0f2Ismael 85d577987fd76595cb52602bd75b2866886e95b0f2Ismael // Calculate least square fitting parameter 86d577987fd76595cb52602bd75b2866886e95b0f2Ismael for (size_t i = 0; i < n.size(); ++i) { 87087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael double gn_i = fitting_curve(n[i]); 88d577987fd76595cb52602bd75b2866886e95b0f2Ismael sigma_gn += gn_i; 89d577987fd76595cb52602bd75b2866886e95b0f2Ismael sigma_gn_squared += gn_i * gn_i; 90d577987fd76595cb52602bd75b2866886e95b0f2Ismael sigma_time += time[i]; 91d577987fd76595cb52602bd75b2866886e95b0f2Ismael sigma_time_gn += time[i] * gn_i; 92d577987fd76595cb52602bd75b2866886e95b0f2Ismael } 93d577987fd76595cb52602bd75b2866886e95b0f2Ismael 94d577987fd76595cb52602bd75b2866886e95b0f2Ismael LeastSq result; 95867f9145a0a45f8b993cec8b48309c19391acaa0Ismael result.complexity = oLambda; 96d577987fd76595cb52602bd75b2866886e95b0f2Ismael 972440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon // Calculate complexity. 98087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael result.coef = sigma_time_gn / sigma_gn_squared; 99d577987fd76595cb52602bd75b2866886e95b0f2Ismael 100d577987fd76595cb52602bd75b2866886e95b0f2Ismael // Calculate RMS 10137ab858e4b245a49805b01358655fab069474a7cIsmael double rms = 0.0; 102d577987fd76595cb52602bd75b2866886e95b0f2Ismael for (size_t i = 0; i < n.size(); ++i) { 103087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael double fit = result.coef * fitting_curve(n[i]); 104d577987fd76595cb52602bd75b2866886e95b0f2Ismael rms += pow((time[i] - fit), 2); 105d577987fd76595cb52602bd75b2866886e95b0f2Ismael } 106d577987fd76595cb52602bd75b2866886e95b0f2Ismael 1072440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon // Normalized RMS by the mean of the observed values 108087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael double mean = sigma_time / n.size(); 1092440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon result.rms = sqrt(rms / n.size()) / mean; 110d577987fd76595cb52602bd75b2866886e95b0f2Ismael 111d577987fd76595cb52602bd75b2866886e95b0f2Ismael return result; 112872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael} 113872ff01a49390ccaf8ee5f13c18ae7be9cce8275Ismael 1142440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon// Find the coefficient for the high-order term in the running time, by 1152440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon// minimizing the sum of squares of relative error. 116ac05c045335d3e32ec75e3aae930ecc1c6533212Ismael// - n : Vector containing the size of the benchmark tests. 117ac05c045335d3e32ec75e3aae930ecc1c6533212Ismael// - time : Vector containing the times for the benchmark tests. 1182440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon// - complexity : If different than oAuto, the fitting curve will stick to 1192440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon// this one. If it is oAuto, it will be calculated the best 1202440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon// fitting curve. 1212440b752fd335d00349b6dd77d67e5a6401565fbDominic HamonLeastSq MinimalLeastSq(const std::vector<int>& n, 122332f677b8bec401641a2743ab5d741c13cc6811dDominic Hamon const std::vector<double>& time, const BigO complexity) { 123d577987fd76595cb52602bd75b2866886e95b0f2Ismael CHECK_EQ(n.size(), time.size()); 12422cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael CHECK_GE(n.size(), 2); // Do not compute fitting curve is less than two 12522cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael // benchmark runs are given 1262f61f8aee0bc8b09429fce8b7d2718f805ed18acIsmael CHECK_NE(complexity, oNone); 127d577987fd76595cb52602bd75b2866886e95b0f2Ismael 128087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael LeastSq best_fit; 129087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael 13022cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael if (complexity == oAuto) { 13122cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael std::vector<BigO> fit_curves = {oLogN, oN, oNLogN, oNSquared, oNCubed}; 132d577987fd76595cb52602bd75b2866886e95b0f2Ismael 1332440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon // Take o1 as default best fitting curve 134867f9145a0a45f8b993cec8b48309c19391acaa0Ismael best_fit = MinimalLeastSq(n, time, FittingCurve(o1)); 1352f61f8aee0bc8b09429fce8b7d2718f805ed18acIsmael best_fit.complexity = o1; 136d577987fd76595cb52602bd75b2866886e95b0f2Ismael 137d577987fd76595cb52602bd75b2866886e95b0f2Ismael // Compute all possible fitting curves and stick to the best one 138d577987fd76595cb52602bd75b2866886e95b0f2Ismael for (const auto& fit : fit_curves) { 139867f9145a0a45f8b993cec8b48309c19391acaa0Ismael LeastSq current_fit = MinimalLeastSq(n, time, FittingCurve(fit)); 1402440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon if (current_fit.rms < best_fit.rms) { 141d577987fd76595cb52602bd75b2866886e95b0f2Ismael best_fit = current_fit; 142087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael best_fit.complexity = fit; 1432440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon } 144d577987fd76595cb52602bd75b2866886e95b0f2Ismael } 145087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael } else { 146867f9145a0a45f8b993cec8b48309c19391acaa0Ismael best_fit = MinimalLeastSq(n, time, FittingCurve(complexity)); 147087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael best_fit.complexity = complexity; 148d577987fd76595cb52602bd75b2866886e95b0f2Ismael } 1492440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon 150087f0d3f1bc6610ceaa346f8e573dd23236cea08Ismael return best_fit; 1512440b752fd335d00349b6dd77d67e5a6401565fbDominic Hamon} 1522f61f8aee0bc8b09429fce8b7d2718f805ed18acIsmael 1531b263fe6d906bb0854b84247f1b395bbacd3b88eEricstd::vector<BenchmarkReporter::Run> ComputeStats( 15422cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael const std::vector<BenchmarkReporter::Run>& reports) { 1551b263fe6d906bb0854b84247f1b395bbacd3b88eEric typedef BenchmarkReporter::Run Run; 1561b263fe6d906bb0854b84247f1b395bbacd3b88eEric std::vector<Run> results; 1571b263fe6d906bb0854b84247f1b395bbacd3b88eEric 15822cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael auto error_count = 15922cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael std::count_if(reports.begin(), reports.end(), 16022cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael [](Run const& run) { return run.error_occurred; }); 1611b263fe6d906bb0854b84247f1b395bbacd3b88eEric 1621b263fe6d906bb0854b84247f1b395bbacd3b88eEric if (reports.size() - error_count < 2) { 1631b263fe6d906bb0854b84247f1b395bbacd3b88eEric // We don't report aggregated data if there was a single run. 1641b263fe6d906bb0854b84247f1b395bbacd3b88eEric return results; 1651b263fe6d906bb0854b84247f1b395bbacd3b88eEric } 1661b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Accumulators. 1671b263fe6d906bb0854b84247f1b395bbacd3b88eEric Stat1_d real_accumulated_time_stat; 1681b263fe6d906bb0854b84247f1b395bbacd3b88eEric Stat1_d cpu_accumulated_time_stat; 1691b263fe6d906bb0854b84247f1b395bbacd3b88eEric Stat1_d bytes_per_second_stat; 1701b263fe6d906bb0854b84247f1b395bbacd3b88eEric Stat1_d items_per_second_stat; 1711b263fe6d906bb0854b84247f1b395bbacd3b88eEric // All repetitions should be run with the same number of iterations so we 1721b263fe6d906bb0854b84247f1b395bbacd3b88eEric // can take this information from the first benchmark. 1731b263fe6d906bb0854b84247f1b395bbacd3b88eEric int64_t const run_iterations = reports.front().iterations; 1741b263fe6d906bb0854b84247f1b395bbacd3b88eEric 1751b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Populate the accumulators. 1761b263fe6d906bb0854b84247f1b395bbacd3b88eEric for (Run const& run : reports) { 1771b263fe6d906bb0854b84247f1b395bbacd3b88eEric CHECK_EQ(reports[0].benchmark_name, run.benchmark_name); 1781b263fe6d906bb0854b84247f1b395bbacd3b88eEric CHECK_EQ(run_iterations, run.iterations); 17922cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael if (run.error_occurred) continue; 1801b263fe6d906bb0854b84247f1b395bbacd3b88eEric real_accumulated_time_stat += 18122cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael Stat1_d(run.real_accumulated_time / run.iterations, run.iterations); 1821b263fe6d906bb0854b84247f1b395bbacd3b88eEric cpu_accumulated_time_stat += 18322cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael Stat1_d(run.cpu_accumulated_time / run.iterations, run.iterations); 1841b263fe6d906bb0854b84247f1b395bbacd3b88eEric items_per_second_stat += Stat1_d(run.items_per_second, run.iterations); 1851b263fe6d906bb0854b84247f1b395bbacd3b88eEric bytes_per_second_stat += Stat1_d(run.bytes_per_second, run.iterations); 1861b263fe6d906bb0854b84247f1b395bbacd3b88eEric } 1871b263fe6d906bb0854b84247f1b395bbacd3b88eEric 1881b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Get the data from the accumulator to BenchmarkReporter::Run's. 1891b263fe6d906bb0854b84247f1b395bbacd3b88eEric Run mean_data; 1901b263fe6d906bb0854b84247f1b395bbacd3b88eEric mean_data.benchmark_name = reports[0].benchmark_name + "_mean"; 1911b263fe6d906bb0854b84247f1b395bbacd3b88eEric mean_data.iterations = run_iterations; 19222cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael mean_data.real_accumulated_time = 19322cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael real_accumulated_time_stat.Mean() * run_iterations; 19422cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael mean_data.cpu_accumulated_time = 19522cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael cpu_accumulated_time_stat.Mean() * run_iterations; 1961b263fe6d906bb0854b84247f1b395bbacd3b88eEric mean_data.bytes_per_second = bytes_per_second_stat.Mean(); 1971b263fe6d906bb0854b84247f1b395bbacd3b88eEric mean_data.items_per_second = items_per_second_stat.Mean(); 1985aa385562739652eddfd018d84e5d43c5c4777b8Marek Kurdej mean_data.time_unit = reports[0].time_unit; 1991b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2001b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Only add label to mean/stddev if it is same for all runs 2011b263fe6d906bb0854b84247f1b395bbacd3b88eEric mean_data.report_label = reports[0].report_label; 2021b263fe6d906bb0854b84247f1b395bbacd3b88eEric for (std::size_t i = 1; i < reports.size(); i++) { 2031b263fe6d906bb0854b84247f1b395bbacd3b88eEric if (reports[i].report_label != reports[0].report_label) { 2041b263fe6d906bb0854b84247f1b395bbacd3b88eEric mean_data.report_label = ""; 2051b263fe6d906bb0854b84247f1b395bbacd3b88eEric break; 2061b263fe6d906bb0854b84247f1b395bbacd3b88eEric } 2071b263fe6d906bb0854b84247f1b395bbacd3b88eEric } 2081b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2091b263fe6d906bb0854b84247f1b395bbacd3b88eEric Run stddev_data; 2101b263fe6d906bb0854b84247f1b395bbacd3b88eEric stddev_data.benchmark_name = reports[0].benchmark_name + "_stddev"; 2111b263fe6d906bb0854b84247f1b395bbacd3b88eEric stddev_data.report_label = mean_data.report_label; 2121b263fe6d906bb0854b84247f1b395bbacd3b88eEric stddev_data.iterations = 0; 21322cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael stddev_data.real_accumulated_time = real_accumulated_time_stat.StdDev(); 21422cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael stddev_data.cpu_accumulated_time = cpu_accumulated_time_stat.StdDev(); 2151b263fe6d906bb0854b84247f1b395bbacd3b88eEric stddev_data.bytes_per_second = bytes_per_second_stat.StdDev(); 2161b263fe6d906bb0854b84247f1b395bbacd3b88eEric stddev_data.items_per_second = items_per_second_stat.StdDev(); 2175aa385562739652eddfd018d84e5d43c5c4777b8Marek Kurdej stddev_data.time_unit = reports[0].time_unit; 2181b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2191b263fe6d906bb0854b84247f1b395bbacd3b88eEric results.push_back(mean_data); 2201b263fe6d906bb0854b84247f1b395bbacd3b88eEric results.push_back(stddev_data); 2211b263fe6d906bb0854b84247f1b395bbacd3b88eEric return results; 2221b263fe6d906bb0854b84247f1b395bbacd3b88eEric} 2231b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2241b263fe6d906bb0854b84247f1b395bbacd3b88eEricstd::vector<BenchmarkReporter::Run> ComputeBigO( 22522cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael const std::vector<BenchmarkReporter::Run>& reports) { 2261b263fe6d906bb0854b84247f1b395bbacd3b88eEric typedef BenchmarkReporter::Run Run; 2271b263fe6d906bb0854b84247f1b395bbacd3b88eEric std::vector<Run> results; 2281b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2291b263fe6d906bb0854b84247f1b395bbacd3b88eEric if (reports.size() < 2) return results; 2301b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2311b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Accumulators. 2321b263fe6d906bb0854b84247f1b395bbacd3b88eEric std::vector<int> n; 2331b263fe6d906bb0854b84247f1b395bbacd3b88eEric std::vector<double> real_time; 2341b263fe6d906bb0854b84247f1b395bbacd3b88eEric std::vector<double> cpu_time; 2351b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2361b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Populate the accumulators. 2371b263fe6d906bb0854b84247f1b395bbacd3b88eEric for (const Run& run : reports) { 238885ca41cf835313eca052ad112608631685ae6f2Ismael CHECK_GT(run.complexity_n, 0) << "Did you forget to call SetComplexityN?"; 2391b263fe6d906bb0854b84247f1b395bbacd3b88eEric n.push_back(run.complexity_n); 24022cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael real_time.push_back(run.real_accumulated_time / run.iterations); 24122cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael cpu_time.push_back(run.cpu_accumulated_time / run.iterations); 2421b263fe6d906bb0854b84247f1b395bbacd3b88eEric } 2431b263fe6d906bb0854b84247f1b395bbacd3b88eEric 244867f9145a0a45f8b993cec8b48309c19391acaa0Ismael LeastSq result_cpu; 245867f9145a0a45f8b993cec8b48309c19391acaa0Ismael LeastSq result_real; 2461b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2471a633969b31b2a486bdfae80576c53f40293281eIsmael if (reports[0].complexity == oLambda) { 248867f9145a0a45f8b993cec8b48309c19391acaa0Ismael result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity_lambda); 249867f9145a0a45f8b993cec8b48309c19391acaa0Ismael result_real = MinimalLeastSq(n, real_time, reports[0].complexity_lambda); 2501a633969b31b2a486bdfae80576c53f40293281eIsmael } else { 2511a633969b31b2a486bdfae80576c53f40293281eIsmael result_cpu = MinimalLeastSq(n, cpu_time, reports[0].complexity); 2521a633969b31b2a486bdfae80576c53f40293281eIsmael result_real = MinimalLeastSq(n, real_time, result_cpu.complexity); 253867f9145a0a45f8b993cec8b48309c19391acaa0Ismael } 25422cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael std::string benchmark_name = 25522cb9d9ce0ff12219f5ca6c4a28124d11730e66fIsmael reports[0].benchmark_name.substr(0, reports[0].benchmark_name.find('/')); 2561b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2571b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Get the data from the accumulator to BenchmarkReporter::Run's. 2581b263fe6d906bb0854b84247f1b395bbacd3b88eEric Run big_o; 2591b263fe6d906bb0854b84247f1b395bbacd3b88eEric big_o.benchmark_name = benchmark_name + "_BigO"; 2601b263fe6d906bb0854b84247f1b395bbacd3b88eEric big_o.iterations = 0; 2611b263fe6d906bb0854b84247f1b395bbacd3b88eEric big_o.real_accumulated_time = result_real.coef; 2621b263fe6d906bb0854b84247f1b395bbacd3b88eEric big_o.cpu_accumulated_time = result_cpu.coef; 2631b263fe6d906bb0854b84247f1b395bbacd3b88eEric big_o.report_big_o = true; 2641b263fe6d906bb0854b84247f1b395bbacd3b88eEric big_o.complexity = result_cpu.complexity; 2651b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2661b263fe6d906bb0854b84247f1b395bbacd3b88eEric double multiplier = GetTimeUnitMultiplier(reports[0].time_unit); 2671b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2681b263fe6d906bb0854b84247f1b395bbacd3b88eEric // Only add label to mean/stddev if it is same for all runs 2691b263fe6d906bb0854b84247f1b395bbacd3b88eEric Run rms; 2701b263fe6d906bb0854b84247f1b395bbacd3b88eEric big_o.report_label = reports[0].report_label; 2711b263fe6d906bb0854b84247f1b395bbacd3b88eEric rms.benchmark_name = benchmark_name + "_RMS"; 2721b263fe6d906bb0854b84247f1b395bbacd3b88eEric rms.report_label = big_o.report_label; 2731b263fe6d906bb0854b84247f1b395bbacd3b88eEric rms.iterations = 0; 2741b263fe6d906bb0854b84247f1b395bbacd3b88eEric rms.real_accumulated_time = result_real.rms / multiplier; 2751b263fe6d906bb0854b84247f1b395bbacd3b88eEric rms.cpu_accumulated_time = result_cpu.rms / multiplier; 2761b263fe6d906bb0854b84247f1b395bbacd3b88eEric rms.report_rms = true; 2771b263fe6d906bb0854b84247f1b395bbacd3b88eEric rms.complexity = result_cpu.complexity; 2781b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2791b263fe6d906bb0854b84247f1b395bbacd3b88eEric results.push_back(big_o); 2801b263fe6d906bb0854b84247f1b395bbacd3b88eEric results.push_back(rms); 2811b263fe6d906bb0854b84247f1b395bbacd3b88eEric return results; 2821b263fe6d906bb0854b84247f1b395bbacd3b88eEric} 2831b263fe6d906bb0854b84247f1b395bbacd3b88eEric 2841ee11056c1f1117142af36dd3ac4df2c2e6ce1bbIsmael} // end namespace benchmark 285