1d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier#undef NDEBUG 2d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier#include <algorithm> 3d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier#include <cassert> 4d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier#include <cmath> 5d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier#include <cstdlib> 6d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier#include <vector> 7d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier#include "benchmark/benchmark.h" 8d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier#include "output_test.h" 9d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 10d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiseliernamespace { 11d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 12d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier#define ADD_COMPLEXITY_CASES(...) \ 13d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier int CONCAT(dummy, __LINE__) = AddComplexityTest(__VA_ARGS__) 14d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier 15d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierint AddComplexityTest(std::string big_o_test_name, std::string rms_test_name, 16d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier std::string big_o) { 17d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier SetSubstitutions({{"%bigo_name", big_o_test_name}, 18d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"%rms_name", rms_test_name}, 19d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"%bigo_str", "[ ]* %float " + big_o}, 20d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"%bigo", big_o}, 21d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"%rms", "[ ]*[0-9]+ %"}}); 22d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier AddCases( 23d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier TC_ConsoleOut, 24d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {{"^%bigo_name %bigo_str %bigo_str[ ]*$"}, 25d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"^%bigo_name", MR_Not}, // Assert we we didn't only matched a name. 26d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"^%rms_name %rms %rms[ ]*$", MR_Next}}); 27d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier AddCases(TC_JSONOut, {{"\"name\": \"%bigo_name\",$"}, 28d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"\"cpu_coefficient\": [0-9]+,$", MR_Next}, 29d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"\"real_coefficient\": [0-9]{1,5},$", MR_Next}, 30d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"\"big_o\": \"%bigo\",$", MR_Next}, 31d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"\"time_unit\": \"ns\"$", MR_Next}, 32d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"}", MR_Next}, 33d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"\"name\": \"%rms_name\",$"}, 34d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"\"rms\": %float$", MR_Next}, 35d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"}", MR_Next}}); 36d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier AddCases(TC_CSVOut, {{"^\"%bigo_name\",,%float,%float,%bigo,,,,,$"}, 37d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"^\"%bigo_name\"", MR_Not}, 38d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier {"^\"%rms_name\",,%float,%float,,,,,,$", MR_Next}}); 39d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier return 0; 40d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier} 41d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 42d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier} // end namespace 43d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 44d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// ========================================================================= // 45d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// --------------------------- Testing BigO O(1) --------------------------- // 46d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// ========================================================================= // 47d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 48d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiseliervoid BM_Complexity_O1(benchmark::State& state) { 49d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier while (state.KeepRunning()) { 50d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier for (int i = 0; i < 1024; ++i) { 51d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier benchmark::DoNotOptimize(&i); 52d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier } 53d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier } 5430b48cb1b3a0c4fcc1887259bd215ad8738d21b4Eric Fiselier state.SetComplexityN(state.range(0)); 55d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier} 56d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric FiselierBENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); 57d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric FiselierBENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(); 58d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric FiselierBENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity([](int) { 59d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier return 1.0; 60d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier}); 61d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier 62d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *big_o_1_test_name = "BM_Complexity_O1_BigO"; 63d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *rms_o_1_test_name = "BM_Complexity_O1_RMS"; 64d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *enum_big_o_1 = "\\([0-9]+\\)"; 65d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier// FIXME: Tolerate both '(1)' and 'lgN' as output when the complexity is auto 66d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier// deduced. 67f76a08728e783fea3a9204a6f85f5a622a48552dEric Fiselier// See https://github.com/google/benchmark/issues/272 68d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *auto_big_o_1 = "(\\([0-9]+\\))|(lgN)"; 69d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *lambda_big_o_1 = "f\\(N\\)"; 70d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 71d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// Add enum tests 72f76a08728e783fea3a9204a6f85f5a622a48552dEric FiselierADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, enum_big_o_1); 73d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 7430b48cb1b3a0c4fcc1887259bd215ad8738d21b4Eric Fiselier// Add auto enum tests 75f76a08728e783fea3a9204a6f85f5a622a48552dEric FiselierADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, auto_big_o_1); 7630b48cb1b3a0c4fcc1887259bd215ad8738d21b4Eric Fiselier 77d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// Add lambda tests 78f76a08728e783fea3a9204a6f85f5a622a48552dEric FiselierADD_COMPLEXITY_CASES(big_o_1_test_name, rms_o_1_test_name, lambda_big_o_1); 79d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 80d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// ========================================================================= // 81d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// --------------------------- Testing BigO O(N) --------------------------- // 82d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// ========================================================================= // 83d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 84d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselierstd::vector<int> ConstructRandomVector(int size) { 85d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier std::vector<int> v; 86d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier v.reserve(size); 87d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier for (int i = 0; i < size; ++i) { 88f76a08728e783fea3a9204a6f85f5a622a48552dEric Fiselier v.push_back(std::rand() % size); 89d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier } 90d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier return v; 91d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier} 92d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 93d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiseliervoid BM_Complexity_O_N(benchmark::State& state) { 9430b48cb1b3a0c4fcc1887259bd215ad8738d21b4Eric Fiselier auto v = ConstructRandomVector(state.range(0)); 95d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier const int item_not_in_vector = 96d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier state.range(0) * 2; // Test worst case scenario (item not in vector) 97d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier while (state.KeepRunning()) { 98d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier benchmark::DoNotOptimize(std::find(v.begin(), v.end(), item_not_in_vector)); 99d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier } 10030b48cb1b3a0c4fcc1887259bd215ad8738d21b4Eric Fiselier state.SetComplexityN(state.range(0)); 101d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier} 102d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric FiselierBENCHMARK(BM_Complexity_O_N) 103d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->RangeMultiplier(2) 104d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Range(1 << 10, 1 << 16) 105d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Complexity(benchmark::oN); 106d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric FiselierBENCHMARK(BM_Complexity_O_N) 107d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->RangeMultiplier(2) 108d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Range(1 << 10, 1 << 16) 109d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Complexity([](int n) -> double { return n; }); 110d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric FiselierBENCHMARK(BM_Complexity_O_N) 111d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->RangeMultiplier(2) 112d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Range(1 << 10, 1 << 16) 113d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Complexity(); 114d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier 115d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *big_o_n_test_name = "BM_Complexity_O_N_BigO"; 116d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *rms_o_n_test_name = "BM_Complexity_O_N_RMS"; 117d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *enum_auto_big_o_n = "N"; 118d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *lambda_big_o_n = "f\\(N\\)"; 119d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 120d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// Add enum tests 121f76a08728e783fea3a9204a6f85f5a622a48552dEric FiselierADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, enum_auto_big_o_n); 122d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 123d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// Add lambda tests 124f76a08728e783fea3a9204a6f85f5a622a48552dEric FiselierADD_COMPLEXITY_CASES(big_o_n_test_name, rms_o_n_test_name, lambda_big_o_n); 125d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 126d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// ========================================================================= // 127d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// ------------------------- Testing BigO O(N*lgN) ------------------------- // 128d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// ========================================================================= // 129d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 130d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselierstatic void BM_Complexity_O_N_log_N(benchmark::State& state) { 13130b48cb1b3a0c4fcc1887259bd215ad8738d21b4Eric Fiselier auto v = ConstructRandomVector(state.range(0)); 132d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier while (state.KeepRunning()) { 133d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier std::sort(v.begin(), v.end()); 134d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier } 13530b48cb1b3a0c4fcc1887259bd215ad8738d21b4Eric Fiselier state.SetComplexityN(state.range(0)); 136d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier} 137d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric FiselierBENCHMARK(BM_Complexity_O_N_log_N) 138d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->RangeMultiplier(2) 139d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Range(1 << 10, 1 << 16) 140d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Complexity(benchmark::oNLogN); 141d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric FiselierBENCHMARK(BM_Complexity_O_N_log_N) 142d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->RangeMultiplier(2) 143d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Range(1 << 10, 1 << 16) 144d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Complexity([](int n) { return n * std::log2(n); }); 145d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric FiselierBENCHMARK(BM_Complexity_O_N_log_N) 146d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->RangeMultiplier(2) 147d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Range(1 << 10, 1 << 16) 148d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier ->Complexity(); 149d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier 150d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *big_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_BigO"; 151d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *rms_o_n_lg_n_test_name = "BM_Complexity_O_N_log_N_RMS"; 152d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *enum_auto_big_o_n_lg_n = "NlgN"; 153d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierconst char *lambda_big_o_n_lg_n = "f\\(N\\)"; 154d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 155d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// Add enum tests 156d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric FiselierADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, 157d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier enum_auto_big_o_n_lg_n); 158d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 159d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// Add lambda tests 160d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric FiselierADD_COMPLEXITY_CASES(big_o_n_lg_n_test_name, rms_o_n_lg_n_test_name, 161d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselier lambda_big_o_n_lg_n); 162d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 163d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// ========================================================================= // 164d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// --------------------------- TEST CASES END ------------------------------ // 165d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier// ========================================================================= // 166d9b9ef75a8ada5f9caaf2f4984bfb8094ade2590Eric Fiselier 167d87eb99b8071fbd5af4d6d04ac864ada752e24bcEric Fiselierint main(int argc, char *argv[]) { RunOutputTests(argc, argv); } 168