benchmark_main.cpp revision 9edb3e004b487e08cbbb54f2af18b15241550513
17be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes/* 27be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * Copyright (C) 2012 The Android Open Source Project 37be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * 47be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * Licensed under the Apache License, Version 2.0 (the "License"); 57be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * you may not use this file except in compliance with the License. 67be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * You may obtain a copy of the License at 77be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * 87be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * http://www.apache.org/licenses/LICENSE-2.0 97be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * 107be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * Unless required by applicable law or agreed to in writing, software 117be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * distributed under the License is distributed on an "AS IS" BASIS, 127be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 137be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * See the License for the specific language governing permissions and 147be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * limitations under the License. 157be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes */ 167be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 177be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include "benchmark.h" 187be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 197be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include <regex.h> 207be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include <stdio.h> 217be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include <stdlib.h> 227be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 237be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include <string> 247be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include <map> 257be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 267be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic int64_t gBytesProcessed; 277be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic int64_t gBenchmarkTotalTimeNs; 287be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic int64_t gBenchmarkStartTimeNs; 297be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 307be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughestypedef std::map<std::string, ::testing::Benchmark*> BenchmarkMap; 317be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughestypedef BenchmarkMap::iterator BenchmarkMapIt; 327be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic BenchmarkMap gBenchmarks; 337be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 347be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic int Round(int n) { 357be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes int base = 1; 367be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes while (base*10 < n) { 377be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes base *= 10; 387be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 397be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (n < 2*base) { 407be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return 2*base; 417be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 427be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (n < 5*base) { 437be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return 5*base; 447be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 457be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return 10*base; 467be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 477be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 487be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic int64_t NanoTime() { 497be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes struct timespec t; 507be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes t.tv_sec = t.tv_nsec = 0; 517be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes clock_gettime(CLOCK_MONOTONIC, &t); 527be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return static_cast<int64_t>(t.tv_sec) * 1000000000LL + t.tv_nsec; 537be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 547be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 557be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesnamespace testing { 567be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 577be369d4c60e9df2316fdb6c73181a40020abef2Elliott HughesBenchmark* Benchmark::Arg(int arg) { 587be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes args_.push_back(arg); 597be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return this; 607be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 617be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 629edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughesconst char* Benchmark::Name() { 639edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes return name_; 649edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes} 659edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes 667be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesbool Benchmark::ShouldRun(int argc, char* argv[]) { 677be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (argc == 1) { 687be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return true; // With no arguments, we run all benchmarks. 697be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 707be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes // Otherwise, we interpret each argument as a regular expression and 717be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes // see if any of our benchmarks match. 727be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes for (int i = 1; i < argc; i++) { 737be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes regex_t re; 747be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (regcomp(&re, argv[i], 0) != 0) { 757be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fprintf(stderr, "couldn't compile \"%s\" as a regular expression!\n", argv[i]); 767be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes exit(EXIT_FAILURE); 777be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 787be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes int match = regexec(&re, name_, 0, NULL, 0); 797be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes regfree(&re); 807be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (match != REG_NOMATCH) { 817be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return true; 827be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 837be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 847be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return false; 857be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 867be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 877be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid Benchmark::Register(const char* name, void (*fn)(int), void (*fn_range)(int, int)) { 887be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes name_ = name; 897be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fn_ = fn; 907be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fn_range_ = fn_range; 917be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 927be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (fn_ == NULL && fn_range_ == NULL) { 937be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fprintf(stderr, "%s: missing function\n", name_); 947be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes exit(EXIT_FAILURE); 957be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 967be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 977be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarks.insert(std::make_pair(name, this)); 987be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 997be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1007be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid Benchmark::Run() { 1019edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes if (fn_ != NULL) { 1029edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes RunWithArg(0); 1039edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes } else { 1049edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes if (args_.empty()) { 1059edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes fprintf(stderr, "%s: no args!\n", name_); 1069edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes exit(EXIT_FAILURE); 1079edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes } 1089edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes for (size_t i = 0; i < args_.size(); ++i) { 1099edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes RunWithArg(args_[i]); 1109edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes } 1117be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1127be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1137be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1147be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid Benchmark::RunRepeatedlyWithArg(int iterations, int arg) { 1157be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBytesProcessed = 0; 1167be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkTotalTimeNs = 0; 1177be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkStartTimeNs = NanoTime(); 1187be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (fn_ != NULL) { 1197be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fn_(iterations); 1207be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } else { 1217be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fn_range_(iterations, arg); 1227be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1237be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarkStartTimeNs != 0) { 1247be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkTotalTimeNs += NanoTime() - gBenchmarkStartTimeNs; 1257be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1267be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1277be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1287be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid Benchmark::RunWithArg(int arg) { 1297be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes // run once in case it's expensive 1307be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes int iterations = 1; 1317be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes RunRepeatedlyWithArg(iterations, arg); 1327be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes while (gBenchmarkTotalTimeNs < 1e9 && iterations < 1e9) { 1337be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes int last = iterations; 1347be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarkTotalTimeNs/iterations == 0) { 1357be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes iterations = 1e9; 1367be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } else { 1377be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes iterations = 1e9 / (gBenchmarkTotalTimeNs/iterations); 1387be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1397be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes iterations = std::max(last + 1, std::min(iterations + iterations/2, 100*last)); 1407be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes iterations = Round(iterations); 1417be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes RunRepeatedlyWithArg(iterations, arg); 1427be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1437be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1447be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes char throughput[100]; 1457be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes throughput[0] = '\0'; 1467be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarkTotalTimeNs > 0 && gBytesProcessed > 0) { 1477be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes double mib_processed = static_cast<double>(gBytesProcessed)/1e6; 1487be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes double seconds = static_cast<double>(gBenchmarkTotalTimeNs)/1e9; 1497be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes snprintf(throughput, sizeof(throughput), " %8.2f MiB/s", mib_processed/seconds); 1507be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1517be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1527be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes char full_name[100]; 1537be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (fn_range_ != NULL) { 1547be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (arg >= (1<<20)) { 1557be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes snprintf(full_name, sizeof(full_name), "%s/%dM", name_, arg/(1<<20)); 1567be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } else if (arg >= (1<<10)) { 1577be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes snprintf(full_name, sizeof(full_name), "%s/%dK", name_, arg/(1<<10)); 1587be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } else { 1597be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes snprintf(full_name, sizeof(full_name), "%s/%d", name_, arg); 1607be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1617be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } else { 1627be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes snprintf(full_name, sizeof(full_name), "%s", name_); 1637be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1647be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1657be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes printf("%-20s %10lld %10lld%s\n", full_name, 1667be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes static_cast<int64_t>(iterations), gBenchmarkTotalTimeNs/iterations, throughput); 1677be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fflush(stdout); 1687be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1697be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1707be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} // namespace testing 1717be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1727be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid SetBenchmarkBytesProcessed(int64_t x) { 1737be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBytesProcessed = x; 1747be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1757be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1767be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid StopBenchmarkTiming() { 1777be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarkStartTimeNs != 0) { 1787be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkTotalTimeNs += NanoTime() - gBenchmarkStartTimeNs; 1797be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1807be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkStartTimeNs = 0; 1817be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1827be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1837be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid StartBenchmarkTiming() { 1847be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarkStartTimeNs == 0) { 1857be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkStartTimeNs = NanoTime(); 1867be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1877be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1887be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1897be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesint main(int argc, char* argv[]) { 1907be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarks.empty()) { 1919edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes fprintf(stderr, "No benchmarks registered!\n"); 1927be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes exit(EXIT_FAILURE); 1937be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1947be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1959edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes bool need_header = true; 1967be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes for (BenchmarkMapIt it = gBenchmarks.begin(); it != gBenchmarks.end(); ++it) { 1977be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes ::testing::Benchmark* b = it->second; 1987be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (b->ShouldRun(argc, argv)) { 1999edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes if (need_header) { 2009edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes printf("%-20s %10s %10s\n", "", "iterations", "ns/op"); 2019edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes fflush(stdout); 2029edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes need_header = false; 2039edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes } 2047be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes b->Run(); 2057be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 2067be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 2079edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes 2089edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes if (need_header) { 2099edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes fprintf(stderr, "No matching benchmarks!\n"); 2109edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes fprintf(stderr, "Available benchmarks:\n"); 2119edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes for (BenchmarkMapIt it = gBenchmarks.begin(); it != gBenchmarks.end(); ++it) { 2129edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes fprintf(stderr, " %s\n", it->second->Name()); 2139edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes } 2149edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes exit(EXIT_FAILURE); 2159edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes } 2169edb3e004b487e08cbbb54f2af18b15241550513Elliott Hughes 2177be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return 0; 2187be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 219