benchmark_main.cpp revision 7be369d4c60e9df2316fdb6c73181a40020abef2
17be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes/* 27be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * Copyright (C) 2012 The Android Open Source Project 37be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * 47be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * Licensed under the Apache License, Version 2.0 (the "License"); 57be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * you may not use this file except in compliance with the License. 67be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * You may obtain a copy of the License at 77be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * 87be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * http://www.apache.org/licenses/LICENSE-2.0 97be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * 107be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * Unless required by applicable law or agreed to in writing, software 117be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * distributed under the License is distributed on an "AS IS" BASIS, 127be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. 137be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * See the License for the specific language governing permissions and 147be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes * limitations under the License. 157be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes */ 167be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 177be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include "benchmark.h" 187be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 197be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include <regex.h> 207be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include <stdio.h> 217be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include <stdlib.h> 227be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 237be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include <string> 247be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes#include <map> 257be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 267be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic int64_t gBytesProcessed; 277be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic int64_t gBenchmarkTotalTimeNs; 287be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic int64_t gBenchmarkStartTimeNs; 297be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 307be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughestypedef std::map<std::string, ::testing::Benchmark*> BenchmarkMap; 317be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughestypedef BenchmarkMap::iterator BenchmarkMapIt; 327be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic BenchmarkMap gBenchmarks; 337be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 347be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic int Round(int n) { 357be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes int base = 1; 367be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes while (base*10 < n) { 377be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes base *= 10; 387be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 397be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (n < 2*base) { 407be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return 2*base; 417be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 427be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (n < 5*base) { 437be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return 5*base; 447be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 457be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return 10*base; 467be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 477be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 487be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesstatic int64_t NanoTime() { 497be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes struct timespec t; 507be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes t.tv_sec = t.tv_nsec = 0; 517be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes clock_gettime(CLOCK_MONOTONIC, &t); 527be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return static_cast<int64_t>(t.tv_sec) * 1000000000LL + t.tv_nsec; 537be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 547be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 557be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesnamespace testing { 567be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 577be369d4c60e9df2316fdb6c73181a40020abef2Elliott HughesBenchmark* Benchmark::Arg(int arg) { 587be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes args_.push_back(arg); 597be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return this; 607be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 617be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 627be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesbool Benchmark::ShouldRun(int argc, char* argv[]) { 637be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (argc == 1) { 647be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return true; // With no arguments, we run all benchmarks. 657be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 667be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes // Otherwise, we interpret each argument as a regular expression and 677be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes // see if any of our benchmarks match. 687be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes for (int i = 1; i < argc; i++) { 697be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes regex_t re; 707be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (regcomp(&re, argv[i], 0) != 0) { 717be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fprintf(stderr, "couldn't compile \"%s\" as a regular expression!\n", argv[i]); 727be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes exit(EXIT_FAILURE); 737be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 747be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes int match = regexec(&re, name_, 0, NULL, 0); 757be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes regfree(&re); 767be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (match != REG_NOMATCH) { 777be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return true; 787be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 797be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 807be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return false; 817be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 827be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 837be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid Benchmark::Register(const char* name, void (*fn)(int), void (*fn_range)(int, int)) { 847be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes name_ = name; 857be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fn_ = fn; 867be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fn_range_ = fn_range; 877be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 887be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (fn_ == NULL && fn_range_ == NULL) { 897be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fprintf(stderr, "%s: missing function\n", name_); 907be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes exit(EXIT_FAILURE); 917be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 927be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 937be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarks.insert(std::make_pair(name, this)); 947be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 957be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 967be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid Benchmark::Run() { 977be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (args_.empty()) { 987be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fprintf(stderr, "%s: no args!\n", name_); 997be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes exit(EXIT_FAILURE); 1007be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1017be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes for (size_t i = 0; i < args_.size(); ++i) { 1027be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes RunWithArg(args_[i]); 1037be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1047be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1057be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1067be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid Benchmark::RunRepeatedlyWithArg(int iterations, int arg) { 1077be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBytesProcessed = 0; 1087be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkTotalTimeNs = 0; 1097be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkStartTimeNs = NanoTime(); 1107be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (fn_ != NULL) { 1117be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fn_(iterations); 1127be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } else { 1137be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fn_range_(iterations, arg); 1147be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1157be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarkStartTimeNs != 0) { 1167be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkTotalTimeNs += NanoTime() - gBenchmarkStartTimeNs; 1177be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1187be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1197be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1207be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid Benchmark::RunWithArg(int arg) { 1217be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes // run once in case it's expensive 1227be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes int iterations = 1; 1237be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes RunRepeatedlyWithArg(iterations, arg); 1247be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes while (gBenchmarkTotalTimeNs < 1e9 && iterations < 1e9) { 1257be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes int last = iterations; 1267be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarkTotalTimeNs/iterations == 0) { 1277be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes iterations = 1e9; 1287be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } else { 1297be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes iterations = 1e9 / (gBenchmarkTotalTimeNs/iterations); 1307be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1317be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes iterations = std::max(last + 1, std::min(iterations + iterations/2, 100*last)); 1327be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes iterations = Round(iterations); 1337be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes RunRepeatedlyWithArg(iterations, arg); 1347be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1357be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1367be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes char throughput[100]; 1377be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes throughput[0] = '\0'; 1387be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarkTotalTimeNs > 0 && gBytesProcessed > 0) { 1397be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes double mib_processed = static_cast<double>(gBytesProcessed)/1e6; 1407be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes double seconds = static_cast<double>(gBenchmarkTotalTimeNs)/1e9; 1417be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes snprintf(throughput, sizeof(throughput), " %8.2f MiB/s", mib_processed/seconds); 1427be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1437be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1447be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes char full_name[100]; 1457be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (fn_range_ != NULL) { 1467be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (arg >= (1<<20)) { 1477be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes snprintf(full_name, sizeof(full_name), "%s/%dM", name_, arg/(1<<20)); 1487be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } else if (arg >= (1<<10)) { 1497be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes snprintf(full_name, sizeof(full_name), "%s/%dK", name_, arg/(1<<10)); 1507be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } else { 1517be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes snprintf(full_name, sizeof(full_name), "%s/%d", name_, arg); 1527be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1537be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } else { 1547be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes snprintf(full_name, sizeof(full_name), "%s", name_); 1557be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1567be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1577be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes printf("%-20s %10lld %10lld%s\n", full_name, 1587be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes static_cast<int64_t>(iterations), gBenchmarkTotalTimeNs/iterations, throughput); 1597be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fflush(stdout); 1607be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1617be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1627be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} // namespace testing 1637be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1647be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid SetBenchmarkBytesProcessed(int64_t x) { 1657be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBytesProcessed = x; 1667be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1677be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1687be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid StopBenchmarkTiming() { 1697be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarkStartTimeNs != 0) { 1707be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkTotalTimeNs += NanoTime() - gBenchmarkStartTimeNs; 1717be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1727be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkStartTimeNs = 0; 1737be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1747be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1757be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesvoid StartBenchmarkTiming() { 1767be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarkStartTimeNs == 0) { 1777be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes gBenchmarkStartTimeNs = NanoTime(); 1787be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1797be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 1807be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1817be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughesint main(int argc, char* argv[]) { 1827be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (gBenchmarks.empty()) { 1837be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fprintf(stderr, "no benchmarks!\n"); 1847be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes exit(EXIT_FAILURE); 1857be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1867be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1877be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes printf("%-20s %10s %10s\n", "", "iterations", "ns/op"); 1887be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes fflush(stdout); 1897be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes 1907be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes for (BenchmarkMapIt it = gBenchmarks.begin(); it != gBenchmarks.end(); ++it) { 1917be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes ::testing::Benchmark* b = it->second; 1927be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes if (b->ShouldRun(argc, argv)) { 1937be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes b->Run(); 1947be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1957be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes } 1967be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes return 0; 1977be369d4c60e9df2316fdb6c73181a40020abef2Elliott Hughes} 198