1 2#undef NDEBUG 3#include <utility> 4 5#include "benchmark/benchmark.h" 6#include "output_test.h" 7 8// ========================================================================= // 9// ---------------------- Testing Prologue Output -------------------------- // 10// ========================================================================= // 11 12ADD_CASES(TC_ConsoleOut, {{"^Benchmark %s Time %s CPU %s Iterations$", MR_Next}, 13 {"^[-]+$", MR_Next}}); 14ADD_CASES(TC_CSVOut, 15 {{"name,iterations,real_time,cpu_time,time_unit,bytes_per_second," 16 "items_per_second,label,error_occurred,error_message"}}); 17 18// ========================================================================= // 19// ------------------------ Testing Basic Output --------------------------- // 20// ========================================================================= // 21 22void BM_basic(benchmark::State& state) { 23 while (state.KeepRunning()) { 24 } 25} 26BENCHMARK(BM_basic); 27 28ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}}); 29ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"}, 30 {"\"iterations\": %int,$", MR_Next}, 31 {"\"real_time\": %int,$", MR_Next}, 32 {"\"cpu_time\": %int,$", MR_Next}, 33 {"\"time_unit\": \"ns\"$", MR_Next}, 34 {"}", MR_Next}}); 35ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}}); 36 37// ========================================================================= // 38// ------------------------ Testing Bytes per Second Output ---------------- // 39// ========================================================================= // 40 41void BM_bytes_per_second(benchmark::State& state) { 42 while (state.KeepRunning()) { 43 } 44 state.SetBytesProcessed(1); 45} 46BENCHMARK(BM_bytes_per_second); 47 48ADD_CASES(TC_ConsoleOut, 49 {{"^BM_bytes_per_second %console_report +%floatB/s$"}}); 50ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"}, 51 {"\"iterations\": %int,$", MR_Next}, 52 {"\"real_time\": %int,$", MR_Next}, 53 {"\"cpu_time\": %int,$", MR_Next}, 54 {"\"time_unit\": \"ns\",$", MR_Next}, 55 {"\"bytes_per_second\": %int$", MR_Next}, 56 {"}", MR_Next}}); 57ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}}); 58 59// ========================================================================= // 60// ------------------------ Testing Items per Second Output ---------------- // 61// ========================================================================= // 62 63void BM_items_per_second(benchmark::State& state) { 64 while (state.KeepRunning()) { 65 } 66 state.SetItemsProcessed(1); 67} 68BENCHMARK(BM_items_per_second); 69 70ADD_CASES(TC_ConsoleOut, 71 {{"^BM_items_per_second %console_report +%float items/s$"}}); 72ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"}, 73 {"\"iterations\": %int,$", MR_Next}, 74 {"\"real_time\": %int,$", MR_Next}, 75 {"\"cpu_time\": %int,$", MR_Next}, 76 {"\"time_unit\": \"ns\",$", MR_Next}, 77 {"\"items_per_second\": %int$", MR_Next}, 78 {"}", MR_Next}}); 79ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}}); 80 81// ========================================================================= // 82// ------------------------ Testing Label Output --------------------------- // 83// ========================================================================= // 84 85void BM_label(benchmark::State& state) { 86 while (state.KeepRunning()) { 87 } 88 state.SetLabel("some label"); 89} 90BENCHMARK(BM_label); 91 92ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}}); 93ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"}, 94 {"\"iterations\": %int,$", MR_Next}, 95 {"\"real_time\": %int,$", MR_Next}, 96 {"\"cpu_time\": %int,$", MR_Next}, 97 {"\"time_unit\": \"ns\",$", MR_Next}, 98 {"\"label\": \"some label\"$", MR_Next}, 99 {"}", MR_Next}}); 100ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some " 101 "label\"%csv_label_report_end$"}}); 102 103// ========================================================================= // 104// ------------------------ Testing Error Output --------------------------- // 105// ========================================================================= // 106 107void BM_error(benchmark::State& state) { 108 state.SkipWithError("message"); 109 while (state.KeepRunning()) { 110 } 111} 112BENCHMARK(BM_error); 113ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}}); 114ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"}, 115 {"\"error_occurred\": true,$", MR_Next}, 116 {"\"error_message\": \"message\",$", MR_Next}}); 117 118ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}}); 119 120// ========================================================================= // 121// ------------------------ Testing No Arg Name Output ----------------------- 122// // 123// ========================================================================= // 124 125void BM_no_arg_name(benchmark::State& state) { 126 while (state.KeepRunning()) { 127 } 128} 129BENCHMARK(BM_no_arg_name)->Arg(3); 130ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}}); 131ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}}); 132ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}}); 133 134// ========================================================================= // 135// ------------------------ Testing Arg Name Output ----------------------- // 136// ========================================================================= // 137 138void BM_arg_name(benchmark::State& state) { 139 while (state.KeepRunning()) { 140 } 141} 142BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3); 143ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}}); 144ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}}); 145ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}}); 146 147// ========================================================================= // 148// ------------------------ Testing Arg Names Output ----------------------- // 149// ========================================================================= // 150 151void BM_arg_names(benchmark::State& state) { 152 while (state.KeepRunning()) { 153 } 154} 155BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"}); 156ADD_CASES(TC_ConsoleOut, 157 {{"^BM_arg_names/first:2/5/third:4 %console_report$"}}); 158ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}}); 159ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}}); 160 161// ========================================================================= // 162// ----------------------- Testing Complexity Output ----------------------- // 163// ========================================================================= // 164 165void BM_Complexity_O1(benchmark::State& state) { 166 while (state.KeepRunning()) { 167 } 168 state.SetComplexityN(state.range(0)); 169} 170BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1); 171SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"}, 172 {"%RMS", "[ ]*[0-9]+ %"}}); 173ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"}, 174 {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}}); 175 176// ========================================================================= // 177// ----------------------- Testing Aggregate Output ------------------------ // 178// ========================================================================= // 179 180// Test that non-aggregate data is printed by default 181void BM_Repeat(benchmark::State& state) { 182 while (state.KeepRunning()) { 183 } 184} 185BENCHMARK(BM_Repeat)->Repetitions(3); 186ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"}, 187 {"^BM_Repeat/repeats:3 %console_report$"}, 188 {"^BM_Repeat/repeats:3 %console_report$"}, 189 {"^BM_Repeat/repeats:3_mean %console_report$"}, 190 {"^BM_Repeat/repeats:3_stddev %console_report$"}}); 191ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"}, 192 {"\"name\": \"BM_Repeat/repeats:3\",$"}, 193 {"\"name\": \"BM_Repeat/repeats:3\",$"}, 194 {"\"name\": \"BM_Repeat/repeats:3_mean\",$"}, 195 {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}}); 196ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"}, 197 {"^\"BM_Repeat/repeats:3\",%csv_report$"}, 198 {"^\"BM_Repeat/repeats:3\",%csv_report$"}, 199 {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"}, 200 {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}}); 201 202// Test that a non-repeated test still prints non-aggregate results even when 203// only-aggregate reports have been requested 204void BM_RepeatOnce(benchmark::State& state) { 205 while (state.KeepRunning()) { 206 } 207} 208BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly(); 209ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}}); 210ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}}); 211ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}}); 212 213// Test that non-aggregate data is not reported 214void BM_SummaryRepeat(benchmark::State& state) { 215 while (state.KeepRunning()) { 216 } 217} 218BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly(); 219ADD_CASES(TC_ConsoleOut, 220 {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 221 {"^BM_SummaryRepeat/repeats:3_mean %console_report$"}, 222 {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}}); 223ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 224 {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"}, 225 {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}}); 226ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not}, 227 {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"}, 228 {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}}); 229 230void BM_RepeatTimeUnit(benchmark::State& state) { 231 while (state.KeepRunning()) { 232 } 233} 234BENCHMARK(BM_RepeatTimeUnit) 235 ->Repetitions(3) 236 ->ReportAggregatesOnly() 237 ->Unit(benchmark::kMicrosecond); 238ADD_CASES(TC_ConsoleOut, 239 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 240 {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"}, 241 {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}}); 242ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 243 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"}, 244 {"\"time_unit\": \"us\",?$"}, 245 {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"}, 246 {"\"time_unit\": \"us\",?$"}}); 247ADD_CASES(TC_CSVOut, 248 {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not}, 249 {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"}, 250 {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}}); 251 252// ========================================================================= // 253// --------------------------- TEST CASES END ------------------------------ // 254// ========================================================================= // 255 256int main(int argc, char* argv[]) { RunOutputTests(argc, argv); } 257