1
2#undef NDEBUG
3#include <utility>
4
5#include "benchmark/benchmark.h"
6#include "output_test.h"
7
8// ========================================================================= //
9// ---------------------- Testing Prologue Output -------------------------- //
10// ========================================================================= //
11
12ADD_CASES(TC_ConsoleOut, {{"^Benchmark %s Time %s CPU %s Iterations$", MR_Next},
13                          {"^[-]+$", MR_Next}});
14ADD_CASES(TC_CSVOut,
15          {{"name,iterations,real_time,cpu_time,time_unit,bytes_per_second,"
16            "items_per_second,label,error_occurred,error_message"}});
17
18// ========================================================================= //
19// ------------------------ Testing Basic Output --------------------------- //
20// ========================================================================= //
21
22void BM_basic(benchmark::State& state) {
23  while (state.KeepRunning()) {
24  }
25}
26BENCHMARK(BM_basic);
27
28ADD_CASES(TC_ConsoleOut, {{"^BM_basic %console_report$"}});
29ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_basic\",$"},
30                       {"\"iterations\": %int,$", MR_Next},
31                       {"\"real_time\": %int,$", MR_Next},
32                       {"\"cpu_time\": %int,$", MR_Next},
33                       {"\"time_unit\": \"ns\"$", MR_Next},
34                       {"}", MR_Next}});
35ADD_CASES(TC_CSVOut, {{"^\"BM_basic\",%csv_report$"}});
36
37// ========================================================================= //
38// ------------------------ Testing Bytes per Second Output ---------------- //
39// ========================================================================= //
40
41void BM_bytes_per_second(benchmark::State& state) {
42  while (state.KeepRunning()) {
43  }
44  state.SetBytesProcessed(1);
45}
46BENCHMARK(BM_bytes_per_second);
47
48ADD_CASES(TC_ConsoleOut,
49          {{"^BM_bytes_per_second %console_report +%floatB/s$"}});
50ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_bytes_per_second\",$"},
51                       {"\"iterations\": %int,$", MR_Next},
52                       {"\"real_time\": %int,$", MR_Next},
53                       {"\"cpu_time\": %int,$", MR_Next},
54                       {"\"time_unit\": \"ns\",$", MR_Next},
55                       {"\"bytes_per_second\": %int$", MR_Next},
56                       {"}", MR_Next}});
57ADD_CASES(TC_CSVOut, {{"^\"BM_bytes_per_second\",%csv_bytes_report$"}});
58
59// ========================================================================= //
60// ------------------------ Testing Items per Second Output ---------------- //
61// ========================================================================= //
62
63void BM_items_per_second(benchmark::State& state) {
64  while (state.KeepRunning()) {
65  }
66  state.SetItemsProcessed(1);
67}
68BENCHMARK(BM_items_per_second);
69
70ADD_CASES(TC_ConsoleOut,
71          {{"^BM_items_per_second %console_report +%float items/s$"}});
72ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_items_per_second\",$"},
73                       {"\"iterations\": %int,$", MR_Next},
74                       {"\"real_time\": %int,$", MR_Next},
75                       {"\"cpu_time\": %int,$", MR_Next},
76                       {"\"time_unit\": \"ns\",$", MR_Next},
77                       {"\"items_per_second\": %int$", MR_Next},
78                       {"}", MR_Next}});
79ADD_CASES(TC_CSVOut, {{"^\"BM_items_per_second\",%csv_items_report$"}});
80
81// ========================================================================= //
82// ------------------------ Testing Label Output --------------------------- //
83// ========================================================================= //
84
85void BM_label(benchmark::State& state) {
86  while (state.KeepRunning()) {
87  }
88  state.SetLabel("some label");
89}
90BENCHMARK(BM_label);
91
92ADD_CASES(TC_ConsoleOut, {{"^BM_label %console_report some label$"}});
93ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_label\",$"},
94                       {"\"iterations\": %int,$", MR_Next},
95                       {"\"real_time\": %int,$", MR_Next},
96                       {"\"cpu_time\": %int,$", MR_Next},
97                       {"\"time_unit\": \"ns\",$", MR_Next},
98                       {"\"label\": \"some label\"$", MR_Next},
99                       {"}", MR_Next}});
100ADD_CASES(TC_CSVOut, {{"^\"BM_label\",%csv_label_report_begin\"some "
101                       "label\"%csv_label_report_end$"}});
102
103// ========================================================================= //
104// ------------------------ Testing Error Output --------------------------- //
105// ========================================================================= //
106
107void BM_error(benchmark::State& state) {
108  state.SkipWithError("message");
109  while (state.KeepRunning()) {
110  }
111}
112BENCHMARK(BM_error);
113ADD_CASES(TC_ConsoleOut, {{"^BM_error[ ]+ERROR OCCURRED: 'message'$"}});
114ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_error\",$"},
115                       {"\"error_occurred\": true,$", MR_Next},
116                       {"\"error_message\": \"message\",$", MR_Next}});
117
118ADD_CASES(TC_CSVOut, {{"^\"BM_error\",,,,,,,,true,\"message\"$"}});
119
120// ========================================================================= //
121// ------------------------ Testing No Arg Name Output -----------------------
122// //
123// ========================================================================= //
124
125void BM_no_arg_name(benchmark::State& state) {
126  while (state.KeepRunning()) {
127  }
128}
129BENCHMARK(BM_no_arg_name)->Arg(3);
130ADD_CASES(TC_ConsoleOut, {{"^BM_no_arg_name/3 %console_report$"}});
131ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_no_arg_name/3\",$"}});
132ADD_CASES(TC_CSVOut, {{"^\"BM_no_arg_name/3\",%csv_report$"}});
133
134// ========================================================================= //
135// ------------------------ Testing Arg Name Output ----------------------- //
136// ========================================================================= //
137
138void BM_arg_name(benchmark::State& state) {
139  while (state.KeepRunning()) {
140  }
141}
142BENCHMARK(BM_arg_name)->ArgName("first")->Arg(3);
143ADD_CASES(TC_ConsoleOut, {{"^BM_arg_name/first:3 %console_report$"}});
144ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_name/first:3\",$"}});
145ADD_CASES(TC_CSVOut, {{"^\"BM_arg_name/first:3\",%csv_report$"}});
146
147// ========================================================================= //
148// ------------------------ Testing Arg Names Output ----------------------- //
149// ========================================================================= //
150
151void BM_arg_names(benchmark::State& state) {
152  while (state.KeepRunning()) {
153  }
154}
155BENCHMARK(BM_arg_names)->Args({2, 5, 4})->ArgNames({"first", "", "third"});
156ADD_CASES(TC_ConsoleOut,
157          {{"^BM_arg_names/first:2/5/third:4 %console_report$"}});
158ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_arg_names/first:2/5/third:4\",$"}});
159ADD_CASES(TC_CSVOut, {{"^\"BM_arg_names/first:2/5/third:4\",%csv_report$"}});
160
161// ========================================================================= //
162// ----------------------- Testing Complexity Output ----------------------- //
163// ========================================================================= //
164
165void BM_Complexity_O1(benchmark::State& state) {
166  while (state.KeepRunning()) {
167  }
168  state.SetComplexityN(state.range(0));
169}
170BENCHMARK(BM_Complexity_O1)->Range(1, 1 << 18)->Complexity(benchmark::o1);
171SET_SUBSTITUTIONS({{"%bigOStr", "[ ]* %float \\([0-9]+\\)"},
172                   {"%RMS", "[ ]*[0-9]+ %"}});
173ADD_CASES(TC_ConsoleOut, {{"^BM_Complexity_O1_BigO %bigOStr %bigOStr[ ]*$"},
174                          {"^BM_Complexity_O1_RMS %RMS %RMS[ ]*$"}});
175
176// ========================================================================= //
177// ----------------------- Testing Aggregate Output ------------------------ //
178// ========================================================================= //
179
180// Test that non-aggregate data is printed by default
181void BM_Repeat(benchmark::State& state) {
182  while (state.KeepRunning()) {
183  }
184}
185BENCHMARK(BM_Repeat)->Repetitions(3);
186ADD_CASES(TC_ConsoleOut, {{"^BM_Repeat/repeats:3 %console_report$"},
187                          {"^BM_Repeat/repeats:3 %console_report$"},
188                          {"^BM_Repeat/repeats:3 %console_report$"},
189                          {"^BM_Repeat/repeats:3_mean %console_report$"},
190                          {"^BM_Repeat/repeats:3_stddev %console_report$"}});
191ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Repeat/repeats:3\",$"},
192                       {"\"name\": \"BM_Repeat/repeats:3\",$"},
193                       {"\"name\": \"BM_Repeat/repeats:3\",$"},
194                       {"\"name\": \"BM_Repeat/repeats:3_mean\",$"},
195                       {"\"name\": \"BM_Repeat/repeats:3_stddev\",$"}});
196ADD_CASES(TC_CSVOut, {{"^\"BM_Repeat/repeats:3\",%csv_report$"},
197                      {"^\"BM_Repeat/repeats:3\",%csv_report$"},
198                      {"^\"BM_Repeat/repeats:3\",%csv_report$"},
199                      {"^\"BM_Repeat/repeats:3_mean\",%csv_report$"},
200                      {"^\"BM_Repeat/repeats:3_stddev\",%csv_report$"}});
201
202// Test that a non-repeated test still prints non-aggregate results even when
203// only-aggregate reports have been requested
204void BM_RepeatOnce(benchmark::State& state) {
205  while (state.KeepRunning()) {
206  }
207}
208BENCHMARK(BM_RepeatOnce)->Repetitions(1)->ReportAggregatesOnly();
209ADD_CASES(TC_ConsoleOut, {{"^BM_RepeatOnce/repeats:1 %console_report$"}});
210ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_RepeatOnce/repeats:1\",$"}});
211ADD_CASES(TC_CSVOut, {{"^\"BM_RepeatOnce/repeats:1\",%csv_report$"}});
212
213// Test that non-aggregate data is not reported
214void BM_SummaryRepeat(benchmark::State& state) {
215  while (state.KeepRunning()) {
216  }
217}
218BENCHMARK(BM_SummaryRepeat)->Repetitions(3)->ReportAggregatesOnly();
219ADD_CASES(TC_ConsoleOut,
220          {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
221           {"^BM_SummaryRepeat/repeats:3_mean %console_report$"},
222           {"^BM_SummaryRepeat/repeats:3_stddev %console_report$"}});
223ADD_CASES(TC_JSONOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
224                       {"\"name\": \"BM_SummaryRepeat/repeats:3_mean\",$"},
225                       {"\"name\": \"BM_SummaryRepeat/repeats:3_stddev\",$"}});
226ADD_CASES(TC_CSVOut, {{".*BM_SummaryRepeat/repeats:3 ", MR_Not},
227                      {"^\"BM_SummaryRepeat/repeats:3_mean\",%csv_report$"},
228                      {"^\"BM_SummaryRepeat/repeats:3_stddev\",%csv_report$"}});
229
230void BM_RepeatTimeUnit(benchmark::State& state) {
231  while (state.KeepRunning()) {
232  }
233}
234BENCHMARK(BM_RepeatTimeUnit)
235    ->Repetitions(3)
236    ->ReportAggregatesOnly()
237    ->Unit(benchmark::kMicrosecond);
238ADD_CASES(TC_ConsoleOut,
239          {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
240           {"^BM_RepeatTimeUnit/repeats:3_mean %console_us_report$"},
241           {"^BM_RepeatTimeUnit/repeats:3_stddev %console_us_report$"}});
242ADD_CASES(TC_JSONOut, {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
243                       {"\"name\": \"BM_RepeatTimeUnit/repeats:3_mean\",$"},
244                       {"\"time_unit\": \"us\",?$"},
245                       {"\"name\": \"BM_RepeatTimeUnit/repeats:3_stddev\",$"},
246                       {"\"time_unit\": \"us\",?$"}});
247ADD_CASES(TC_CSVOut,
248          {{".*BM_RepeatTimeUnit/repeats:3 ", MR_Not},
249           {"^\"BM_RepeatTimeUnit/repeats:3_mean\",%csv_us_report$"},
250           {"^\"BM_RepeatTimeUnit/repeats:3_stddev\",%csv_us_report$"}});
251
252// ========================================================================= //
253// --------------------------- TEST CASES END ------------------------------ //
254// ========================================================================= //
255
256int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
257