1
2#undef NDEBUG
3
4#include "benchmark/benchmark.h"
5#include "output_test.h"
6
7// @todo: <jpmag> this checks the full output at once; the rule for
8// CounterSet1 was failing because it was not matching "^[-]+$".
9// @todo: <jpmag> check that the counters are vertically aligned.
10ADD_CASES(TC_ConsoleOut, {
11// keeping these lines long improves readability, so:
12// clang-format off
13    {"^[-]+$", MR_Next},
14    {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Bat %s Baz %s Foo %s Frob %s Lob$", MR_Next},
15    {"^[-]+$", MR_Next},
16    {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
17    {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
18    {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
19    {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
20    {"^BM_Counters_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
21    {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
22    {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
23    {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
24    {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
25    {"^BM_CounterRates_Tabular/threads:%int %console_report [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s [ ]*%hrfloat/s$", MR_Next},
26    {"^[-]+$", MR_Next},
27    {"^Benchmark %s Time %s CPU %s Iterations %s Bar %s Baz %s Foo$", MR_Next},
28    {"^[-]+$", MR_Next},
29    {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
30    {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
31    {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
32    {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
33    {"^BM_CounterSet0_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
34    {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
35    {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
36    {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
37    {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
38    {"^BM_CounterSet1_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
39    {"^[-]+$", MR_Next},
40    {"^Benchmark %s Time %s CPU %s Iterations %s Bat %s Baz %s Foo$", MR_Next},
41    {"^[-]+$", MR_Next},
42    {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
43    {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
44    {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
45    {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$", MR_Next},
46    {"^BM_CounterSet2_Tabular/threads:%int %console_report [ ]*%hrfloat [ ]*%hrfloat [ ]*%hrfloat$"},
47// clang-format on
48});
49ADD_CASES(TC_CSVOut, {{"%csv_header,"
50                       "\"Bar\",\"Bat\",\"Baz\",\"Foo\",\"Frob\",\"Lob\""}});
51
52// ========================================================================= //
53// ------------------------- Tabular Counters Output ----------------------- //
54// ========================================================================= //
55
56void BM_Counters_Tabular(benchmark::State& state) {
57  for (auto _ : state) {
58  }
59  namespace bm = benchmark;
60  state.counters.insert({
61    {"Foo",  { 1, bm::Counter::kAvgThreads}},
62    {"Bar",  { 2, bm::Counter::kAvgThreads}},
63    {"Baz",  { 4, bm::Counter::kAvgThreads}},
64    {"Bat",  { 8, bm::Counter::kAvgThreads}},
65    {"Frob", {16, bm::Counter::kAvgThreads}},
66    {"Lob",  {32, bm::Counter::kAvgThreads}},
67  });
68}
69BENCHMARK(BM_Counters_Tabular)->ThreadRange(1, 16);
70ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_Counters_Tabular/threads:%int\",$"},
71                       {"\"iterations\": %int,$", MR_Next},
72                       {"\"real_time\": %float,$", MR_Next},
73                       {"\"cpu_time\": %float,$", MR_Next},
74                       {"\"time_unit\": \"ns\",$", MR_Next},
75                       {"\"Bar\": %float,$", MR_Next},
76                       {"\"Bat\": %float,$", MR_Next},
77                       {"\"Baz\": %float,$", MR_Next},
78                       {"\"Foo\": %float,$", MR_Next},
79                       {"\"Frob\": %float,$", MR_Next},
80                       {"\"Lob\": %float$", MR_Next},
81                       {"}", MR_Next}});
82ADD_CASES(TC_CSVOut, {{"^\"BM_Counters_Tabular/threads:%int\",%csv_report,"
83                       "%float,%float,%float,%float,%float,%float$"}});
84// VS2013 does not allow this function to be passed as a lambda argument
85// to CHECK_BENCHMARK_RESULTS()
86void CheckTabular(Results const& e) {
87  CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 1);
88  CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 2);
89  CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 4);
90  CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 8);
91  CHECK_COUNTER_VALUE(e, int, "Frob", EQ, 16);
92  CHECK_COUNTER_VALUE(e, int, "Lob", EQ, 32);
93}
94CHECK_BENCHMARK_RESULTS("BM_Counters_Tabular/threads:%int", &CheckTabular);
95
96// ========================================================================= //
97// -------------------- Tabular+Rate Counters Output ----------------------- //
98// ========================================================================= //
99
100void BM_CounterRates_Tabular(benchmark::State& state) {
101  for (auto _ : state) {
102  }
103  namespace bm = benchmark;
104  state.counters.insert({
105    {"Foo",  { 1, bm::Counter::kAvgThreadsRate}},
106    {"Bar",  { 2, bm::Counter::kAvgThreadsRate}},
107    {"Baz",  { 4, bm::Counter::kAvgThreadsRate}},
108    {"Bat",  { 8, bm::Counter::kAvgThreadsRate}},
109    {"Frob", {16, bm::Counter::kAvgThreadsRate}},
110    {"Lob",  {32, bm::Counter::kAvgThreadsRate}},
111  });
112}
113BENCHMARK(BM_CounterRates_Tabular)->ThreadRange(1, 16);
114ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterRates_Tabular/threads:%int\",$"},
115                       {"\"iterations\": %int,$", MR_Next},
116                       {"\"real_time\": %float,$", MR_Next},
117                       {"\"cpu_time\": %float,$", MR_Next},
118                       {"\"time_unit\": \"ns\",$", MR_Next},
119                       {"\"Bar\": %float,$", MR_Next},
120                       {"\"Bat\": %float,$", MR_Next},
121                       {"\"Baz\": %float,$", MR_Next},
122                       {"\"Foo\": %float,$", MR_Next},
123                       {"\"Frob\": %float,$", MR_Next},
124                       {"\"Lob\": %float$", MR_Next},
125                       {"}", MR_Next}});
126ADD_CASES(TC_CSVOut, {{"^\"BM_CounterRates_Tabular/threads:%int\",%csv_report,"
127                       "%float,%float,%float,%float,%float,%float$"}});
128// VS2013 does not allow this function to be passed as a lambda argument
129// to CHECK_BENCHMARK_RESULTS()
130void CheckTabularRate(Results const& e) {
131  double t = e.DurationCPUTime();
132  CHECK_FLOAT_COUNTER_VALUE(e, "Foo", EQ, 1./t, 0.001);
133  CHECK_FLOAT_COUNTER_VALUE(e, "Bar", EQ, 2./t, 0.001);
134  CHECK_FLOAT_COUNTER_VALUE(e, "Baz", EQ, 4./t, 0.001);
135  CHECK_FLOAT_COUNTER_VALUE(e, "Bat", EQ, 8./t, 0.001);
136  CHECK_FLOAT_COUNTER_VALUE(e, "Frob", EQ, 16./t, 0.001);
137  CHECK_FLOAT_COUNTER_VALUE(e, "Lob", EQ, 32./t, 0.001);
138}
139CHECK_BENCHMARK_RESULTS("BM_CounterRates_Tabular/threads:%int",
140                        &CheckTabularRate);
141
142// ========================================================================= //
143// ------------------------- Tabular Counters Output ----------------------- //
144// ========================================================================= //
145
146// set only some of the counters
147void BM_CounterSet0_Tabular(benchmark::State& state) {
148  for (auto _ : state) {
149  }
150  namespace bm = benchmark;
151  state.counters.insert({
152    {"Foo", {10, bm::Counter::kAvgThreads}},
153    {"Bar", {20, bm::Counter::kAvgThreads}},
154    {"Baz", {40, bm::Counter::kAvgThreads}},
155  });
156}
157BENCHMARK(BM_CounterSet0_Tabular)->ThreadRange(1, 16);
158ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet0_Tabular/threads:%int\",$"},
159                       {"\"iterations\": %int,$", MR_Next},
160                       {"\"real_time\": %float,$", MR_Next},
161                       {"\"cpu_time\": %float,$", MR_Next},
162                       {"\"time_unit\": \"ns\",$", MR_Next},
163                       {"\"Bar\": %float,$", MR_Next},
164                       {"\"Baz\": %float,$", MR_Next},
165                       {"\"Foo\": %float$", MR_Next},
166                       {"}", MR_Next}});
167ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet0_Tabular/threads:%int\",%csv_report,"
168                       "%float,,%float,%float,,"}});
169// VS2013 does not allow this function to be passed as a lambda argument
170// to CHECK_BENCHMARK_RESULTS()
171void CheckSet0(Results const& e) {
172  CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
173  CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 20);
174  CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
175}
176CHECK_BENCHMARK_RESULTS("BM_CounterSet0_Tabular", &CheckSet0);
177
178// again.
179void BM_CounterSet1_Tabular(benchmark::State& state) {
180  for (auto _ : state) {
181  }
182  namespace bm = benchmark;
183  state.counters.insert({
184    {"Foo", {15, bm::Counter::kAvgThreads}},
185    {"Bar", {25, bm::Counter::kAvgThreads}},
186    {"Baz", {45, bm::Counter::kAvgThreads}},
187  });
188}
189BENCHMARK(BM_CounterSet1_Tabular)->ThreadRange(1, 16);
190ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet1_Tabular/threads:%int\",$"},
191                       {"\"iterations\": %int,$", MR_Next},
192                       {"\"real_time\": %float,$", MR_Next},
193                       {"\"cpu_time\": %float,$", MR_Next},
194                       {"\"time_unit\": \"ns\",$", MR_Next},
195                       {"\"Bar\": %float,$", MR_Next},
196                       {"\"Baz\": %float,$", MR_Next},
197                       {"\"Foo\": %float$", MR_Next},
198                       {"}", MR_Next}});
199ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet1_Tabular/threads:%int\",%csv_report,"
200                       "%float,,%float,%float,,"}});
201// VS2013 does not allow this function to be passed as a lambda argument
202// to CHECK_BENCHMARK_RESULTS()
203void CheckSet1(Results const& e) {
204  CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 15);
205  CHECK_COUNTER_VALUE(e, int, "Bar", EQ, 25);
206  CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 45);
207}
208CHECK_BENCHMARK_RESULTS("BM_CounterSet1_Tabular/threads:%int", &CheckSet1);
209
210// ========================================================================= //
211// ------------------------- Tabular Counters Output ----------------------- //
212// ========================================================================= //
213
214// set only some of the counters, different set now.
215void BM_CounterSet2_Tabular(benchmark::State& state) {
216  for (auto _ : state) {
217  }
218  namespace bm = benchmark;
219  state.counters.insert({
220    {"Foo", {10, bm::Counter::kAvgThreads}},
221    {"Bat", {30, bm::Counter::kAvgThreads}},
222    {"Baz", {40, bm::Counter::kAvgThreads}},
223  });
224}
225BENCHMARK(BM_CounterSet2_Tabular)->ThreadRange(1, 16);
226ADD_CASES(TC_JSONOut, {{"\"name\": \"BM_CounterSet2_Tabular/threads:%int\",$"},
227                       {"\"iterations\": %int,$", MR_Next},
228                       {"\"real_time\": %float,$", MR_Next},
229                       {"\"cpu_time\": %float,$", MR_Next},
230                       {"\"time_unit\": \"ns\",$", MR_Next},
231                       {"\"Bat\": %float,$", MR_Next},
232                       {"\"Baz\": %float,$", MR_Next},
233                       {"\"Foo\": %float$", MR_Next},
234                       {"}", MR_Next}});
235ADD_CASES(TC_CSVOut, {{"^\"BM_CounterSet2_Tabular/threads:%int\",%csv_report,"
236                       ",%float,%float,%float,,"}});
237// VS2013 does not allow this function to be passed as a lambda argument
238// to CHECK_BENCHMARK_RESULTS()
239void CheckSet2(Results const& e) {
240  CHECK_COUNTER_VALUE(e, int, "Foo", EQ, 10);
241  CHECK_COUNTER_VALUE(e, int, "Bat", EQ, 30);
242  CHECK_COUNTER_VALUE(e, int, "Baz", EQ, 40);
243}
244CHECK_BENCHMARK_RESULTS("BM_CounterSet2_Tabular", &CheckSet2);
245
246// ========================================================================= //
247// --------------------------- TEST CASES END ------------------------------ //
248// ========================================================================= //
249
250int main(int argc, char* argv[]) { RunOutputTests(argc, argv); }
251