1
2#undef NDEBUG
3#include <cassert>
4#include <vector>
5
6#include "../src/check.h"  // NOTE: check.h is for internal use only!
7#include "benchmark/benchmark.h"
8
9namespace {
10
11class TestReporter : public benchmark::ConsoleReporter {
12 public:
13  virtual bool ReportContext(const Context& context) {
14    return ConsoleReporter::ReportContext(context);
15  };
16
17  virtual void ReportRuns(const std::vector<Run>& report) {
18    all_runs_.insert(all_runs_.end(), begin(report), end(report));
19    ConsoleReporter::ReportRuns(report);
20  }
21
22  TestReporter() {}
23  virtual ~TestReporter() {}
24
25  mutable std::vector<Run> all_runs_;
26};
27
28struct TestCase {
29  std::string name;
30  bool error_occurred;
31  std::string error_message;
32
33  typedef benchmark::BenchmarkReporter::Run Run;
34
35  void CheckRun(Run const& run) const {
36    CHECK(name == run.benchmark_name) << "expected " << name << " got "
37                                      << run.benchmark_name;
38    CHECK(error_occurred == run.error_occurred);
39    CHECK(error_message == run.error_message);
40    if (error_occurred) {
41      // CHECK(run.iterations == 0);
42    } else {
43      CHECK(run.iterations != 0);
44    }
45  }
46};
47
48std::vector<TestCase> ExpectedResults;
49
50int AddCases(const char* base_name, std::initializer_list<TestCase> const& v) {
51  for (auto TC : v) {
52    TC.name = base_name + TC.name;
53    ExpectedResults.push_back(std::move(TC));
54  }
55  return 0;
56}
57
58#define CONCAT(x, y) CONCAT2(x, y)
59#define CONCAT2(x, y) x##y
60#define ADD_CASES(...) int CONCAT(dummy, __LINE__) = AddCases(__VA_ARGS__)
61
62}  // end namespace
63
64void BM_error_before_running(benchmark::State& state) {
65  state.SkipWithError("error message");
66  while (state.KeepRunning()) {
67    assert(false);
68  }
69}
70BENCHMARK(BM_error_before_running);
71ADD_CASES("BM_error_before_running", {{"", true, "error message"}});
72
73void BM_error_during_running(benchmark::State& state) {
74  int first_iter = true;
75  while (state.KeepRunning()) {
76    if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
77      assert(first_iter);
78      first_iter = false;
79      state.SkipWithError("error message");
80    } else {
81      state.PauseTiming();
82      state.ResumeTiming();
83    }
84  }
85}
86BENCHMARK(BM_error_during_running)->Arg(1)->Arg(2)->ThreadRange(1, 8);
87ADD_CASES("BM_error_during_running", {{"/1/threads:1", true, "error message"},
88                                      {"/1/threads:2", true, "error message"},
89                                      {"/1/threads:4", true, "error message"},
90                                      {"/1/threads:8", true, "error message"},
91                                      {"/2/threads:1", false, ""},
92                                      {"/2/threads:2", false, ""},
93                                      {"/2/threads:4", false, ""},
94                                      {"/2/threads:8", false, ""}});
95
96void BM_error_after_running(benchmark::State& state) {
97  while (state.KeepRunning()) {
98    benchmark::DoNotOptimize(state.iterations());
99  }
100  if (state.thread_index <= (state.threads / 2))
101    state.SkipWithError("error message");
102}
103BENCHMARK(BM_error_after_running)->ThreadRange(1, 8);
104ADD_CASES("BM_error_after_running", {{"/threads:1", true, "error message"},
105                                     {"/threads:2", true, "error message"},
106                                     {"/threads:4", true, "error message"},
107                                     {"/threads:8", true, "error message"}});
108
109void BM_error_while_paused(benchmark::State& state) {
110  bool first_iter = true;
111  while (state.KeepRunning()) {
112    if (state.range(0) == 1 && state.thread_index <= (state.threads / 2)) {
113      assert(first_iter);
114      first_iter = false;
115      state.PauseTiming();
116      state.SkipWithError("error message");
117    } else {
118      state.PauseTiming();
119      state.ResumeTiming();
120    }
121  }
122}
123BENCHMARK(BM_error_while_paused)->Arg(1)->Arg(2)->ThreadRange(1, 8);
124ADD_CASES("BM_error_while_paused", {{"/1/threads:1", true, "error message"},
125                                    {"/1/threads:2", true, "error message"},
126                                    {"/1/threads:4", true, "error message"},
127                                    {"/1/threads:8", true, "error message"},
128                                    {"/2/threads:1", false, ""},
129                                    {"/2/threads:2", false, ""},
130                                    {"/2/threads:4", false, ""},
131                                    {"/2/threads:8", false, ""}});
132
133int main(int argc, char* argv[]) {
134  benchmark::Initialize(&argc, argv);
135
136  TestReporter test_reporter;
137  benchmark::RunSpecifiedBenchmarks(&test_reporter);
138
139  typedef benchmark::BenchmarkReporter::Run Run;
140  auto EB = ExpectedResults.begin();
141
142  for (Run const& run : test_reporter.all_runs_) {
143    assert(EB != ExpectedResults.end());
144    EB->CheckRun(run);
145    ++EB;
146  }
147  assert(EB == ExpectedResults.end());
148
149  return 0;
150}
151