1#include <iostream>
2#include <map>
3#include <memory>
4#include <sstream>
5
6#include "../src/check.h"  // NOTE: check.h is for internal use only!
7#include "../src/re.h"     // NOTE: re.h is for internal use only
8#include "output_test.h"
9
10// ========================================================================= //
11// ------------------------------ Internals -------------------------------- //
12// ========================================================================= //
13namespace internal {
14namespace {
15
16using TestCaseList = std::vector<TestCase>;
17
18// Use a vector because the order elements are added matters during iteration.
19// std::map/unordered_map don't guarantee that.
20// For example:
21//  SetSubstitutions({{"%HelloWorld", "Hello"}, {"%Hello", "Hi"}});
22//     Substitute("%HelloWorld") // Always expands to Hello.
23using SubMap = std::vector<std::pair<std::string, std::string>>;
24
25TestCaseList& GetTestCaseList(TestCaseID ID) {
26  // Uses function-local statics to ensure initialization occurs
27  // before first use.
28  static TestCaseList lists[TC_NumID];
29  return lists[ID];
30}
31
32SubMap& GetSubstitutions() {
33  // Don't use 'dec_re' from header because it may not yet be initialized.
34  static std::string safe_dec_re = "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?";
35  static SubMap map = {
36      {"%float", "[0-9]*[.]?[0-9]+([eE][-+][0-9]+)?"},
37      {"%int", "[ ]*[0-9]+"},
38      {" %s ", "[ ]+"},
39      {"%time", "[ ]*[0-9]{1,5} ns"},
40      {"%console_report", "[ ]*[0-9]{1,5} ns [ ]*[0-9]{1,5} ns [ ]*[0-9]+"},
41      {"%console_us_report", "[ ]*[0-9] us [ ]*[0-9] us [ ]*[0-9]+"},
42      {"%csv_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,,,"},
43      {"%csv_us_report", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",us,,,,,"},
44      {"%csv_bytes_report",
45       "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns," + safe_dec_re + ",,,,"},
46      {"%csv_items_report",
47       "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,," + safe_dec_re + ",,,"},
48      {"%csv_label_report_begin", "[0-9]+," + safe_dec_re + "," + safe_dec_re + ",ns,,,"},
49      {"%csv_label_report_end", ",,"}};
50  return map;
51}
52
53std::string PerformSubstitutions(std::string source) {
54  SubMap const& subs = GetSubstitutions();
55  using SizeT = std::string::size_type;
56  for (auto const& KV : subs) {
57    SizeT pos;
58    SizeT next_start = 0;
59    while ((pos = source.find(KV.first, next_start)) != std::string::npos) {
60      next_start = pos + KV.second.size();
61      source.replace(pos, KV.first.size(), KV.second);
62    }
63  }
64  return source;
65}
66
67void CheckCase(std::stringstream& remaining_output, TestCase const& TC,
68               TestCaseList const& not_checks) {
69  std::string first_line;
70  bool on_first = true;
71  std::string line;
72  while (remaining_output.eof() == false) {
73    CHECK(remaining_output.good());
74    std::getline(remaining_output, line);
75    if (on_first) {
76      first_line = line;
77      on_first = false;
78    }
79    for (const auto& NC : not_checks) {
80      CHECK(!NC.regex->Match(line))
81          << "Unexpected match for line \"" << line << "\" for MR_Not regex \""
82          << NC.regex_str << "\""
83          << "\n    actual regex string \"" << TC.substituted_regex << "\""
84          << "\n    started matching near: " << first_line;
85    }
86    if (TC.regex->Match(line)) return;
87    CHECK(TC.match_rule != MR_Next)
88        << "Expected line \"" << line << "\" to match regex \"" << TC.regex_str
89        << "\""
90        << "\n    actual regex string \"" << TC.substituted_regex << "\""
91        << "\n    started matching near: " << first_line;
92  }
93  CHECK(remaining_output.eof() == false)
94      << "End of output reached before match for regex \"" << TC.regex_str
95      << "\" was found"
96      << "\n    actual regex string \"" << TC.substituted_regex << "\""
97      << "\n    started matching near: " << first_line;
98}
99
100void CheckCases(TestCaseList const& checks, std::stringstream& output) {
101  std::vector<TestCase> not_checks;
102  for (size_t i = 0; i < checks.size(); ++i) {
103    const auto& TC = checks[i];
104    if (TC.match_rule == MR_Not) {
105      not_checks.push_back(TC);
106      continue;
107    }
108    CheckCase(output, TC, not_checks);
109    not_checks.clear();
110  }
111}
112
113class TestReporter : public benchmark::BenchmarkReporter {
114 public:
115  TestReporter(std::vector<benchmark::BenchmarkReporter*> reps)
116      : reporters_(reps) {}
117
118  virtual bool ReportContext(const Context& context) {
119    bool last_ret = false;
120    bool first = true;
121    for (auto rep : reporters_) {
122      bool new_ret = rep->ReportContext(context);
123      CHECK(first || new_ret == last_ret)
124          << "Reports return different values for ReportContext";
125      first = false;
126      last_ret = new_ret;
127    }
128    (void)first;
129    return last_ret;
130  }
131
132  void ReportRuns(const std::vector<Run>& report) {
133    for (auto rep : reporters_) rep->ReportRuns(report);
134  }
135  void Finalize() {
136    for (auto rep : reporters_) rep->Finalize();
137  }
138
139 private:
140  std::vector<benchmark::BenchmarkReporter *> reporters_;
141};
142}
143}  // end namespace internal
144
145// ========================================================================= //
146// -------------------------- Public API Definitions------------------------ //
147// ========================================================================= //
148
149TestCase::TestCase(std::string re, int rule)
150    : regex_str(std::move(re)),
151      match_rule(rule),
152      substituted_regex(internal::PerformSubstitutions(regex_str)),
153      regex(std::make_shared<benchmark::Regex>()) {
154  std::string err_str;
155  regex->Init(substituted_regex,& err_str);
156  CHECK(err_str.empty()) << "Could not construct regex \"" << substituted_regex
157                         << "\""
158                         << "\n    originally \"" << regex_str << "\""
159                         << "\n    got error: " << err_str;
160}
161
162int AddCases(TestCaseID ID, std::initializer_list<TestCase> il) {
163  auto& L = internal::GetTestCaseList(ID);
164  L.insert(L.end(), il);
165  return 0;
166}
167
168int SetSubstitutions(
169    std::initializer_list<std::pair<std::string, std::string>> il) {
170  auto& subs = internal::GetSubstitutions();
171  for (auto KV : il) {
172    bool exists = false;
173    KV.second = internal::PerformSubstitutions(KV.second);
174    for (auto& EKV : subs) {
175      if (EKV.first == KV.first) {
176        EKV.second = std::move(KV.second);
177        exists = true;
178        break;
179      }
180    }
181    if (!exists) subs.push_back(std::move(KV));
182  }
183  return 0;
184}
185
186void RunOutputTests(int argc, char* argv[]) {
187  using internal::GetTestCaseList;
188  benchmark::Initialize(&argc, argv);
189  benchmark::ConsoleReporter CR(benchmark::ConsoleReporter::OO_None);
190  benchmark::JSONReporter JR;
191  benchmark::CSVReporter CSVR;
192  struct ReporterTest {
193    const char* name;
194    std::vector<TestCase>& output_cases;
195    std::vector<TestCase>& error_cases;
196    benchmark::BenchmarkReporter& reporter;
197    std::stringstream out_stream;
198    std::stringstream err_stream;
199
200    ReporterTest(const char* n, std::vector<TestCase>& out_tc,
201                 std::vector<TestCase>& err_tc,
202                 benchmark::BenchmarkReporter& br)
203        : name(n), output_cases(out_tc), error_cases(err_tc), reporter(br) {
204      reporter.SetOutputStream(&out_stream);
205      reporter.SetErrorStream(&err_stream);
206    }
207  } TestCases[] = {
208      {"ConsoleReporter", GetTestCaseList(TC_ConsoleOut),
209       GetTestCaseList(TC_ConsoleErr), CR},
210      {"JSONReporter", GetTestCaseList(TC_JSONOut), GetTestCaseList(TC_JSONErr),
211       JR},
212      {"CSVReporter", GetTestCaseList(TC_CSVOut), GetTestCaseList(TC_CSVErr),
213       CSVR},
214  };
215
216  // Create the test reporter and run the benchmarks.
217  std::cout << "Running benchmarks...\n";
218  internal::TestReporter test_rep({&CR, &JR, &CSVR});
219  benchmark::RunSpecifiedBenchmarks(&test_rep);
220
221  for (auto& rep_test : TestCases) {
222    std::string msg = std::string("\nTesting ") + rep_test.name + " Output\n";
223    std::string banner(msg.size() - 1, '-');
224    std::cout << banner << msg << banner << "\n";
225
226    std::cerr << rep_test.err_stream.str();
227    std::cout << rep_test.out_stream.str();
228
229    internal::CheckCases(rep_test.error_cases, rep_test.err_stream);
230    internal::CheckCases(rep_test.output_cases, rep_test.out_stream);
231
232    std::cout << "\n";
233  }
234}
235