1// Copyright (c) 2012 The Chromium Authors. All rights reserved.
2// Use of this source code is governed by a BSD-style license that can be
3// found in the LICENSE file.
4
5#include "ppapi/tests/testing_instance.h"
6
7#include <algorithm>
8#include <cstring>
9#include <iomanip>
10#include <sstream>
11#include <vector>
12
13#include "ppapi/cpp/core.h"
14#include "ppapi/cpp/module.h"
15#include "ppapi/cpp/var.h"
16#include "ppapi/cpp/view.h"
17#include "ppapi/tests/test_case.h"
18
19TestCaseFactory* TestCaseFactory::head_ = NULL;
20
21// Cookie value we use to signal "we're still working." See the comment above
22// the class declaration for how this works.
23static const char kProgressSignal[] = "...";
24
25// Returns a new heap-allocated test case for the given test, or NULL on
26// failure.
27TestingInstance::TestingInstance(PP_Instance instance)
28#if (defined __native_client__)
29    : pp::Instance(instance),
30#else
31    : pp::InstancePrivate(instance),
32#endif
33      current_case_(NULL),
34      executed_tests_(false),
35      number_tests_executed_(0),
36      nacl_mode_(false),
37      ssl_server_port_(-1),
38      websocket_port_(-1),
39      remove_plugin_(true) {
40  callback_factory_.Initialize(this);
41}
42
43TestingInstance::~TestingInstance() {
44  if (current_case_)
45    delete current_case_;
46}
47
48bool TestingInstance::Init(uint32_t argc,
49                           const char* argn[],
50                           const char* argv[]) {
51  for (uint32_t i = 0; i < argc; i++) {
52    if (std::strcmp(argn[i], "mode") == 0) {
53      if (std::strcmp(argv[i], "nacl") == 0)
54        nacl_mode_ = true;
55    } else if (std::strcmp(argn[i], "protocol") == 0) {
56      protocol_ = argv[i];
57    } else if (std::strcmp(argn[i], "websocket_host") == 0) {
58      websocket_host_ = argv[i];
59    } else if (std::strcmp(argn[i], "websocket_port") == 0) {
60      websocket_port_ = atoi(argv[i]);
61    } else if (std::strcmp(argn[i], "ssl_server_port") == 0) {
62      ssl_server_port_ = atoi(argv[i]);
63    }
64  }
65  // Create the proper test case from the argument.
66  for (uint32_t i = 0; i < argc; i++) {
67    if (std::strcmp(argn[i], "testcase") == 0) {
68      if (argv[i][0] == '\0')
69        break;
70      current_case_ = CaseForTestName(argv[i]);
71      test_filter_ = argv[i];
72      if (!current_case_)
73        errors_.append(std::string("Unknown test case ") + argv[i]);
74      else if (!current_case_->Init())
75        errors_.append(" Test case could not initialize.");
76      return true;
77    }
78  }
79
80  // In DidChangeView, we'll dump out a list of all available tests.
81  return true;
82}
83
84#if !(defined __native_client__)
85pp::Var TestingInstance::GetInstanceObject() {
86  if (current_case_)
87    return current_case_->GetTestObject();
88
89  return pp::VarPrivate();
90}
91#endif
92
93void TestingInstance::HandleMessage(const pp::Var& message_data) {
94  if (current_case_)
95    current_case_->HandleMessage(message_data);
96}
97
98void TestingInstance::DidChangeView(const pp::View& view) {
99  if (!executed_tests_) {
100    executed_tests_ = true;
101    pp::Module::Get()->core()->CallOnMainThread(
102        0,
103        callback_factory_.NewCallback(&TestingInstance::ExecuteTests));
104  }
105  if (current_case_)
106    current_case_->DidChangeView(view);
107}
108
109bool TestingInstance::HandleInputEvent(const pp::InputEvent& event) {
110  if (current_case_)
111    return current_case_->HandleInputEvent(event);
112  return false;
113}
114
115void TestingInstance::EvalScript(const std::string& script) {
116  SendTestCommand("EvalScript", script);
117}
118
119void TestingInstance::SetCookie(const std::string& name,
120                                const std::string& value) {
121  SendTestCommand("SetCookie", name + "=" + value);
122}
123
124void TestingInstance::LogTest(const std::string& test_name,
125                              const std::string& error_message,
126                              PP_TimeTicks start_time) {
127  current_test_name_ = test_name;
128
129  // Compute the time to run the test and save it in a string for logging:
130  PP_TimeTicks end_time(pp::Module::Get()->core()->GetTimeTicks());
131  std::ostringstream number_stream;
132  PP_TimeTicks elapsed_time(end_time - start_time);
133  number_stream << std::fixed << std::setprecision(3) << elapsed_time;
134  std::string time_string(number_stream.str());
135
136  // Tell the browser we're still working.
137  ReportProgress(kProgressSignal);
138
139  number_tests_executed_++;
140
141  std::string html;
142  html.append("<div class=\"test_line\"><span class=\"test_name\">");
143  html.append(test_name);
144  html.append("</span> ");
145  if (error_message.empty()) {
146    html.append("<span class=\"pass\">PASS</span>");
147  } else {
148    html.append("<span class=\"fail\">FAIL</span>: <span class=\"err_msg\">");
149    html.append(error_message);
150    html.append("</span>");
151
152    if (!errors_.empty())
153      errors_.append(", ");  // Separator for different error messages.
154    errors_.append(test_name + " FAIL: " + error_message);
155  }
156  html.append(" <span class=\"time\">(");
157  html.append(time_string);
158  html.append("s)</span>");
159
160  html.append("</div>");
161  LogHTML(html);
162
163  std::string test_time;
164  test_time.append(test_name);
165  test_time.append(" finished in ");
166  test_time.append(time_string);
167  test_time.append(" seconds.");
168  LogTestTime(test_time);
169
170  current_test_name_.clear();
171}
172
173void TestingInstance::AppendError(const std::string& message) {
174  if (!errors_.empty())
175    errors_.append(", ");
176  errors_.append(message);
177}
178
179void TestingInstance::ExecuteTests(int32_t unused) {
180  ReportProgress(kProgressSignal);
181
182  // Clear the console.
183  SendTestCommand("ClearConsole");
184
185  if (!errors_.empty()) {
186    // Catch initialization errors and output the current error string to
187    // the console.
188    LogError("Plugin initialization failed: " + errors_);
189  } else if (!current_case_) {
190    LogAvailableTests();
191    errors_.append("FAIL: Only listed tests");
192  } else {
193    current_case_->RunTests(test_filter_);
194
195    if (number_tests_executed_ == 0) {
196      errors_.append("No tests executed. The test filter might be too "
197                     "restrictive: '" + test_filter_ + "'.");
198      LogError(errors_);
199    }
200    if (current_case_->skipped_tests().size()) {
201      // TODO(dmichael): Convert all TestCases to run all tests in one fixture,
202      //                 and enable this check. Currently, a lot of our tests
203      //                 run 1 test per fixture, which is slow.
204      /*
205      errors_.append("Some tests were not listed and thus were not run. Make "
206                     "sure all tests are passed in the test_case URL (even if "
207                     "they are marked DISABLED_). Forgotten tests: ");
208      std::set<std::string>::const_iterator iter =
209          current_case_->skipped_tests().begin();
210      for (; iter != current_case_->skipped_tests().end(); ++iter) {
211        errors_.append(*iter);
212        errors_.append(" ");
213      }
214      LogError(errors_);
215      */
216    }
217    if (current_case_->remaining_tests().size()) {
218      errors_.append("Some listed tests were not found in the TestCase. Check "
219                     "the test names that were passed to make sure they match "
220                     "tests in the TestCase. Unknown tests: ");
221      std::map<std::string, bool>::const_iterator iter =
222          current_case_->remaining_tests().begin();
223      for (; iter != current_case_->remaining_tests().end(); ++iter) {
224        errors_.append(iter->first);
225        errors_.append(" ");
226      }
227      LogError(errors_);
228    }
229  }
230
231  if (remove_plugin_)
232    SendTestCommand("RemovePluginWhenFinished");
233  std::string result(errors_);
234  if (result.empty())
235    result = "PASS";
236  SendTestCommand("DidExecuteTests", result);
237  // Note, DidExecuteTests may unload the plugin. We can't really do anything
238  // after this point.
239}
240
241TestCase* TestingInstance::CaseForTestName(const std::string& name) {
242  std::string case_name = name.substr(0, name.find_first_of('_'));
243  TestCaseFactory* iter = TestCaseFactory::head_;
244  while (iter != NULL) {
245    if (case_name == iter->name_)
246      return iter->method_(this);
247    iter = iter->next_;
248  }
249  return NULL;
250}
251
252void TestingInstance::SendTestCommand(const std::string& command) {
253  std::string msg("TESTING_MESSAGE:");
254  msg += command;
255  PostMessage(pp::Var(msg));
256}
257
258void TestingInstance::SendTestCommand(const std::string& command,
259                                      const std::string& params) {
260  SendTestCommand(command + ":" + params);
261}
262
263
264void TestingInstance::LogAvailableTests() {
265  // Print out a listing of all tests.
266  std::vector<std::string> test_cases;
267  TestCaseFactory* iter = TestCaseFactory::head_;
268  while (iter != NULL) {
269    test_cases.push_back(iter->name_);
270    iter = iter->next_;
271  }
272  std::sort(test_cases.begin(), test_cases.end());
273
274  std::string html;
275  html.append("Available test cases: <dl>");
276  for (size_t i = 0; i < test_cases.size(); ++i) {
277    html.append("<dd><a href='?testcase=");
278    html.append(test_cases[i]);
279    if (nacl_mode_)
280       html.append("&mode=nacl");
281    html.append("'>");
282    html.append(test_cases[i]);
283    html.append("</a></dd>");
284  }
285  html.append("</dl>");
286  html.append("<button onclick='RunAll()'>Run All Tests</button>");
287
288  LogHTML(html);
289}
290
291void TestingInstance::LogError(const std::string& text) {
292  std::string html;
293  html.append("<span class=\"fail\">FAIL</span>: <span class=\"err_msg\">");
294  html.append(text);
295  html.append("</span>");
296  LogHTML(html);
297}
298
299void TestingInstance::LogHTML(const std::string& html) {
300  SendTestCommand("LogHTML", html);
301}
302
303void TestingInstance::ReportProgress(const std::string& progress_value) {
304  SendTestCommand("ReportProgress", progress_value);
305}
306
307void TestingInstance::AddPostCondition(const std::string& script) {
308  SendTestCommand("AddPostCondition", script);
309}
310
311void TestingInstance::LogTestTime(const std::string& test_time) {
312  SendTestCommand("LogTestTime", test_time);
313}
314
315class Module : public pp::Module {
316 public:
317  Module() : pp::Module() {}
318  virtual ~Module() {}
319
320  virtual pp::Instance* CreateInstance(PP_Instance instance) {
321    return new TestingInstance(instance);
322  }
323};
324
325namespace pp {
326
327Module* CreateModule() {
328  return new ::Module();
329}
330
331}  // namespace pp
332