1// Protocol Buffers - Google's data interchange format
2// Copyright 2008 Google Inc.  All rights reserved.
3// https://developers.google.com/protocol-buffers/
4//
5// Redistribution and use in source and binary forms, with or without
6// modification, are permitted provided that the following conditions are
7// met:
8//
9//     * Redistributions of source code must retain the above copyright
10// notice, this list of conditions and the following disclaimer.
11//     * Redistributions in binary form must reproduce the above
12// copyright notice, this list of conditions and the following disclaimer
13// in the documentation and/or other materials provided with the
14// distribution.
15//     * Neither the name of Google Inc. nor the names of its
16// contributors may be used to endorse or promote products derived from
17// this software without specific prior written permission.
18//
19// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20// "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21// LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22// A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23// OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24// SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25// LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26// DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27// THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29// OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30
31// This file contains a program for running the test suite in a separate
32// process.  The other alternative is to run the suite in-process.  See
33// conformance.proto for pros/cons of these two options.
34//
35// This program will fork the process under test and communicate with it over
36// its stdin/stdout:
37//
38//     +--------+   pipe   +----------+
39//     | tester | <------> | testee   |
40//     |        |          |          |
41//     |  C++   |          | any lang |
42//     +--------+          +----------+
43//
44// The tester contains all of the test cases and their expected output.
45// The testee is a simple program written in the target language that reads
46// each test case and attempts to produce acceptable output for it.
47//
48// Every test consists of a ConformanceRequest/ConformanceResponse
49// request/reply pair.  The protocol on the pipe is simply:
50//
51//   1. tester sends 4-byte length N (little endian)
52//   2. tester sends N bytes representing a ConformanceRequest proto
53//   3. testee sends 4-byte length M (little endian)
54//   4. testee sends M bytes representing a ConformanceResponse proto
55
56#include <algorithm>
57#include <errno.h>
58#include <fstream>
59#include <sys/types.h>
60#include <sys/wait.h>
61#include <unistd.h>
62#include <vector>
63
64#include <google/protobuf/stubs/stringprintf.h>
65
66#include "conformance.pb.h"
67#include "conformance_test.h"
68
69using conformance::ConformanceRequest;
70using conformance::ConformanceResponse;
71using google::protobuf::internal::scoped_array;
72using google::protobuf::StringAppendF;
73using std::string;
74using std::vector;
75
76#define STRINGIFY(x) #x
77#define TOSTRING(x) STRINGIFY(x)
78#define CHECK_SYSCALL(call) \
79  if (call < 0) { \
80    perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \
81    exit(1); \
82  }
83
84// Test runner that spawns the process being tested and communicates with it
85// over a pipe.
86class ForkPipeRunner : public google::protobuf::ConformanceTestRunner {
87 public:
88  ForkPipeRunner(const std::string &executable)
89      : child_pid_(-1), executable_(executable) {}
90
91  virtual ~ForkPipeRunner() {}
92
93  void RunTest(const std::string& test_name,
94               const std::string& request,
95               std::string* response) {
96    if (child_pid_ < 0) {
97      SpawnTestProgram();
98    }
99
100    current_test_name_ = test_name;
101
102    uint32_t len = request.size();
103    CheckedWrite(write_fd_, &len, sizeof(uint32_t));
104    CheckedWrite(write_fd_, request.c_str(), request.size());
105
106    if (!TryRead(read_fd_, &len, sizeof(uint32_t))) {
107      // We failed to read from the child, assume a crash and try to reap.
108      GOOGLE_LOG(INFO) << "Trying to reap child, pid=" << child_pid_;
109
110      int status;
111      waitpid(child_pid_, &status, WEXITED);
112
113      string error_msg;
114      if (WIFEXITED(status)) {
115        StringAppendF(&error_msg,
116                      "child exited, status=%d", WEXITSTATUS(status));
117      } else if (WIFSIGNALED(status)) {
118        StringAppendF(&error_msg,
119                      "child killed by signal %d", WTERMSIG(status));
120      }
121      GOOGLE_LOG(INFO) << error_msg;
122      child_pid_ = -1;
123
124      conformance::ConformanceResponse response_obj;
125      response_obj.set_runtime_error(error_msg);
126      response_obj.SerializeToString(response);
127      return;
128    }
129
130    response->resize(len);
131    CheckedRead(read_fd_, (void*)response->c_str(), len);
132  }
133
134 private:
135  // TODO(haberman): make this work on Windows, instead of using these
136  // UNIX-specific APIs.
137  //
138  // There is a platform-agnostic API in
139  //    src/google/protobuf/compiler/subprocess.h
140  //
141  // However that API only supports sending a single message to the subprocess.
142  // We really want to be able to send messages and receive responses one at a
143  // time:
144  //
145  // 1. Spawning a new process for each test would take way too long for thousands
146  //    of tests and subprocesses like java that can take 100ms or more to start
147  //    up.
148  //
149  // 2. Sending all the tests in one big message and receiving all results in one
150  //    big message would take away our visibility about which test(s) caused a
151  //    crash or other fatal error.  It would also give us only a single failure
152  //    instead of all of them.
153  void SpawnTestProgram() {
154    int toproc_pipe_fd[2];
155    int fromproc_pipe_fd[2];
156    if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) {
157      perror("pipe");
158      exit(1);
159    }
160
161    pid_t pid = fork();
162    if (pid < 0) {
163      perror("fork");
164      exit(1);
165    }
166
167    if (pid) {
168      // Parent.
169      CHECK_SYSCALL(close(toproc_pipe_fd[0]));
170      CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
171      write_fd_ = toproc_pipe_fd[1];
172      read_fd_ = fromproc_pipe_fd[0];
173      child_pid_ = pid;
174    } else {
175      // Child.
176      CHECK_SYSCALL(close(STDIN_FILENO));
177      CHECK_SYSCALL(close(STDOUT_FILENO));
178      CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO));
179      CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO));
180
181      CHECK_SYSCALL(close(toproc_pipe_fd[0]));
182      CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
183      CHECK_SYSCALL(close(toproc_pipe_fd[1]));
184      CHECK_SYSCALL(close(fromproc_pipe_fd[0]));
185
186      scoped_array<char> executable(new char[executable_.size() + 1]);
187      memcpy(executable.get(), executable_.c_str(), executable_.size());
188      executable[executable_.size()] = '\0';
189
190      char *const argv[] = {executable.get(), NULL};
191      CHECK_SYSCALL(execv(executable.get(), argv));  // Never returns.
192    }
193  }
194
195  void CheckedWrite(int fd, const void *buf, size_t len) {
196    if (write(fd, buf, len) != len) {
197      GOOGLE_LOG(FATAL) << current_test_name_
198                        << ": error writing to test program: "
199                        << strerror(errno);
200    }
201  }
202
203  bool TryRead(int fd, void *buf, size_t len) {
204    size_t ofs = 0;
205    while (len > 0) {
206      ssize_t bytes_read = read(fd, (char*)buf + ofs, len);
207
208      if (bytes_read == 0) {
209        GOOGLE_LOG(ERROR) << current_test_name_
210                          << ": unexpected EOF from test program";
211        return false;
212      } else if (bytes_read < 0) {
213        GOOGLE_LOG(ERROR) << current_test_name_
214                          << ": error reading from test program: "
215                          << strerror(errno);
216        return false;
217      }
218
219      len -= bytes_read;
220      ofs += bytes_read;
221    }
222
223    return true;
224  }
225
226  void CheckedRead(int fd, void *buf, size_t len) {
227    if (!TryRead(fd, buf, len)) {
228      GOOGLE_LOG(FATAL) << current_test_name_
229                        << ": error reading from test program: "
230                        << strerror(errno);
231    }
232  }
233
234  int write_fd_;
235  int read_fd_;
236  pid_t child_pid_;
237  std::string executable_;
238  std::string current_test_name_;
239};
240
241void UsageError() {
242  fprintf(stderr,
243          "Usage: conformance-test-runner [options] <test-program>\n");
244  fprintf(stderr, "\n");
245  fprintf(stderr, "Options:\n");
246  fprintf(stderr,
247          "  --failure_list <filename>   Use to specify list of tests\n");
248  fprintf(stderr,
249          "                              that are expected to fail.  File\n");
250  fprintf(stderr,
251          "                              should contain one test name per\n");
252  fprintf(stderr,
253          "                              line.  Use '#' for comments.\n");
254  exit(1);
255}
256
257void ParseFailureList(const char *filename, vector<string>* failure_list) {
258  std::ifstream infile(filename);
259
260  if (!infile.is_open()) {
261    fprintf(stderr, "Couldn't open failure list file: %s\n", filename);
262    exit(1);
263  }
264
265  for (string line; getline(infile, line);) {
266    // Remove whitespace.
267    line.erase(std::remove_if(line.begin(), line.end(), ::isspace),
268               line.end());
269
270    // Remove comments.
271    line = line.substr(0, line.find("#"));
272
273    if (!line.empty()) {
274      failure_list->push_back(line);
275    }
276  }
277}
278
279int main(int argc, char *argv[]) {
280  char *program;
281  google::protobuf::ConformanceTestSuite suite;
282
283  vector<string> failure_list;
284
285  for (int arg = 1; arg < argc; ++arg) {
286    if (strcmp(argv[arg], "--failure_list") == 0) {
287      if (++arg == argc) UsageError();
288      ParseFailureList(argv[arg], &failure_list);
289    } else if (strcmp(argv[arg], "--verbose") == 0) {
290      suite.SetVerbose(true);
291    } else if (argv[arg][0] == '-') {
292      fprintf(stderr, "Unknown option: %s\n", argv[arg]);
293      UsageError();
294    } else {
295      if (arg != argc - 1) {
296        fprintf(stderr, "Too many arguments.\n");
297        UsageError();
298      }
299      program = argv[arg];
300    }
301  }
302
303  suite.SetFailureList(failure_list);
304  ForkPipeRunner runner(program);
305
306  std::string output;
307  bool ok = suite.RunSuite(&runner, &output);
308
309  fwrite(output.c_str(), 1, output.size(), stderr);
310
311  return ok ? EXIT_SUCCESS : EXIT_FAILURE;
312}
313