• Home
  • History
  • Annotate
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1  // Protocol Buffers - Google's data interchange format
2  // Copyright 2008 Google Inc.  All rights reserved.
3  // https://developers.google.com/protocol-buffers/
4  //
5  // Redistribution and use in source and binary forms, with or without
6  // modification, are permitted provided that the following conditions are
7  // met:
8  //
9  //     * Redistributions of source code must retain the above copyright
10  // notice, this list of conditions and the following disclaimer.
11  //     * Redistributions in binary form must reproduce the above
12  // copyright notice, this list of conditions and the following disclaimer
13  // in the documentation and/or other materials provided with the
14  // distribution.
15  //     * Neither the name of Google Inc. nor the names of its
16  // contributors may be used to endorse or promote products derived from
17  // this software without specific prior written permission.
18  //
19  // THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS
20  // "AS IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT
21  // LIMITED TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR
22  // A PARTICULAR PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT
23  // OWNER OR CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL,
24  // SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT
25  // LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
26  // DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
27  // THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
28  // (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
29  // OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
30  
31  // This file contains a program for running the test suite in a separate
32  // process.  The other alternative is to run the suite in-process.  See
33  // conformance.proto for pros/cons of these two options.
34  //
35  // This program will fork the process under test and communicate with it over
36  // its stdin/stdout:
37  //
38  //     +--------+   pipe   +----------+
39  //     | tester | <------> | testee   |
40  //     |        |          |          |
41  //     |  C++   |          | any lang |
42  //     +--------+          +----------+
43  //
44  // The tester contains all of the test cases and their expected output.
45  // The testee is a simple program written in the target language that reads
46  // each test case and attempts to produce acceptable output for it.
47  //
48  // Every test consists of a ConformanceRequest/ConformanceResponse
49  // request/reply pair.  The protocol on the pipe is simply:
50  //
51  //   1. tester sends 4-byte length N (little endian)
52  //   2. tester sends N bytes representing a ConformanceRequest proto
53  //   3. testee sends 4-byte length M (little endian)
54  //   4. testee sends M bytes representing a ConformanceResponse proto
55  
56  #include <algorithm>
57  #include <errno.h>
58  #include <fstream>
59  #include <sys/types.h>
60  #include <sys/wait.h>
61  #include <unistd.h>
62  #include <vector>
63  
64  #include <google/protobuf/stubs/stringprintf.h>
65  
66  #include "conformance.pb.h"
67  #include "conformance_test.h"
68  
69  using conformance::ConformanceRequest;
70  using conformance::ConformanceResponse;
71  using google::protobuf::internal::scoped_array;
72  using google::protobuf::StringAppendF;
73  using std::string;
74  using std::vector;
75  
76  #define STRINGIFY(x) #x
77  #define TOSTRING(x) STRINGIFY(x)
78  #define CHECK_SYSCALL(call) \
79    if (call < 0) { \
80      perror(#call " " __FILE__ ":" TOSTRING(__LINE__)); \
81      exit(1); \
82    }
83  
84  // Test runner that spawns the process being tested and communicates with it
85  // over a pipe.
86  class ForkPipeRunner : public google::protobuf::ConformanceTestRunner {
87   public:
ForkPipeRunner(const std::string & executable)88    ForkPipeRunner(const std::string &executable)
89        : child_pid_(-1), executable_(executable) {}
90  
~ForkPipeRunner()91    virtual ~ForkPipeRunner() {}
92  
RunTest(const std::string & test_name,const std::string & request,std::string * response)93    void RunTest(const std::string& test_name,
94                 const std::string& request,
95                 std::string* response) {
96      if (child_pid_ < 0) {
97        SpawnTestProgram();
98      }
99  
100      current_test_name_ = test_name;
101  
102      uint32_t len = request.size();
103      CheckedWrite(write_fd_, &len, sizeof(uint32_t));
104      CheckedWrite(write_fd_, request.c_str(), request.size());
105  
106      if (!TryRead(read_fd_, &len, sizeof(uint32_t))) {
107        // We failed to read from the child, assume a crash and try to reap.
108        GOOGLE_LOG(INFO) << "Trying to reap child, pid=" << child_pid_;
109  
110        int status;
111        waitpid(child_pid_, &status, WEXITED);
112  
113        string error_msg;
114        if (WIFEXITED(status)) {
115          StringAppendF(&error_msg,
116                        "child exited, status=%d", WEXITSTATUS(status));
117        } else if (WIFSIGNALED(status)) {
118          StringAppendF(&error_msg,
119                        "child killed by signal %d", WTERMSIG(status));
120        }
121        GOOGLE_LOG(INFO) << error_msg;
122        child_pid_ = -1;
123  
124        conformance::ConformanceResponse response_obj;
125        response_obj.set_runtime_error(error_msg);
126        response_obj.SerializeToString(response);
127        return;
128      }
129  
130      response->resize(len);
131      CheckedRead(read_fd_, (void*)response->c_str(), len);
132    }
133  
134   private:
135    // TODO(haberman): make this work on Windows, instead of using these
136    // UNIX-specific APIs.
137    //
138    // There is a platform-agnostic API in
139    //    src/google/protobuf/compiler/subprocess.h
140    //
141    // However that API only supports sending a single message to the subprocess.
142    // We really want to be able to send messages and receive responses one at a
143    // time:
144    //
145    // 1. Spawning a new process for each test would take way too long for thousands
146    //    of tests and subprocesses like java that can take 100ms or more to start
147    //    up.
148    //
149    // 2. Sending all the tests in one big message and receiving all results in one
150    //    big message would take away our visibility about which test(s) caused a
151    //    crash or other fatal error.  It would also give us only a single failure
152    //    instead of all of them.
SpawnTestProgram()153    void SpawnTestProgram() {
154      int toproc_pipe_fd[2];
155      int fromproc_pipe_fd[2];
156      if (pipe(toproc_pipe_fd) < 0 || pipe(fromproc_pipe_fd) < 0) {
157        perror("pipe");
158        exit(1);
159      }
160  
161      pid_t pid = fork();
162      if (pid < 0) {
163        perror("fork");
164        exit(1);
165      }
166  
167      if (pid) {
168        // Parent.
169        CHECK_SYSCALL(close(toproc_pipe_fd[0]));
170        CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
171        write_fd_ = toproc_pipe_fd[1];
172        read_fd_ = fromproc_pipe_fd[0];
173        child_pid_ = pid;
174      } else {
175        // Child.
176        CHECK_SYSCALL(close(STDIN_FILENO));
177        CHECK_SYSCALL(close(STDOUT_FILENO));
178        CHECK_SYSCALL(dup2(toproc_pipe_fd[0], STDIN_FILENO));
179        CHECK_SYSCALL(dup2(fromproc_pipe_fd[1], STDOUT_FILENO));
180  
181        CHECK_SYSCALL(close(toproc_pipe_fd[0]));
182        CHECK_SYSCALL(close(fromproc_pipe_fd[1]));
183        CHECK_SYSCALL(close(toproc_pipe_fd[1]));
184        CHECK_SYSCALL(close(fromproc_pipe_fd[0]));
185  
186        scoped_array<char> executable(new char[executable_.size() + 1]);
187        memcpy(executable.get(), executable_.c_str(), executable_.size());
188        executable[executable_.size()] = '\0';
189  
190        char *const argv[] = {executable.get(), NULL};
191        CHECK_SYSCALL(execv(executable.get(), argv));  // Never returns.
192      }
193    }
194  
CheckedWrite(int fd,const void * buf,size_t len)195    void CheckedWrite(int fd, const void *buf, size_t len) {
196      if (write(fd, buf, len) != len) {
197        GOOGLE_LOG(FATAL) << current_test_name_
198                          << ": error writing to test program: "
199                          << strerror(errno);
200      }
201    }
202  
TryRead(int fd,void * buf,size_t len)203    bool TryRead(int fd, void *buf, size_t len) {
204      size_t ofs = 0;
205      while (len > 0) {
206        ssize_t bytes_read = read(fd, (char*)buf + ofs, len);
207  
208        if (bytes_read == 0) {
209          GOOGLE_LOG(ERROR) << current_test_name_
210                            << ": unexpected EOF from test program";
211          return false;
212        } else if (bytes_read < 0) {
213          GOOGLE_LOG(ERROR) << current_test_name_
214                            << ": error reading from test program: "
215                            << strerror(errno);
216          return false;
217        }
218  
219        len -= bytes_read;
220        ofs += bytes_read;
221      }
222  
223      return true;
224    }
225  
CheckedRead(int fd,void * buf,size_t len)226    void CheckedRead(int fd, void *buf, size_t len) {
227      if (!TryRead(fd, buf, len)) {
228        GOOGLE_LOG(FATAL) << current_test_name_
229                          << ": error reading from test program: "
230                          << strerror(errno);
231      }
232    }
233  
234    int write_fd_;
235    int read_fd_;
236    pid_t child_pid_;
237    std::string executable_;
238    std::string current_test_name_;
239  };
240  
UsageError()241  void UsageError() {
242    fprintf(stderr,
243            "Usage: conformance-test-runner [options] <test-program>\n");
244    fprintf(stderr, "\n");
245    fprintf(stderr, "Options:\n");
246    fprintf(stderr,
247            "  --failure_list <filename>   Use to specify list of tests\n");
248    fprintf(stderr,
249            "                              that are expected to fail.  File\n");
250    fprintf(stderr,
251            "                              should contain one test name per\n");
252    fprintf(stderr,
253            "                              line.  Use '#' for comments.\n");
254    exit(1);
255  }
256  
ParseFailureList(const char * filename,vector<string> * failure_list)257  void ParseFailureList(const char *filename, vector<string>* failure_list) {
258    std::ifstream infile(filename);
259  
260    if (!infile.is_open()) {
261      fprintf(stderr, "Couldn't open failure list file: %s\n", filename);
262      exit(1);
263    }
264  
265    for (string line; getline(infile, line);) {
266      // Remove whitespace.
267      line.erase(std::remove_if(line.begin(), line.end(), ::isspace),
268                 line.end());
269  
270      // Remove comments.
271      line = line.substr(0, line.find("#"));
272  
273      if (!line.empty()) {
274        failure_list->push_back(line);
275      }
276    }
277  }
278  
main(int argc,char * argv[])279  int main(int argc, char *argv[]) {
280    char *program;
281    google::protobuf::ConformanceTestSuite suite;
282  
283    vector<string> failure_list;
284  
285    for (int arg = 1; arg < argc; ++arg) {
286      if (strcmp(argv[arg], "--failure_list") == 0) {
287        if (++arg == argc) UsageError();
288        ParseFailureList(argv[arg], &failure_list);
289      } else if (strcmp(argv[arg], "--verbose") == 0) {
290        suite.SetVerbose(true);
291      } else if (argv[arg][0] == '-') {
292        fprintf(stderr, "Unknown option: %s\n", argv[arg]);
293        UsageError();
294      } else {
295        if (arg != argc - 1) {
296          fprintf(stderr, "Too many arguments.\n");
297          UsageError();
298        }
299        program = argv[arg];
300      }
301    }
302  
303    suite.SetFailureList(failure_list);
304    ForkPipeRunner runner(program);
305  
306    std::string output;
307    bool ok = suite.RunSuite(&runner, &output);
308  
309    fwrite(output.c_str(), 1, output.size(), stderr);
310  
311    return ok ? EXIT_SUCCESS : EXIT_FAILURE;
312  }
313