1 /* tinytest.c -- Copyright 2009-2012 Nick Mathewson
2 *
3 * Redistribution and use in source and binary forms, with or without
4 * modification, are permitted provided that the following conditions
5 * are met:
6 * 1. Redistributions of source code must retain the above copyright
7 * notice, this list of conditions and the following disclaimer.
8 * 2. Redistributions in binary form must reproduce the above copyright
9 * notice, this list of conditions and the following disclaimer in the
10 * documentation and/or other materials provided with the distribution.
11 * 3. The name of the author may not be used to endorse or promote products
12 * derived from this software without specific prior written permission.
13 *
14 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR
15 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES
16 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED.
17 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT,
18 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT
19 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE,
20 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY
21 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
22 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF
23 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
24 */
25 #ifdef TINYTEST_LOCAL
26 #include "tinytest_local.h"
27 #endif
28
29 #include <stdio.h>
30 #include <stdlib.h>
31 #include <string.h>
32 #include <assert.h>
33
34 #ifndef NO_FORKING
35
36 #ifdef _WIN32
37 #include <windows.h>
38 #else
39 #include <sys/types.h>
40 #include <sys/wait.h>
41 #include <unistd.h>
42 #endif
43
44 #if defined(__APPLE__) && defined(__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__)
45 #if (__ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ >= 1060 && \
46 __ENVIRONMENT_MAC_OS_X_VERSION_MIN_REQUIRED__ < 1070)
47 /* Workaround for a stupid bug in OSX 10.6 */
48 #define FORK_BREAKS_GCOV
49 #include <vproc.h>
50 #endif
51 #endif
52
53 #endif /* !NO_FORKING */
54
55 #ifndef __GNUC__
56 #define __attribute__(x)
57 #endif
58
59 #include "tinytest.h"
60 #include "tinytest_macros.h"
61
62 #define LONGEST_TEST_NAME 16384
63
64 static int in_tinytest_main = 0; /**< true if we're in tinytest_main().*/
65 static int n_ok = 0; /**< Number of tests that have passed */
66 static int n_bad = 0; /**< Number of tests that have failed. */
67 static int n_skipped = 0; /**< Number of tests that have been skipped. */
68
69 static int opt_forked = 0; /**< True iff we're called from inside a win32 fork*/
70 static int opt_nofork = 0; /**< Suppress calls to fork() for debugging. */
71 static int opt_verbosity = 1; /**< -==quiet,0==terse,1==normal,2==verbose */
72 const char *verbosity_flag = "";
73
74 const struct testlist_alias_t *cfg_aliases=NULL;
75
76 enum outcome { SKIP=2, OK=1, FAIL=0 };
77 static enum outcome cur_test_outcome = 0;
78 const char *cur_test_prefix = NULL; /**< prefix of the current test group */
79 /** Name of the current test, if we haven't logged is yet. Used for --quiet */
80 const char *cur_test_name = NULL;
81
82 #ifdef _WIN32
83 /* Copy of argv[0] for win32. */
84 static char commandname[MAX_PATH+1];
85 #endif
86
87 static void usage(struct testgroup_t *groups, int list_groups)
88 __attribute__((noreturn));
89 static int process_test_option(struct testgroup_t *groups, const char *test);
90
91 static enum outcome
testcase_run_bare_(const struct testcase_t * testcase)92 testcase_run_bare_(const struct testcase_t *testcase)
93 {
94 void *env = NULL;
95 int outcome;
96 if (testcase->setup) {
97 env = testcase->setup->setup_fn(testcase);
98 if (!env)
99 return FAIL;
100 else if (env == (void*)TT_SKIP)
101 return SKIP;
102 }
103
104 cur_test_outcome = OK;
105 testcase->fn(env);
106 outcome = cur_test_outcome;
107
108 if (testcase->setup) {
109 if (testcase->setup->cleanup_fn(testcase, env) == 0)
110 outcome = FAIL;
111 }
112
113 return outcome;
114 }
115
116 #define MAGIC_EXITCODE 42
117
118 #ifndef NO_FORKING
119
120 static enum outcome
testcase_run_forked_(const struct testgroup_t * group,const struct testcase_t * testcase)121 testcase_run_forked_(const struct testgroup_t *group,
122 const struct testcase_t *testcase)
123 {
124 #ifdef _WIN32
125 /* Fork? On Win32? How primitive! We'll do what the smart kids do:
126 we'll invoke our own exe (whose name we recall from the command
127 line) with a command line that tells it to run just the test we
128 want, and this time without forking.
129
130 (No, threads aren't an option. The whole point of forking is to
131 share no state between tests.)
132 */
133 int ok;
134 char buffer[LONGEST_TEST_NAME+256];
135 STARTUPINFOA si;
136 PROCESS_INFORMATION info;
137 DWORD exitcode;
138
139 if (!in_tinytest_main) {
140 printf("\nERROR. On Windows, testcase_run_forked_ must be"
141 " called from within tinytest_main.\n");
142 abort();
143 }
144 if (opt_verbosity>0)
145 printf("[forking] ");
146
147 snprintf(buffer, sizeof(buffer), "%s --RUNNING-FORKED %s %s%s",
148 commandname, verbosity_flag, group->prefix, testcase->name);
149
150 memset(&si, 0, sizeof(si));
151 memset(&info, 0, sizeof(info));
152 si.cb = sizeof(si);
153
154 ok = CreateProcessA(commandname, buffer, NULL, NULL, 0,
155 0, NULL, NULL, &si, &info);
156 if (!ok) {
157 printf("CreateProcess failed!\n");
158 return 0;
159 }
160 WaitForSingleObject(info.hProcess, INFINITE);
161 GetExitCodeProcess(info.hProcess, &exitcode);
162 CloseHandle(info.hProcess);
163 CloseHandle(info.hThread);
164 if (exitcode == 0)
165 return OK;
166 else if (exitcode == MAGIC_EXITCODE)
167 return SKIP;
168 else
169 return FAIL;
170 #else
171 int outcome_pipe[2];
172 pid_t pid;
173 (void)group;
174
175 if (pipe(outcome_pipe))
176 perror("opening pipe");
177
178 if (opt_verbosity>0)
179 printf("[forking] ");
180 pid = fork();
181 #ifdef FORK_BREAKS_GCOV
182 vproc_transaction_begin(0);
183 #endif
184 if (!pid) {
185 /* child. */
186 int test_r, write_r;
187 char b[1];
188 close(outcome_pipe[0]);
189 test_r = testcase_run_bare_(testcase);
190 assert(0<=(int)test_r && (int)test_r<=2);
191 b[0] = "NYS"[test_r];
192 write_r = (int)write(outcome_pipe[1], b, 1);
193 if (write_r != 1) {
194 perror("write outcome to pipe");
195 exit(1);
196 }
197 exit(0);
198 return FAIL; /* unreachable */
199 } else {
200 /* parent */
201 int status, r;
202 char b[1];
203 /* Close this now, so that if the other side closes it,
204 * our read fails. */
205 close(outcome_pipe[1]);
206 r = (int)read(outcome_pipe[0], b, 1);
207 if (r == 0) {
208 printf("[Lost connection!] ");
209 return 0;
210 } else if (r != 1) {
211 perror("read outcome from pipe");
212 }
213 waitpid(pid, &status, 0);
214 close(outcome_pipe[0]);
215 return b[0]=='Y' ? OK : (b[0]=='S' ? SKIP : FAIL);
216 }
217 #endif
218 }
219
220 #endif /* !NO_FORKING */
221
222 int
testcase_run_one(const struct testgroup_t * group,const struct testcase_t * testcase)223 testcase_run_one(const struct testgroup_t *group,
224 const struct testcase_t *testcase)
225 {
226 enum outcome outcome;
227
228 if (testcase->flags & (TT_SKIP|TT_OFF_BY_DEFAULT)) {
229 if (opt_verbosity>0)
230 printf("%s%s: %s\n",
231 group->prefix, testcase->name,
232 (testcase->flags & TT_SKIP) ? "SKIPPED" : "DISABLED");
233 ++n_skipped;
234 return SKIP;
235 }
236
237 if (opt_verbosity>0 && !opt_forked) {
238 printf("%s%s: ", group->prefix, testcase->name);
239 } else {
240 if (opt_verbosity==0) printf(".");
241 cur_test_prefix = group->prefix;
242 cur_test_name = testcase->name;
243 }
244
245 #ifndef NO_FORKING
246 if ((testcase->flags & TT_FORK) && !(opt_forked||opt_nofork)) {
247 outcome = testcase_run_forked_(group, testcase);
248 } else {
249 #else
250 {
251 #endif
252 outcome = testcase_run_bare_(testcase);
253 }
254
255 if (outcome == OK) {
256 ++n_ok;
257 if (opt_verbosity>0 && !opt_forked)
258 puts(opt_verbosity==1?"OK":"");
259 } else if (outcome == SKIP) {
260 ++n_skipped;
261 if (opt_verbosity>0 && !opt_forked)
262 puts("SKIPPED");
263 } else {
264 ++n_bad;
265 if (!opt_forked)
266 printf("\n [%s FAILED]\n", testcase->name);
267 }
268
269 if (opt_forked) {
270 exit(outcome==OK ? 0 : (outcome==SKIP?MAGIC_EXITCODE : 1));
271 return 1; /* unreachable */
272 } else {
273 return (int)outcome;
274 }
275 }
276
277 int
278 tinytest_set_flag_(struct testgroup_t *groups, const char *arg, int set, unsigned long flag)
279 {
280 int i, j;
281 size_t length = LONGEST_TEST_NAME;
282 char fullname[LONGEST_TEST_NAME];
283 int found=0;
284 if (strstr(arg, ".."))
285 length = strstr(arg,"..")-arg;
286 for (i=0; groups[i].prefix; ++i) {
287 for (j=0; groups[i].cases[j].name; ++j) {
288 struct testcase_t *testcase = &groups[i].cases[j];
289 snprintf(fullname, sizeof(fullname), "%s%s",
290 groups[i].prefix, testcase->name);
291 if (!flag) { /* Hack! */
292 printf(" %s", fullname);
293 if (testcase->flags & TT_OFF_BY_DEFAULT)
294 puts(" (Off by default)");
295 else if (testcase->flags & TT_SKIP)
296 puts(" (DISABLED)");
297 else
298 puts("");
299 }
300 if (!strncmp(fullname, arg, length)) {
301 if (set)
302 testcase->flags |= flag;
303 else
304 testcase->flags &= ~flag;
305 ++found;
306 }
307 }
308 }
309 return found;
310 }
311
312 static void
313 usage(struct testgroup_t *groups, int list_groups)
314 {
315 puts("Options are: [--verbose|--quiet|--terse] [--no-fork]");
316 puts(" Specify tests by name, or using a prefix ending with '..'");
317 puts(" To skip a test, prefix its name with a colon.");
318 puts(" To enable a disabled test, prefix its name with a plus.");
319 puts(" Use --list-tests for a list of tests.");
320 if (list_groups) {
321 puts("Known tests are:");
322 tinytest_set_flag_(groups, "..", 1, 0);
323 }
324 exit(0);
325 }
326
327 static int
328 process_test_alias(struct testgroup_t *groups, const char *test)
329 {
330 int i, j, n, r;
331 for (i=0; cfg_aliases && cfg_aliases[i].name; ++i) {
332 if (!strcmp(cfg_aliases[i].name, test)) {
333 n = 0;
334 for (j = 0; cfg_aliases[i].tests[j]; ++j) {
335 r = process_test_option(groups, cfg_aliases[i].tests[j]);
336 if (r<0)
337 return -1;
338 n += r;
339 }
340 return n;
341 }
342 }
343 printf("No such test alias as @%s!",test);
344 return -1;
345 }
346
347 static int
348 process_test_option(struct testgroup_t *groups, const char *test)
349 {
350 int flag = TT_ENABLED_;
351 int n = 0;
352 if (test[0] == '@') {
353 return process_test_alias(groups, test + 1);
354 } else if (test[0] == ':') {
355 ++test;
356 flag = TT_SKIP;
357 } else if (test[0] == '+') {
358 ++test;
359 ++n;
360 if (!tinytest_set_flag_(groups, test, 0, TT_OFF_BY_DEFAULT)) {
361 printf("No such test as %s!\n", test);
362 return -1;
363 }
364 } else {
365 ++n;
366 }
367 if (!tinytest_set_flag_(groups, test, 1, flag)) {
368 printf("No such test as %s!\n", test);
369 return -1;
370 }
371 return n;
372 }
373
374 void
375 tinytest_set_aliases(const struct testlist_alias_t *aliases)
376 {
377 cfg_aliases = aliases;
378 }
379
380 int
381 tinytest_main(int c, const char **v, struct testgroup_t *groups)
382 {
383 int i, j, n=0;
384
385 #ifdef _WIN32
386 const char *sp = strrchr(v[0], '.');
387 const char *extension = "";
388 if (!sp || stricmp(sp, ".exe"))
389 extension = ".exe"; /* Add an exe so CreateProcess will work */
390 snprintf(commandname, sizeof(commandname), "%s%s", v[0], extension);
391 commandname[MAX_PATH]='\0';
392 #endif
393 for (i=1; i<c; ++i) {
394 if (v[i][0] == '-') {
395 if (!strcmp(v[i], "--RUNNING-FORKED")) {
396 opt_forked = 1;
397 } else if (!strcmp(v[i], "--no-fork")) {
398 opt_nofork = 1;
399 } else if (!strcmp(v[i], "--quiet")) {
400 opt_verbosity = -1;
401 verbosity_flag = "--quiet";
402 } else if (!strcmp(v[i], "--verbose")) {
403 opt_verbosity = 2;
404 verbosity_flag = "--verbose";
405 } else if (!strcmp(v[i], "--terse")) {
406 opt_verbosity = 0;
407 verbosity_flag = "--terse";
408 } else if (!strcmp(v[i], "--help")) {
409 usage(groups, 0);
410 } else if (!strcmp(v[i], "--list-tests")) {
411 usage(groups, 1);
412 } else {
413 printf("Unknown option %s. Try --help\n",v[i]);
414 return -1;
415 }
416 } else {
417 int r = process_test_option(groups, v[i]);
418 if (r<0)
419 return -1;
420 n += r;
421 }
422 }
423 if (!n)
424 tinytest_set_flag_(groups, "..", 1, TT_ENABLED_);
425
426 #ifdef _IONBF
427 setvbuf(stdout, NULL, _IONBF, 0);
428 #endif
429
430 ++in_tinytest_main;
431 for (i=0; groups[i].prefix; ++i)
432 for (j=0; groups[i].cases[j].name; ++j)
433 if (groups[i].cases[j].flags & TT_ENABLED_)
434 testcase_run_one(&groups[i],
435 &groups[i].cases[j]);
436
437 --in_tinytest_main;
438
439 if (opt_verbosity==0)
440 puts("");
441
442 if (n_bad)
443 printf("%d/%d TESTS FAILED. (%d skipped)\n", n_bad,
444 n_bad+n_ok,n_skipped);
445 else if (opt_verbosity >= 1)
446 printf("%d tests ok. (%d skipped)\n", n_ok, n_skipped);
447
448 return (n_bad == 0) ? 0 : 1;
449 }
450
451 int
452 tinytest_get_verbosity_(void)
453 {
454 return opt_verbosity;
455 }
456
457 void
458 tinytest_set_test_failed_(void)
459 {
460 if (opt_verbosity <= 0 && cur_test_name) {
461 if (opt_verbosity==0) puts("");
462 printf("%s%s: ", cur_test_prefix, cur_test_name);
463 cur_test_name = NULL;
464 }
465 cur_test_outcome = 0;
466 }
467
468 void
469 tinytest_set_test_skipped_(void)
470 {
471 if (cur_test_outcome==OK)
472 cur_test_outcome = SKIP;
473 }
474
475 char *
476 tinytest_format_hex_(const void *val_, unsigned long len)
477 {
478 const unsigned char *val = val_;
479 char *result, *cp;
480 size_t i;
481
482 if (!val)
483 return strdup("null");
484 if (!(result = malloc(len*2+1)))
485 return strdup("<allocation failure>");
486 cp = result;
487 for (i=0;i<len;++i) {
488 *cp++ = "0123456789ABCDEF"[val[i] >> 4];
489 *cp++ = "0123456789ABCDEF"[val[i] & 0x0f];
490 }
491 *cp = 0;
492 return result;
493 }
494