2 * test-run-command.c: test run command API.
4 * (C) 2009 Ilari Liusvaara <ilari.liusvaara@elisanet.fi>
6 * This code is free software; you can redistribute it and/or modify
7 * it under the terms of the GNU General Public License version 2 as
8 * published by the Free Software Foundation.
11 #include "test-tool.h"
12 #include "git-compat-util.h"
14 #include "run-command.h"
17 #include "parse-options.h"
18 #include "string-list.h"
19 #include "thread-utils.h"
20 #include "wildmatch.h"
22 #include "parse-options.h"
24 static int number_callbacks;
25 static int parallel_next(struct child_process *cp,
30 struct child_process *d = cb;
31 if (number_callbacks >= 4)
34 strvec_pushv(&cp->args, d->argv);
36 cp->no_stdin = d->no_stdin;
37 strbuf_addstr(err, "preloaded output of a child\n");
40 *task_cb = xmalloc(sizeof(int));
41 *(int*)(*task_cb) = 2;
45 static int no_job(struct child_process *cp,
50 strbuf_addstr(err, "no further jobs available\n");
54 static void test_consume_sideband(struct strbuf *output, void *cb)
58 sideband = fopen("./sideband", "a");
60 strbuf_write(output, sideband);
64 static int task_finished(int result,
69 strbuf_addstr(err, "asking for a quick stop\n");
73 static int test_stdin(struct strbuf *pipe, void *cb, void *task_cb)
75 int *lines_remaining = task_cb;
78 strbuf_addf(pipe, "sample stdin %d\n", --(*lines_remaining));
80 return !(*lines_remaining);
85 struct string_list tests, failed;
87 int quiet, immediate, verbose, verbose_log, trace, write_junit_xml;
89 #define TESTSUITE_INIT \
90 { STRING_LIST_INIT_DUP, STRING_LIST_INIT_DUP, -1, 0, 0, 0, 0, 0, 0 }
92 static int next_test(struct child_process *cp, struct strbuf *err, void *cb,
95 struct testsuite *suite = cb;
97 if (suite->next >= suite->tests.nr)
100 test = suite->tests.items[suite->next++].string;
101 strvec_pushl(&cp->args, "sh", test, NULL);
103 strvec_push(&cp->args, "--quiet");
104 if (suite->immediate)
105 strvec_push(&cp->args, "-i");
107 strvec_push(&cp->args, "-v");
108 if (suite->verbose_log)
109 strvec_push(&cp->args, "-V");
111 strvec_push(&cp->args, "-x");
112 if (suite->write_junit_xml)
113 strvec_push(&cp->args, "--write-junit-xml");
115 strbuf_addf(err, "Output of '%s':\n", test);
116 *task_cb = (void *)test;
121 static int test_finished(int result, struct strbuf *err, void *cb,
124 struct testsuite *suite = cb;
125 const char *name = (const char *)task_cb;
128 string_list_append(&suite->failed, name);
130 strbuf_addf(err, "%s: '%s'\n", result ? "FAIL" : "SUCCESS", name);
135 static int test_failed(struct strbuf *out, void *cb, void *task_cb)
137 struct testsuite *suite = cb;
138 const char *name = (const char *)task_cb;
140 string_list_append(&suite->failed, name);
141 strbuf_addf(out, "FAILED TO START: '%s'\n", name);
146 static const char * const testsuite_usage[] = {
147 "test-run-command testsuite [<options>] [<pattern>...]",
151 static int testsuite(int argc, const char **argv)
153 struct testsuite suite = TESTSUITE_INIT;
154 int max_jobs = 1, i, ret;
157 struct option options[] = {
158 OPT_BOOL('i', "immediate", &suite.immediate,
159 "stop at first failed test case(s)"),
160 OPT_INTEGER('j', "jobs", &max_jobs, "run <N> jobs in parallel"),
161 OPT_BOOL('q', "quiet", &suite.quiet, "be terse"),
162 OPT_BOOL('v', "verbose", &suite.verbose, "be verbose"),
163 OPT_BOOL('V', "verbose-log", &suite.verbose_log,
164 "be verbose, redirected to a file"),
165 OPT_BOOL('x', "trace", &suite.trace, "trace shell commands"),
166 OPT_BOOL(0, "write-junit-xml", &suite.write_junit_xml,
167 "write JUnit-style XML files"),
171 memset(&suite, 0, sizeof(suite));
172 suite.tests.strdup_strings = suite.failed.strdup_strings = 1;
174 argc = parse_options(argc, argv, NULL, options,
175 testsuite_usage, PARSE_OPT_STOP_AT_NON_OPTION);
178 max_jobs = online_cpus();
182 die("Could not open the current directory");
183 while ((d = readdir(dir))) {
184 const char *p = d->d_name;
186 if (*p != 't' || !isdigit(p[1]) || !isdigit(p[2]) ||
187 !isdigit(p[3]) || !isdigit(p[4]) || p[5] != '-' ||
188 !ends_with(p, ".sh"))
191 /* No pattern: match all */
193 string_list_append(&suite.tests, p);
197 for (i = 0; i < argc; i++)
198 if (!wildmatch(argv[i], p, 0)) {
199 string_list_append(&suite.tests, p);
206 die("No tests match!");
207 if (max_jobs > suite.tests.nr)
208 max_jobs = suite.tests.nr;
210 fprintf(stderr, "Running %d tests (%d at a time)\n",
211 suite.tests.nr, max_jobs);
213 ret = run_processes_parallel(max_jobs, next_test, test_failed,
214 test_stdin, NULL, test_finished, &suite);
216 if (suite.failed.nr > 0) {
218 fprintf(stderr, "%d tests failed:\n\n", suite.failed.nr);
219 for (i = 0; i < suite.failed.nr; i++)
220 fprintf(stderr, "\t%s\n", suite.failed.items[i].string);
223 string_list_clear(&suite.tests, 0);
224 string_list_clear(&suite.failed, 0);
229 static uint64_t my_random_next = 1234;
231 static uint64_t my_random(void)
233 uint64_t res = my_random_next;
234 my_random_next = my_random_next * 1103515245 + 12345;
238 static int quote_stress_test(int argc, const char **argv)
241 * We are running a quote-stress test.
242 * spawn a subprocess that runs quote-stress with a
243 * special option that echoes back the arguments that
246 char special[] = ".?*\\^_\"'`{}()[]<>@~&+:;$%"; // \t\r\n\a";
247 int i, j, k, trials = 100, skip = 0, msys2 = 0;
248 struct strbuf out = STRBUF_INIT;
249 struct strvec args = STRVEC_INIT;
250 struct option options[] = {
251 OPT_INTEGER('n', "trials", &trials, "Number of trials"),
252 OPT_INTEGER('s', "skip", &skip, "Skip <n> trials"),
253 OPT_BOOL('m', "msys2", &msys2, "Test quoting for MSYS2's sh"),
256 const char * const usage[] = {
257 "test-tool run-command quote-stress-test <options>",
261 argc = parse_options(argc, argv, NULL, options, usage, 0);
263 setenv("MSYS_NO_PATHCONV", "1", 0);
265 for (i = 0; i < trials; i++) {
266 struct child_process cp = CHILD_PROCESS_INIT;
267 size_t arg_count, arg_offset;
272 strvec_pushl(&args, "sh", "-c",
273 "printf %s\\\\0 \"$@\"", "skip", NULL);
275 strvec_pushl(&args, "test-tool", "run-command",
277 arg_offset = args.nr;
282 for (j = 0; j < arg_count; j++)
283 strvec_push(&args, argv[j]);
285 arg_count = 1 + (my_random() % 5);
286 for (j = 0; j < arg_count; j++) {
289 size_t arg_len = min_len +
290 (my_random() % (ARRAY_SIZE(buf) - min_len));
292 for (k = 0; k < arg_len; k++)
293 buf[k] = special[my_random() %
294 ARRAY_SIZE(special)];
297 strvec_push(&args, buf);
306 if (pipe_command(&cp, NULL, 0, &out, 0, NULL, 0) < 0)
307 return error("Failed to spawn child process");
309 for (j = 0, k = 0; j < arg_count; j++) {
310 const char *arg = args.v[j + arg_offset];
312 if (strcmp(arg, out.buf + k))
313 ret = error("incorrectly quoted arg: '%s', "
314 "echoed back as '%s'",
316 k += strlen(out.buf + k) + 1;
320 ret = error("got %d bytes, but consumed only %d",
321 (int)out.len, (int)k);
324 fprintf(stderr, "Trial #%d failed. Arguments:\n", i);
325 for (j = 0; j < arg_count; j++)
326 fprintf(stderr, "arg #%d: '%s'\n",
327 (int)j, args.v[j + arg_offset]);
329 strbuf_release(&out);
335 if (i && (i % 100) == 0)
336 fprintf(stderr, "Trials completed: %d\n", (int)i);
339 strbuf_release(&out);
345 static int quote_echo(int argc, const char **argv)
348 fwrite(argv[1], strlen(argv[1]), 1, stdout);
357 static int inherit_handle(const char *argv0)
359 struct child_process cp = CHILD_PROCESS_INIT;
363 /* First, open an inheritable handle */
364 xsnprintf(path, sizeof(path), "out-XXXXXX");
365 tmp = xmkstemp(path);
367 strvec_pushl(&cp.args,
368 "test-tool", argv0, "inherited-handle-child", NULL);
370 cp.no_stdout = cp.no_stderr = 1;
371 if (start_command(&cp) < 0)
372 die("Could not start child process");
374 /* Then close it, and try to delete it. */
377 die("Could not delete '%s'", path);
379 if (close(cp.in) < 0 || finish_command(&cp) < 0)
380 die("Child did not finish");
385 static int inherit_handle_child(void)
387 struct strbuf buf = STRBUF_INIT;
389 if (strbuf_read(&buf, 0, 0) < 0)
390 die("Could not read stdin");
391 printf("Received %s\n", buf.buf);
392 strbuf_release(&buf);
397 int cmd__run_command(int argc, const char **argv)
399 struct child_process proc = CHILD_PROCESS_INIT;
402 if (argc > 1 && !strcmp(argv[1], "testsuite"))
403 exit(testsuite(argc - 1, argv + 1));
404 if (!strcmp(argv[1], "inherited-handle"))
405 exit(inherit_handle(argv[0]));
406 if (!strcmp(argv[1], "inherited-handle-child"))
407 exit(inherit_handle_child());
409 if (argc >= 2 && !strcmp(argv[1], "quote-stress-test"))
410 return !!quote_stress_test(argc - 1, argv + 1);
412 if (argc >= 2 && !strcmp(argv[1], "quote-echo"))
413 return !!quote_echo(argc - 1, argv + 1);
417 while (!strcmp(argv[1], "env")) {
419 die("env specifier without a value");
420 strvec_push(&proc.env_array, argv[2]);
426 proc.argv = (const char **)argv + 2;
428 if (!strcmp(argv[1], "start-command-ENOENT")) {
429 if (start_command(&proc) < 0 && errno == ENOENT)
431 fprintf(stderr, "FAIL %s\n", argv[1]);
434 if (!strcmp(argv[1], "run-command"))
435 exit(run_command(&proc));
437 jobs = atoi(argv[2]);
438 proc.argv = (const char **)argv + 3;
440 if (!strcmp(argv[1], "run-command-parallel"))
441 exit(run_processes_parallel(jobs, parallel_next,
442 NULL, NULL, NULL, NULL, &proc));
444 if (!strcmp(argv[1], "run-command-abort"))
445 exit(run_processes_parallel(jobs, parallel_next,
446 NULL, NULL, NULL, task_finished, &proc));
448 if (!strcmp(argv[1], "run-command-no-jobs"))
449 exit(run_processes_parallel(jobs, no_job,
450 NULL, NULL, NULL, task_finished, &proc));
452 if (!strcmp(argv[1], "run-command-stdin")) {
455 exit (run_processes_parallel(jobs, parallel_next, NULL,
456 test_stdin, NULL, NULL, &proc));
459 if (!strcmp(argv[1], "run-command-sideband"))
460 exit(run_processes_parallel(jobs, parallel_next, NULL, NULL,
461 test_consume_sideband, NULL,
464 fprintf(stderr, "check usage\n");