Merge branch 'jk/notes-merge-config'
[git] / transport-helper.c
1 #include "cache.h"
2 #include "transport.h"
3 #include "quote.h"
4 #include "run-command.h"
5 #include "commit.h"
6 #include "diff.h"
7 #include "revision.h"
8 #include "remote.h"
9 #include "string-list.h"
10 #include "thread-utils.h"
11 #include "sigchain.h"
12 #include "argv-array.h"
13 #include "refs.h"
14
15 static int debug;
16
17 struct helper_data {
18         const char *name;
19         struct child_process *helper;
20         FILE *out;
21         unsigned fetch : 1,
22                 import : 1,
23                 bidi_import : 1,
24                 export : 1,
25                 option : 1,
26                 push : 1,
27                 connect : 1,
28                 signed_tags : 1,
29                 check_connectivity : 1,
30                 no_disconnect_req : 1,
31                 no_private_update : 1;
32         char *export_marks;
33         char *import_marks;
34         /* These go from remote name (as in "list") to private name */
35         struct refspec *refspecs;
36         int refspec_nr;
37         /* Transport options for fetch-pack/send-pack (should one of
38          * those be invoked).
39          */
40         struct git_transport_options transport_options;
41 };
42
43 static void sendline(struct helper_data *helper, struct strbuf *buffer)
44 {
45         if (debug)
46                 fprintf(stderr, "Debug: Remote helper: -> %s", buffer->buf);
47         if (write_in_full(helper->helper->in, buffer->buf, buffer->len)
48                 != buffer->len)
49                 die_errno("Full write to remote helper failed");
50 }
51
52 static int recvline_fh(FILE *helper, struct strbuf *buffer, const char *name)
53 {
54         strbuf_reset(buffer);
55         if (debug)
56                 fprintf(stderr, "Debug: Remote helper: Waiting...\n");
57         if (strbuf_getline(buffer, helper, '\n') == EOF) {
58                 if (debug)
59                         fprintf(stderr, "Debug: Remote helper quit.\n");
60                 return 1;
61         }
62
63         if (debug)
64                 fprintf(stderr, "Debug: Remote helper: <- %s\n", buffer->buf);
65         return 0;
66 }
67
68 static int recvline(struct helper_data *helper, struct strbuf *buffer)
69 {
70         return recvline_fh(helper->out, buffer, helper->name);
71 }
72
73 static void write_constant(int fd, const char *str)
74 {
75         if (debug)
76                 fprintf(stderr, "Debug: Remote helper: -> %s", str);
77         if (write_in_full(fd, str, strlen(str)) != strlen(str))
78                 die_errno("Full write to remote helper failed");
79 }
80
81 static const char *remove_ext_force(const char *url)
82 {
83         if (url) {
84                 const char *colon = strchr(url, ':');
85                 if (colon && colon[1] == ':')
86                         return colon + 2;
87         }
88         return url;
89 }
90
91 static void do_take_over(struct transport *transport)
92 {
93         struct helper_data *data;
94         data = (struct helper_data *)transport->data;
95         transport_take_over(transport, data->helper);
96         fclose(data->out);
97         free(data);
98 }
99
100 static void standard_options(struct transport *t);
101
102 static struct child_process *get_helper(struct transport *transport)
103 {
104         struct helper_data *data = transport->data;
105         struct strbuf buf = STRBUF_INIT;
106         struct child_process *helper;
107         const char **refspecs = NULL;
108         int refspec_nr = 0;
109         int refspec_alloc = 0;
110         int duped;
111         int code;
112
113         if (data->helper)
114                 return data->helper;
115
116         helper = xmalloc(sizeof(*helper));
117         child_process_init(helper);
118         helper->in = -1;
119         helper->out = -1;
120         helper->err = 0;
121         argv_array_pushf(&helper->args, "git-remote-%s", data->name);
122         argv_array_push(&helper->args, transport->remote->name);
123         argv_array_push(&helper->args, remove_ext_force(transport->url));
124         helper->git_cmd = 0;
125         helper->silent_exec_failure = 1;
126
127         argv_array_pushf(&helper->env_array, "%s=%s", GIT_DIR_ENVIRONMENT,
128                          get_git_dir());
129
130         code = start_command(helper);
131         if (code < 0 && errno == ENOENT)
132                 die("Unable to find remote helper for '%s'", data->name);
133         else if (code != 0)
134                 exit(code);
135
136         data->helper = helper;
137         data->no_disconnect_req = 0;
138
139         /*
140          * Open the output as FILE* so strbuf_getline() can be used.
141          * Do this with duped fd because fclose() will close the fd,
142          * and stuff like taking over will require the fd to remain.
143          */
144         duped = dup(helper->out);
145         if (duped < 0)
146                 die_errno("Can't dup helper output fd");
147         data->out = xfdopen(duped, "r");
148
149         write_constant(helper->in, "capabilities\n");
150
151         while (1) {
152                 const char *capname, *arg;
153                 int mandatory = 0;
154                 if (recvline(data, &buf))
155                         exit(128);
156
157                 if (!*buf.buf)
158                         break;
159
160                 if (*buf.buf == '*') {
161                         capname = buf.buf + 1;
162                         mandatory = 1;
163                 } else
164                         capname = buf.buf;
165
166                 if (debug)
167                         fprintf(stderr, "Debug: Got cap %s\n", capname);
168                 if (!strcmp(capname, "fetch"))
169                         data->fetch = 1;
170                 else if (!strcmp(capname, "option"))
171                         data->option = 1;
172                 else if (!strcmp(capname, "push"))
173                         data->push = 1;
174                 else if (!strcmp(capname, "import"))
175                         data->import = 1;
176                 else if (!strcmp(capname, "bidi-import"))
177                         data->bidi_import = 1;
178                 else if (!strcmp(capname, "export"))
179                         data->export = 1;
180                 else if (!strcmp(capname, "check-connectivity"))
181                         data->check_connectivity = 1;
182                 else if (!data->refspecs && skip_prefix(capname, "refspec ", &arg)) {
183                         ALLOC_GROW(refspecs,
184                                    refspec_nr + 1,
185                                    refspec_alloc);
186                         refspecs[refspec_nr++] = xstrdup(arg);
187                 } else if (!strcmp(capname, "connect")) {
188                         data->connect = 1;
189                 } else if (!strcmp(capname, "signed-tags")) {
190                         data->signed_tags = 1;
191                 } else if (skip_prefix(capname, "export-marks ", &arg)) {
192                         data->export_marks = xstrdup(arg);
193                 } else if (skip_prefix(capname, "import-marks ", &arg)) {
194                         data->import_marks = xstrdup(arg);
195                 } else if (starts_with(capname, "no-private-update")) {
196                         data->no_private_update = 1;
197                 } else if (mandatory) {
198                         die("Unknown mandatory capability %s. This remote "
199                             "helper probably needs newer version of Git.",
200                             capname);
201                 }
202         }
203         if (refspecs) {
204                 int i;
205                 data->refspec_nr = refspec_nr;
206                 data->refspecs = parse_fetch_refspec(refspec_nr, refspecs);
207                 for (i = 0; i < refspec_nr; i++)
208                         free((char *)refspecs[i]);
209                 free(refspecs);
210         } else if (data->import || data->bidi_import || data->export) {
211                 warning("This remote helper should implement refspec capability.");
212         }
213         strbuf_release(&buf);
214         if (debug)
215                 fprintf(stderr, "Debug: Capabilities complete.\n");
216         standard_options(transport);
217         return data->helper;
218 }
219
220 static int disconnect_helper(struct transport *transport)
221 {
222         struct helper_data *data = transport->data;
223         int res = 0;
224
225         if (data->helper) {
226                 if (debug)
227                         fprintf(stderr, "Debug: Disconnecting.\n");
228                 if (!data->no_disconnect_req) {
229                         /*
230                          * Ignore write errors; there's nothing we can do,
231                          * since we're about to close the pipe anyway. And the
232                          * most likely error is EPIPE due to the helper dying
233                          * to report an error itself.
234                          */
235                         sigchain_push(SIGPIPE, SIG_IGN);
236                         xwrite(data->helper->in, "\n", 1);
237                         sigchain_pop(SIGPIPE);
238                 }
239                 close(data->helper->in);
240                 close(data->helper->out);
241                 fclose(data->out);
242                 res = finish_command(data->helper);
243                 free(data->helper);
244                 data->helper = NULL;
245         }
246         return res;
247 }
248
249 static const char *unsupported_options[] = {
250         TRANS_OPT_UPLOADPACK,
251         TRANS_OPT_RECEIVEPACK,
252         TRANS_OPT_THIN,
253         TRANS_OPT_KEEP
254         };
255
256 static const char *boolean_options[] = {
257         TRANS_OPT_THIN,
258         TRANS_OPT_KEEP,
259         TRANS_OPT_FOLLOWTAGS,
260         TRANS_OPT_PUSH_CERT
261         };
262
263 static int set_helper_option(struct transport *transport,
264                           const char *name, const char *value)
265 {
266         struct helper_data *data = transport->data;
267         struct strbuf buf = STRBUF_INIT;
268         int i, ret, is_bool = 0;
269
270         get_helper(transport);
271
272         if (!data->option)
273                 return 1;
274
275         for (i = 0; i < ARRAY_SIZE(unsupported_options); i++) {
276                 if (!strcmp(name, unsupported_options[i]))
277                         return 1;
278         }
279
280         for (i = 0; i < ARRAY_SIZE(boolean_options); i++) {
281                 if (!strcmp(name, boolean_options[i])) {
282                         is_bool = 1;
283                         break;
284                 }
285         }
286
287         strbuf_addf(&buf, "option %s ", name);
288         if (is_bool)
289                 strbuf_addstr(&buf, value ? "true" : "false");
290         else
291                 quote_c_style(value, &buf, NULL, 0);
292         strbuf_addch(&buf, '\n');
293
294         sendline(data, &buf);
295         if (recvline(data, &buf))
296                 exit(128);
297
298         if (!strcmp(buf.buf, "ok"))
299                 ret = 0;
300         else if (starts_with(buf.buf, "error")) {
301                 ret = -1;
302         } else if (!strcmp(buf.buf, "unsupported"))
303                 ret = 1;
304         else {
305                 warning("%s unexpectedly said: '%s'", data->name, buf.buf);
306                 ret = 1;
307         }
308         strbuf_release(&buf);
309         return ret;
310 }
311
312 static void standard_options(struct transport *t)
313 {
314         char buf[16];
315         int n;
316         int v = t->verbose;
317
318         set_helper_option(t, "progress", t->progress ? "true" : "false");
319
320         n = snprintf(buf, sizeof(buf), "%d", v + 1);
321         if (n >= sizeof(buf))
322                 die("impossibly large verbosity value");
323         set_helper_option(t, "verbosity", buf);
324 }
325
326 static int release_helper(struct transport *transport)
327 {
328         int res = 0;
329         struct helper_data *data = transport->data;
330         free_refspec(data->refspec_nr, data->refspecs);
331         data->refspecs = NULL;
332         res = disconnect_helper(transport);
333         free(transport->data);
334         return res;
335 }
336
337 static int fetch_with_fetch(struct transport *transport,
338                             int nr_heads, struct ref **to_fetch)
339 {
340         struct helper_data *data = transport->data;
341         int i;
342         struct strbuf buf = STRBUF_INIT;
343
344         for (i = 0; i < nr_heads; i++) {
345                 const struct ref *posn = to_fetch[i];
346                 if (posn->status & REF_STATUS_UPTODATE)
347                         continue;
348
349                 strbuf_addf(&buf, "fetch %s %s\n",
350                             sha1_to_hex(posn->old_sha1),
351                             posn->symref ? posn->symref : posn->name);
352         }
353
354         strbuf_addch(&buf, '\n');
355         sendline(data, &buf);
356
357         while (1) {
358                 if (recvline(data, &buf))
359                         exit(128);
360
361                 if (starts_with(buf.buf, "lock ")) {
362                         const char *name = buf.buf + 5;
363                         if (transport->pack_lockfile)
364                                 warning("%s also locked %s", data->name, name);
365                         else
366                                 transport->pack_lockfile = xstrdup(name);
367                 }
368                 else if (data->check_connectivity &&
369                          data->transport_options.check_self_contained_and_connected &&
370                          !strcmp(buf.buf, "connectivity-ok"))
371                         data->transport_options.self_contained_and_connected = 1;
372                 else if (!buf.len)
373                         break;
374                 else
375                         warning("%s unexpectedly said: '%s'", data->name, buf.buf);
376         }
377         strbuf_release(&buf);
378         return 0;
379 }
380
381 static int get_importer(struct transport *transport, struct child_process *fastimport)
382 {
383         struct child_process *helper = get_helper(transport);
384         struct helper_data *data = transport->data;
385         int cat_blob_fd, code;
386         child_process_init(fastimport);
387         fastimport->in = helper->out;
388         argv_array_push(&fastimport->args, "fast-import");
389         argv_array_push(&fastimport->args, debug ? "--stats" : "--quiet");
390
391         if (data->bidi_import) {
392                 cat_blob_fd = xdup(helper->in);
393                 argv_array_pushf(&fastimport->args, "--cat-blob-fd=%d", cat_blob_fd);
394         }
395         fastimport->git_cmd = 1;
396
397         code = start_command(fastimport);
398         return code;
399 }
400
401 static int get_exporter(struct transport *transport,
402                         struct child_process *fastexport,
403                         struct string_list *revlist_args)
404 {
405         struct helper_data *data = transport->data;
406         struct child_process *helper = get_helper(transport);
407         int i;
408
409         child_process_init(fastexport);
410
411         /* we need to duplicate helper->in because we want to use it after
412          * fastexport is done with it. */
413         fastexport->out = dup(helper->in);
414         argv_array_push(&fastexport->args, "fast-export");
415         argv_array_push(&fastexport->args, "--use-done-feature");
416         argv_array_push(&fastexport->args, data->signed_tags ?
417                 "--signed-tags=verbatim" : "--signed-tags=warn-strip");
418         if (data->export_marks)
419                 argv_array_pushf(&fastexport->args, "--export-marks=%s.tmp", data->export_marks);
420         if (data->import_marks)
421                 argv_array_pushf(&fastexport->args, "--import-marks=%s", data->import_marks);
422
423         for (i = 0; i < revlist_args->nr; i++)
424                 argv_array_push(&fastexport->args, revlist_args->items[i].string);
425
426         fastexport->git_cmd = 1;
427         return start_command(fastexport);
428 }
429
430 static int fetch_with_import(struct transport *transport,
431                              int nr_heads, struct ref **to_fetch)
432 {
433         struct child_process fastimport;
434         struct helper_data *data = transport->data;
435         int i;
436         struct ref *posn;
437         struct strbuf buf = STRBUF_INIT;
438
439         get_helper(transport);
440
441         if (get_importer(transport, &fastimport))
442                 die("Couldn't run fast-import");
443
444         for (i = 0; i < nr_heads; i++) {
445                 posn = to_fetch[i];
446                 if (posn->status & REF_STATUS_UPTODATE)
447                         continue;
448
449                 strbuf_addf(&buf, "import %s\n",
450                             posn->symref ? posn->symref : posn->name);
451                 sendline(data, &buf);
452                 strbuf_reset(&buf);
453         }
454
455         write_constant(data->helper->in, "\n");
456         /*
457          * remote-helpers that advertise the bidi-import capability are required to
458          * buffer the complete batch of import commands until this newline before
459          * sending data to fast-import.
460          * These helpers read back data from fast-import on their stdin, which could
461          * be mixed with import commands, otherwise.
462          */
463
464         if (finish_command(&fastimport))
465                 die("Error while running fast-import");
466
467         /*
468          * The fast-import stream of a remote helper that advertises
469          * the "refspec" capability writes to the refs named after the
470          * right hand side of the first refspec matching each ref we
471          * were fetching.
472          *
473          * (If no "refspec" capability was specified, for historical
474          * reasons we default to the equivalent of *:*.)
475          *
476          * Store the result in to_fetch[i].old_sha1.  Callers such
477          * as "git fetch" can use the value to write feedback to the
478          * terminal, populate FETCH_HEAD, and determine what new value
479          * should be written to peer_ref if the update is a
480          * fast-forward or this is a forced update.
481          */
482         for (i = 0; i < nr_heads; i++) {
483                 char *private, *name;
484                 posn = to_fetch[i];
485                 if (posn->status & REF_STATUS_UPTODATE)
486                         continue;
487                 name = posn->symref ? posn->symref : posn->name;
488                 if (data->refspecs)
489                         private = apply_refspecs(data->refspecs, data->refspec_nr, name);
490                 else
491                         private = xstrdup(name);
492                 if (private) {
493                         if (read_ref(private, posn->old_sha1) < 0)
494                                 die("Could not read ref %s", private);
495                         free(private);
496                 }
497         }
498         strbuf_release(&buf);
499         return 0;
500 }
501
502 static int process_connect_service(struct transport *transport,
503                                    const char *name, const char *exec)
504 {
505         struct helper_data *data = transport->data;
506         struct strbuf cmdbuf = STRBUF_INIT;
507         struct child_process *helper;
508         int r, duped, ret = 0;
509         FILE *input;
510
511         helper = get_helper(transport);
512
513         /*
514          * Yes, dup the pipe another time, as we need unbuffered version
515          * of input pipe as FILE*. fclose() closes the underlying fd and
516          * stream buffering only can be changed before first I/O operation
517          * on it.
518          */
519         duped = dup(helper->out);
520         if (duped < 0)
521                 die_errno("Can't dup helper output fd");
522         input = xfdopen(duped, "r");
523         setvbuf(input, NULL, _IONBF, 0);
524
525         /*
526          * Handle --upload-pack and friends. This is fire and forget...
527          * just warn if it fails.
528          */
529         if (strcmp(name, exec)) {
530                 r = set_helper_option(transport, "servpath", exec);
531                 if (r > 0)
532                         warning("Setting remote service path not supported by protocol.");
533                 else if (r < 0)
534                         warning("Invalid remote service path.");
535         }
536
537         if (data->connect)
538                 strbuf_addf(&cmdbuf, "connect %s\n", name);
539         else
540                 goto exit;
541
542         sendline(data, &cmdbuf);
543         if (recvline_fh(input, &cmdbuf, name))
544                 exit(128);
545
546         if (!strcmp(cmdbuf.buf, "")) {
547                 data->no_disconnect_req = 1;
548                 if (debug)
549                         fprintf(stderr, "Debug: Smart transport connection "
550                                 "ready.\n");
551                 ret = 1;
552         } else if (!strcmp(cmdbuf.buf, "fallback")) {
553                 if (debug)
554                         fprintf(stderr, "Debug: Falling back to dumb "
555                                 "transport.\n");
556         } else
557                 die("Unknown response to connect: %s",
558                         cmdbuf.buf);
559
560 exit:
561         fclose(input);
562         return ret;
563 }
564
565 static int process_connect(struct transport *transport,
566                                      int for_push)
567 {
568         struct helper_data *data = transport->data;
569         const char *name;
570         const char *exec;
571
572         name = for_push ? "git-receive-pack" : "git-upload-pack";
573         if (for_push)
574                 exec = data->transport_options.receivepack;
575         else
576                 exec = data->transport_options.uploadpack;
577
578         return process_connect_service(transport, name, exec);
579 }
580
581 static int connect_helper(struct transport *transport, const char *name,
582                    const char *exec, int fd[2])
583 {
584         struct helper_data *data = transport->data;
585
586         /* Get_helper so connect is inited. */
587         get_helper(transport);
588         if (!data->connect)
589                 die("Operation not supported by protocol.");
590
591         if (!process_connect_service(transport, name, exec))
592                 die("Can't connect to subservice %s.", name);
593
594         fd[0] = data->helper->out;
595         fd[1] = data->helper->in;
596         return 0;
597 }
598
599 static int fetch(struct transport *transport,
600                  int nr_heads, struct ref **to_fetch)
601 {
602         struct helper_data *data = transport->data;
603         int i, count;
604
605         if (process_connect(transport, 0)) {
606                 do_take_over(transport);
607                 return transport->fetch(transport, nr_heads, to_fetch);
608         }
609
610         count = 0;
611         for (i = 0; i < nr_heads; i++)
612                 if (!(to_fetch[i]->status & REF_STATUS_UPTODATE))
613                         count++;
614
615         if (!count)
616                 return 0;
617
618         if (data->check_connectivity &&
619             data->transport_options.check_self_contained_and_connected)
620                 set_helper_option(transport, "check-connectivity", "true");
621
622         if (transport->cloning)
623                 set_helper_option(transport, "cloning", "true");
624
625         if (data->transport_options.update_shallow)
626                 set_helper_option(transport, "update-shallow", "true");
627
628         if (data->fetch)
629                 return fetch_with_fetch(transport, nr_heads, to_fetch);
630
631         if (data->import)
632                 return fetch_with_import(transport, nr_heads, to_fetch);
633
634         return -1;
635 }
636
637 static int push_update_ref_status(struct strbuf *buf,
638                                    struct ref **ref,
639                                    struct ref *remote_refs)
640 {
641         char *refname, *msg;
642         int status, forced = 0;
643
644         if (starts_with(buf->buf, "ok ")) {
645                 status = REF_STATUS_OK;
646                 refname = buf->buf + 3;
647         } else if (starts_with(buf->buf, "error ")) {
648                 status = REF_STATUS_REMOTE_REJECT;
649                 refname = buf->buf + 6;
650         } else
651                 die("expected ok/error, helper said '%s'", buf->buf);
652
653         msg = strchr(refname, ' ');
654         if (msg) {
655                 struct strbuf msg_buf = STRBUF_INIT;
656                 const char *end;
657
658                 *msg++ = '\0';
659                 if (!unquote_c_style(&msg_buf, msg, &end))
660                         msg = strbuf_detach(&msg_buf, NULL);
661                 else
662                         msg = xstrdup(msg);
663                 strbuf_release(&msg_buf);
664
665                 if (!strcmp(msg, "no match")) {
666                         status = REF_STATUS_NONE;
667                         free(msg);
668                         msg = NULL;
669                 }
670                 else if (!strcmp(msg, "up to date")) {
671                         status = REF_STATUS_UPTODATE;
672                         free(msg);
673                         msg = NULL;
674                 }
675                 else if (!strcmp(msg, "non-fast forward")) {
676                         status = REF_STATUS_REJECT_NONFASTFORWARD;
677                         free(msg);
678                         msg = NULL;
679                 }
680                 else if (!strcmp(msg, "already exists")) {
681                         status = REF_STATUS_REJECT_ALREADY_EXISTS;
682                         free(msg);
683                         msg = NULL;
684                 }
685                 else if (!strcmp(msg, "fetch first")) {
686                         status = REF_STATUS_REJECT_FETCH_FIRST;
687                         free(msg);
688                         msg = NULL;
689                 }
690                 else if (!strcmp(msg, "needs force")) {
691                         status = REF_STATUS_REJECT_NEEDS_FORCE;
692                         free(msg);
693                         msg = NULL;
694                 }
695                 else if (!strcmp(msg, "stale info")) {
696                         status = REF_STATUS_REJECT_STALE;
697                         free(msg);
698                         msg = NULL;
699                 }
700                 else if (!strcmp(msg, "forced update")) {
701                         forced = 1;
702                         free(msg);
703                         msg = NULL;
704                 }
705         }
706
707         if (*ref)
708                 *ref = find_ref_by_name(*ref, refname);
709         if (!*ref)
710                 *ref = find_ref_by_name(remote_refs, refname);
711         if (!*ref) {
712                 warning("helper reported unexpected status of %s", refname);
713                 return 1;
714         }
715
716         if ((*ref)->status != REF_STATUS_NONE) {
717                 /*
718                  * Earlier, the ref was marked not to be pushed, so ignore the ref
719                  * status reported by the remote helper if the latter is 'no match'.
720                  */
721                 if (status == REF_STATUS_NONE)
722                         return 1;
723         }
724
725         (*ref)->status = status;
726         (*ref)->forced_update |= forced;
727         (*ref)->remote_status = msg;
728         return !(status == REF_STATUS_OK);
729 }
730
731 static int push_update_refs_status(struct helper_data *data,
732                                     struct ref *remote_refs,
733                                     int flags)
734 {
735         struct strbuf buf = STRBUF_INIT;
736         struct ref *ref = remote_refs;
737         int ret = 0;
738
739         for (;;) {
740                 char *private;
741
742                 if (recvline(data, &buf)) {
743                         ret = 1;
744                         break;
745                 }
746
747                 if (!buf.len)
748                         break;
749
750                 if (push_update_ref_status(&buf, &ref, remote_refs))
751                         continue;
752
753                 if (flags & TRANSPORT_PUSH_DRY_RUN || !data->refspecs || data->no_private_update)
754                         continue;
755
756                 /* propagate back the update to the remote namespace */
757                 private = apply_refspecs(data->refspecs, data->refspec_nr, ref->name);
758                 if (!private)
759                         continue;
760                 update_ref("update by helper", private, ref->new_sha1, NULL, 0, 0);
761                 free(private);
762         }
763         strbuf_release(&buf);
764         return ret;
765 }
766
767 static int push_refs_with_push(struct transport *transport,
768                                struct ref *remote_refs, int flags)
769 {
770         int force_all = flags & TRANSPORT_PUSH_FORCE;
771         int mirror = flags & TRANSPORT_PUSH_MIRROR;
772         struct helper_data *data = transport->data;
773         struct strbuf buf = STRBUF_INIT;
774         struct ref *ref;
775         struct string_list cas_options = STRING_LIST_INIT_DUP;
776         struct string_list_item *cas_option;
777
778         get_helper(transport);
779         if (!data->push)
780                 return 1;
781
782         for (ref = remote_refs; ref; ref = ref->next) {
783                 if (!ref->peer_ref && !mirror)
784                         continue;
785
786                 /* Check for statuses set by set_ref_status_for_push() */
787                 switch (ref->status) {
788                 case REF_STATUS_REJECT_NONFASTFORWARD:
789                 case REF_STATUS_REJECT_STALE:
790                 case REF_STATUS_REJECT_ALREADY_EXISTS:
791                 case REF_STATUS_UPTODATE:
792                         continue;
793                 default:
794                         ; /* do nothing */
795                 }
796
797                 if (force_all)
798                         ref->force = 1;
799
800                 strbuf_addstr(&buf, "push ");
801                 if (!ref->deletion) {
802                         if (ref->force)
803                                 strbuf_addch(&buf, '+');
804                         if (ref->peer_ref)
805                                 strbuf_addstr(&buf, ref->peer_ref->name);
806                         else
807                                 strbuf_addstr(&buf, sha1_to_hex(ref->new_sha1));
808                 }
809                 strbuf_addch(&buf, ':');
810                 strbuf_addstr(&buf, ref->name);
811                 strbuf_addch(&buf, '\n');
812
813                 /*
814                  * The "--force-with-lease" options without explicit
815                  * values to expect have already been expanded into
816                  * the ref->old_sha1_expect[] field; we can ignore
817                  * transport->smart_options->cas altogether and instead
818                  * can enumerate them from the refs.
819                  */
820                 if (ref->expect_old_sha1) {
821                         struct strbuf cas = STRBUF_INIT;
822                         strbuf_addf(&cas, "%s:%s",
823                                     ref->name, sha1_to_hex(ref->old_sha1_expect));
824                         string_list_append(&cas_options, strbuf_detach(&cas, NULL));
825                 }
826         }
827         if (buf.len == 0) {
828                 string_list_clear(&cas_options, 0);
829                 return 0;
830         }
831
832         for_each_string_list_item(cas_option, &cas_options)
833                 set_helper_option(transport, "cas", cas_option->string);
834
835         if (flags & TRANSPORT_PUSH_DRY_RUN) {
836                 if (set_helper_option(transport, "dry-run", "true") != 0)
837                         die("helper %s does not support dry-run", data->name);
838         } else if (flags & TRANSPORT_PUSH_CERT) {
839                 if (set_helper_option(transport, TRANS_OPT_PUSH_CERT, "true") != 0)
840                         die("helper %s does not support --signed", data->name);
841         }
842
843         strbuf_addch(&buf, '\n');
844         sendline(data, &buf);
845         strbuf_release(&buf);
846
847         return push_update_refs_status(data, remote_refs, flags);
848 }
849
850 static int push_refs_with_export(struct transport *transport,
851                 struct ref *remote_refs, int flags)
852 {
853         struct ref *ref;
854         struct child_process *helper, exporter;
855         struct helper_data *data = transport->data;
856         struct string_list revlist_args = STRING_LIST_INIT_DUP;
857         struct strbuf buf = STRBUF_INIT;
858
859         if (!data->refspecs)
860                 die("remote-helper doesn't support push; refspec needed");
861
862         if (flags & TRANSPORT_PUSH_DRY_RUN) {
863                 if (set_helper_option(transport, "dry-run", "true") != 0)
864                         die("helper %s does not support dry-run", data->name);
865         } else if (flags & TRANSPORT_PUSH_CERT) {
866                 if (set_helper_option(transport, TRANS_OPT_PUSH_CERT, "true") != 0)
867                         die("helper %s does not support --signed", data->name);
868         }
869
870         if (flags & TRANSPORT_PUSH_FORCE) {
871                 if (set_helper_option(transport, "force", "true") != 0)
872                         warning("helper %s does not support 'force'", data->name);
873         }
874
875         helper = get_helper(transport);
876
877         write_constant(helper->in, "export\n");
878
879         for (ref = remote_refs; ref; ref = ref->next) {
880                 char *private;
881                 unsigned char sha1[20];
882
883                 private = apply_refspecs(data->refspecs, data->refspec_nr, ref->name);
884                 if (private && !get_sha1(private, sha1)) {
885                         strbuf_addf(&buf, "^%s", private);
886                         string_list_append(&revlist_args, strbuf_detach(&buf, NULL));
887                         hashcpy(ref->old_sha1, sha1);
888                 }
889                 free(private);
890
891                 if (ref->peer_ref) {
892                         if (strcmp(ref->name, ref->peer_ref->name)) {
893                                 if (!ref->deletion) {
894                                         const char *name;
895                                         int flag;
896
897                                         /* Follow symbolic refs (mainly for HEAD). */
898                                         name = resolve_ref_unsafe(
899                                                  ref->peer_ref->name,
900                                                  RESOLVE_REF_READING,
901                                                  sha1, &flag);
902                                         if (!name || !(flag & REF_ISSYMREF))
903                                                 name = ref->peer_ref->name;
904
905                                         strbuf_addf(&buf, "%s:%s", name, ref->name);
906                                 } else
907                                         strbuf_addf(&buf, ":%s", ref->name);
908
909                                 string_list_append(&revlist_args, "--refspec");
910                                 string_list_append(&revlist_args, buf.buf);
911                                 strbuf_release(&buf);
912                         }
913                         if (!ref->deletion)
914                                 string_list_append(&revlist_args, ref->peer_ref->name);
915                 }
916         }
917
918         if (get_exporter(transport, &exporter, &revlist_args))
919                 die("Couldn't run fast-export");
920
921         string_list_clear(&revlist_args, 1);
922
923         if (finish_command(&exporter))
924                 die("Error while running fast-export");
925         if (push_update_refs_status(data, remote_refs, flags))
926                 return 1;
927
928         if (data->export_marks) {
929                 strbuf_addf(&buf, "%s.tmp", data->export_marks);
930                 rename(buf.buf, data->export_marks);
931                 strbuf_release(&buf);
932         }
933
934         return 0;
935 }
936
937 static int push_refs(struct transport *transport,
938                 struct ref *remote_refs, int flags)
939 {
940         struct helper_data *data = transport->data;
941
942         if (process_connect(transport, 1)) {
943                 do_take_over(transport);
944                 return transport->push_refs(transport, remote_refs, flags);
945         }
946
947         if (!remote_refs) {
948                 fprintf(stderr, "No refs in common and none specified; doing nothing.\n"
949                         "Perhaps you should specify a branch such as 'master'.\n");
950                 return 0;
951         }
952
953         if (data->push)
954                 return push_refs_with_push(transport, remote_refs, flags);
955
956         if (data->export)
957                 return push_refs_with_export(transport, remote_refs, flags);
958
959         return -1;
960 }
961
962
963 static int has_attribute(const char *attrs, const char *attr) {
964         int len;
965         if (!attrs)
966                 return 0;
967
968         len = strlen(attr);
969         for (;;) {
970                 const char *space = strchrnul(attrs, ' ');
971                 if (len == space - attrs && !strncmp(attrs, attr, len))
972                         return 1;
973                 if (!*space)
974                         return 0;
975                 attrs = space + 1;
976         }
977 }
978
979 static struct ref *get_refs_list(struct transport *transport, int for_push)
980 {
981         struct helper_data *data = transport->data;
982         struct child_process *helper;
983         struct ref *ret = NULL;
984         struct ref **tail = &ret;
985         struct ref *posn;
986         struct strbuf buf = STRBUF_INIT;
987
988         helper = get_helper(transport);
989
990         if (process_connect(transport, for_push)) {
991                 do_take_over(transport);
992                 return transport->get_refs_list(transport, for_push);
993         }
994
995         if (data->push && for_push)
996                 write_str_in_full(helper->in, "list for-push\n");
997         else
998                 write_str_in_full(helper->in, "list\n");
999
1000         while (1) {
1001                 char *eov, *eon;
1002                 if (recvline(data, &buf))
1003                         exit(128);
1004
1005                 if (!*buf.buf)
1006                         break;
1007
1008                 eov = strchr(buf.buf, ' ');
1009                 if (!eov)
1010                         die("Malformed response in ref list: %s", buf.buf);
1011                 eon = strchr(eov + 1, ' ');
1012                 *eov = '\0';
1013                 if (eon)
1014                         *eon = '\0';
1015                 *tail = alloc_ref(eov + 1);
1016                 if (buf.buf[0] == '@')
1017                         (*tail)->symref = xstrdup(buf.buf + 1);
1018                 else if (buf.buf[0] != '?')
1019                         get_sha1_hex(buf.buf, (*tail)->old_sha1);
1020                 if (eon) {
1021                         if (has_attribute(eon + 1, "unchanged")) {
1022                                 (*tail)->status |= REF_STATUS_UPTODATE;
1023                                 if (read_ref((*tail)->name,
1024                                              (*tail)->old_sha1) < 0)
1025                                         die(N_("Could not read ref %s"),
1026                                             (*tail)->name);
1027                         }
1028                 }
1029                 tail = &((*tail)->next);
1030         }
1031         if (debug)
1032                 fprintf(stderr, "Debug: Read ref listing.\n");
1033         strbuf_release(&buf);
1034
1035         for (posn = ret; posn; posn = posn->next)
1036                 resolve_remote_symref(posn, ret);
1037
1038         return ret;
1039 }
1040
1041 int transport_helper_init(struct transport *transport, const char *name)
1042 {
1043         struct helper_data *data = xcalloc(1, sizeof(*data));
1044         data->name = name;
1045
1046         if (getenv("GIT_TRANSPORT_HELPER_DEBUG"))
1047                 debug = 1;
1048
1049         transport->data = data;
1050         transport->set_option = set_helper_option;
1051         transport->get_refs_list = get_refs_list;
1052         transport->fetch = fetch;
1053         transport->push_refs = push_refs;
1054         transport->disconnect = release_helper;
1055         transport->connect = connect_helper;
1056         transport->smart_options = &(data->transport_options);
1057         return 0;
1058 }
1059
1060 /*
1061  * Linux pipes can buffer 65536 bytes at once (and most platforms can
1062  * buffer less), so attempt reads and writes with up to that size.
1063  */
1064 #define BUFFERSIZE 65536
1065 /* This should be enough to hold debugging message. */
1066 #define PBUFFERSIZE 8192
1067
1068 /* Print bidirectional transfer loop debug message. */
1069 __attribute__((format (printf, 1, 2)))
1070 static void transfer_debug(const char *fmt, ...)
1071 {
1072         va_list args;
1073         char msgbuf[PBUFFERSIZE];
1074         static int debug_enabled = -1;
1075
1076         if (debug_enabled < 0)
1077                 debug_enabled = getenv("GIT_TRANSLOOP_DEBUG") ? 1 : 0;
1078         if (!debug_enabled)
1079                 return;
1080
1081         va_start(args, fmt);
1082         vsnprintf(msgbuf, PBUFFERSIZE, fmt, args);
1083         va_end(args);
1084         fprintf(stderr, "Transfer loop debugging: %s\n", msgbuf);
1085 }
1086
1087 /* Stream state: More data may be coming in this direction. */
1088 #define SSTATE_TRANSFERING 0
1089 /*
1090  * Stream state: No more data coming in this direction, flushing rest of
1091  * data.
1092  */
1093 #define SSTATE_FLUSHING 1
1094 /* Stream state: Transfer in this direction finished. */
1095 #define SSTATE_FINISHED 2
1096
1097 #define STATE_NEEDS_READING(state) ((state) <= SSTATE_TRANSFERING)
1098 #define STATE_NEEDS_WRITING(state) ((state) <= SSTATE_FLUSHING)
1099 #define STATE_NEEDS_CLOSING(state) ((state) == SSTATE_FLUSHING)
1100
1101 /* Unidirectional transfer. */
1102 struct unidirectional_transfer {
1103         /* Source */
1104         int src;
1105         /* Destination */
1106         int dest;
1107         /* Is source socket? */
1108         int src_is_sock;
1109         /* Is destination socket? */
1110         int dest_is_sock;
1111         /* Transfer state (TRANSFERRING/FLUSHING/FINISHED) */
1112         int state;
1113         /* Buffer. */
1114         char buf[BUFFERSIZE];
1115         /* Buffer used. */
1116         size_t bufuse;
1117         /* Name of source. */
1118         const char *src_name;
1119         /* Name of destination. */
1120         const char *dest_name;
1121 };
1122
1123 /* Closes the target (for writing) if transfer has finished. */
1124 static void udt_close_if_finished(struct unidirectional_transfer *t)
1125 {
1126         if (STATE_NEEDS_CLOSING(t->state) && !t->bufuse) {
1127                 t->state = SSTATE_FINISHED;
1128                 if (t->dest_is_sock)
1129                         shutdown(t->dest, SHUT_WR);
1130                 else
1131                         close(t->dest);
1132                 transfer_debug("Closed %s.", t->dest_name);
1133         }
1134 }
1135
1136 /*
1137  * Tries to read read data from source into buffer. If buffer is full,
1138  * no data is read. Returns 0 on success, -1 on error.
1139  */
1140 static int udt_do_read(struct unidirectional_transfer *t)
1141 {
1142         ssize_t bytes;
1143
1144         if (t->bufuse == BUFFERSIZE)
1145                 return 0;       /* No space for more. */
1146
1147         transfer_debug("%s is readable", t->src_name);
1148         bytes = read(t->src, t->buf + t->bufuse, BUFFERSIZE - t->bufuse);
1149         if (bytes < 0 && errno != EWOULDBLOCK && errno != EAGAIN &&
1150                 errno != EINTR) {
1151                 error("read(%s) failed: %s", t->src_name, strerror(errno));
1152                 return -1;
1153         } else if (bytes == 0) {
1154                 transfer_debug("%s EOF (with %i bytes in buffer)",
1155                         t->src_name, (int)t->bufuse);
1156                 t->state = SSTATE_FLUSHING;
1157         } else if (bytes > 0) {
1158                 t->bufuse += bytes;
1159                 transfer_debug("Read %i bytes from %s (buffer now at %i)",
1160                         (int)bytes, t->src_name, (int)t->bufuse);
1161         }
1162         return 0;
1163 }
1164
1165 /* Tries to write data from buffer into destination. If buffer is empty,
1166  * no data is written. Returns 0 on success, -1 on error.
1167  */
1168 static int udt_do_write(struct unidirectional_transfer *t)
1169 {
1170         ssize_t bytes;
1171
1172         if (t->bufuse == 0)
1173                 return 0;       /* Nothing to write. */
1174
1175         transfer_debug("%s is writable", t->dest_name);
1176         bytes = xwrite(t->dest, t->buf, t->bufuse);
1177         if (bytes < 0 && errno != EWOULDBLOCK) {
1178                 error("write(%s) failed: %s", t->dest_name, strerror(errno));
1179                 return -1;
1180         } else if (bytes > 0) {
1181                 t->bufuse -= bytes;
1182                 if (t->bufuse)
1183                         memmove(t->buf, t->buf + bytes, t->bufuse);
1184                 transfer_debug("Wrote %i bytes to %s (buffer now at %i)",
1185                         (int)bytes, t->dest_name, (int)t->bufuse);
1186         }
1187         return 0;
1188 }
1189
1190
1191 /* State of bidirectional transfer loop. */
1192 struct bidirectional_transfer_state {
1193         /* Direction from program to git. */
1194         struct unidirectional_transfer ptg;
1195         /* Direction from git to program. */
1196         struct unidirectional_transfer gtp;
1197 };
1198
1199 static void *udt_copy_task_routine(void *udt)
1200 {
1201         struct unidirectional_transfer *t = (struct unidirectional_transfer *)udt;
1202         while (t->state != SSTATE_FINISHED) {
1203                 if (STATE_NEEDS_READING(t->state))
1204                         if (udt_do_read(t))
1205                                 return NULL;
1206                 if (STATE_NEEDS_WRITING(t->state))
1207                         if (udt_do_write(t))
1208                                 return NULL;
1209                 if (STATE_NEEDS_CLOSING(t->state))
1210                         udt_close_if_finished(t);
1211         }
1212         return udt;     /* Just some non-NULL value. */
1213 }
1214
1215 #ifndef NO_PTHREADS
1216
1217 /*
1218  * Join thread, with appropriate errors on failure. Name is name for the
1219  * thread (for error messages). Returns 0 on success, 1 on failure.
1220  */
1221 static int tloop_join(pthread_t thread, const char *name)
1222 {
1223         int err;
1224         void *tret;
1225         err = pthread_join(thread, &tret);
1226         if (!tret) {
1227                 error("%s thread failed", name);
1228                 return 1;
1229         }
1230         if (err) {
1231                 error("%s thread failed to join: %s", name, strerror(err));
1232                 return 1;
1233         }
1234         return 0;
1235 }
1236
1237 /*
1238  * Spawn the transfer tasks and then wait for them. Returns 0 on success,
1239  * -1 on failure.
1240  */
1241 static int tloop_spawnwait_tasks(struct bidirectional_transfer_state *s)
1242 {
1243         pthread_t gtp_thread;
1244         pthread_t ptg_thread;
1245         int err;
1246         int ret = 0;
1247         err = pthread_create(&gtp_thread, NULL, udt_copy_task_routine,
1248                 &s->gtp);
1249         if (err)
1250                 die("Can't start thread for copying data: %s", strerror(err));
1251         err = pthread_create(&ptg_thread, NULL, udt_copy_task_routine,
1252                 &s->ptg);
1253         if (err)
1254                 die("Can't start thread for copying data: %s", strerror(err));
1255
1256         ret |= tloop_join(gtp_thread, "Git to program copy");
1257         ret |= tloop_join(ptg_thread, "Program to git copy");
1258         return ret;
1259 }
1260 #else
1261
1262 /* Close the source and target (for writing) for transfer. */
1263 static void udt_kill_transfer(struct unidirectional_transfer *t)
1264 {
1265         t->state = SSTATE_FINISHED;
1266         /*
1267          * Socket read end left open isn't a disaster if nobody
1268          * attempts to read from it (mingw compat headers do not
1269          * have SHUT_RD)...
1270          *
1271          * We can't fully close the socket since otherwise gtp
1272          * task would first close the socket it sends data to
1273          * while closing the ptg file descriptors.
1274          */
1275         if (!t->src_is_sock)
1276                 close(t->src);
1277         if (t->dest_is_sock)
1278                 shutdown(t->dest, SHUT_WR);
1279         else
1280                 close(t->dest);
1281 }
1282
1283 /*
1284  * Join process, with appropriate errors on failure. Name is name for the
1285  * process (for error messages). Returns 0 on success, 1 on failure.
1286  */
1287 static int tloop_join(pid_t pid, const char *name)
1288 {
1289         int tret;
1290         if (waitpid(pid, &tret, 0) < 0) {
1291                 error("%s process failed to wait: %s", name, strerror(errno));
1292                 return 1;
1293         }
1294         if (!WIFEXITED(tret) || WEXITSTATUS(tret)) {
1295                 error("%s process failed", name);
1296                 return 1;
1297         }
1298         return 0;
1299 }
1300
1301 /*
1302  * Spawn the transfer tasks and then wait for them. Returns 0 on success,
1303  * -1 on failure.
1304  */
1305 static int tloop_spawnwait_tasks(struct bidirectional_transfer_state *s)
1306 {
1307         pid_t pid1, pid2;
1308         int ret = 0;
1309
1310         /* Fork thread #1: git to program. */
1311         pid1 = fork();
1312         if (pid1 < 0)
1313                 die_errno("Can't start thread for copying data");
1314         else if (pid1 == 0) {
1315                 udt_kill_transfer(&s->ptg);
1316                 exit(udt_copy_task_routine(&s->gtp) ? 0 : 1);
1317         }
1318
1319         /* Fork thread #2: program to git. */
1320         pid2 = fork();
1321         if (pid2 < 0)
1322                 die_errno("Can't start thread for copying data");
1323         else if (pid2 == 0) {
1324                 udt_kill_transfer(&s->gtp);
1325                 exit(udt_copy_task_routine(&s->ptg) ? 0 : 1);
1326         }
1327
1328         /*
1329          * Close both streams in parent as to not interfere with
1330          * end of file detection and wait for both tasks to finish.
1331          */
1332         udt_kill_transfer(&s->gtp);
1333         udt_kill_transfer(&s->ptg);
1334         ret |= tloop_join(pid1, "Git to program copy");
1335         ret |= tloop_join(pid2, "Program to git copy");
1336         return ret;
1337 }
1338 #endif
1339
1340 /*
1341  * Copies data from stdin to output and from input to stdout simultaneously.
1342  * Additionally filtering through given filter. If filter is NULL, uses
1343  * identity filter.
1344  */
1345 int bidirectional_transfer_loop(int input, int output)
1346 {
1347         struct bidirectional_transfer_state state;
1348
1349         /* Fill the state fields. */
1350         state.ptg.src = input;
1351         state.ptg.dest = 1;
1352         state.ptg.src_is_sock = (input == output);
1353         state.ptg.dest_is_sock = 0;
1354         state.ptg.state = SSTATE_TRANSFERING;
1355         state.ptg.bufuse = 0;
1356         state.ptg.src_name = "remote input";
1357         state.ptg.dest_name = "stdout";
1358
1359         state.gtp.src = 0;
1360         state.gtp.dest = output;
1361         state.gtp.src_is_sock = 0;
1362         state.gtp.dest_is_sock = (input == output);
1363         state.gtp.state = SSTATE_TRANSFERING;
1364         state.gtp.bufuse = 0;
1365         state.gtp.src_name = "stdin";
1366         state.gtp.dest_name = "remote output";
1367
1368         return tloop_spawnwait_tasks(&state);
1369 }