remote-curl: don't request v2 when pushing
[git] / remote-curl.c
1 #include "cache.h"
2 #include "config.h"
3 #include "remote.h"
4 #include "connect.h"
5 #include "strbuf.h"
6 #include "walker.h"
7 #include "http.h"
8 #include "exec_cmd.h"
9 #include "run-command.h"
10 #include "pkt-line.h"
11 #include "string-list.h"
12 #include "sideband.h"
13 #include "argv-array.h"
14 #include "credential.h"
15 #include "sha1-array.h"
16 #include "send-pack.h"
17 #include "protocol.h"
18
19 static struct remote *remote;
20 /* always ends with a trailing slash */
21 static struct strbuf url = STRBUF_INIT;
22
23 struct options {
24         int verbosity;
25         unsigned long depth;
26         char *deepen_since;
27         struct string_list deepen_not;
28         struct string_list push_options;
29         unsigned progress : 1,
30                 check_self_contained_and_connected : 1,
31                 cloning : 1,
32                 update_shallow : 1,
33                 followtags : 1,
34                 dry_run : 1,
35                 thin : 1,
36                 /* One of the SEND_PACK_PUSH_CERT_* constants. */
37                 push_cert : 2,
38                 deepen_relative : 1;
39 };
40 static struct options options;
41 static struct string_list cas_options = STRING_LIST_INIT_DUP;
42
43 static int set_option(const char *name, const char *value)
44 {
45         if (!strcmp(name, "verbosity")) {
46                 char *end;
47                 int v = strtol(value, &end, 10);
48                 if (value == end || *end)
49                         return -1;
50                 options.verbosity = v;
51                 return 0;
52         }
53         else if (!strcmp(name, "progress")) {
54                 if (!strcmp(value, "true"))
55                         options.progress = 1;
56                 else if (!strcmp(value, "false"))
57                         options.progress = 0;
58                 else
59                         return -1;
60                 return 0;
61         }
62         else if (!strcmp(name, "depth")) {
63                 char *end;
64                 unsigned long v = strtoul(value, &end, 10);
65                 if (value == end || *end)
66                         return -1;
67                 options.depth = v;
68                 return 0;
69         }
70         else if (!strcmp(name, "deepen-since")) {
71                 options.deepen_since = xstrdup(value);
72                 return 0;
73         }
74         else if (!strcmp(name, "deepen-not")) {
75                 string_list_append(&options.deepen_not, value);
76                 return 0;
77         }
78         else if (!strcmp(name, "deepen-relative")) {
79                 if (!strcmp(value, "true"))
80                         options.deepen_relative = 1;
81                 else if (!strcmp(value, "false"))
82                         options.deepen_relative = 0;
83                 else
84                         return -1;
85                 return 0;
86         }
87         else if (!strcmp(name, "followtags")) {
88                 if (!strcmp(value, "true"))
89                         options.followtags = 1;
90                 else if (!strcmp(value, "false"))
91                         options.followtags = 0;
92                 else
93                         return -1;
94                 return 0;
95         }
96         else if (!strcmp(name, "dry-run")) {
97                 if (!strcmp(value, "true"))
98                         options.dry_run = 1;
99                 else if (!strcmp(value, "false"))
100                         options.dry_run = 0;
101                 else
102                         return -1;
103                 return 0;
104         }
105         else if (!strcmp(name, "check-connectivity")) {
106                 if (!strcmp(value, "true"))
107                         options.check_self_contained_and_connected = 1;
108                 else if (!strcmp(value, "false"))
109                         options.check_self_contained_and_connected = 0;
110                 else
111                         return -1;
112                 return 0;
113         }
114         else if (!strcmp(name, "cas")) {
115                 struct strbuf val = STRBUF_INIT;
116                 strbuf_addf(&val, "--" CAS_OPT_NAME "=%s", value);
117                 string_list_append(&cas_options, val.buf);
118                 strbuf_release(&val);
119                 return 0;
120         } else if (!strcmp(name, "cloning")) {
121                 if (!strcmp(value, "true"))
122                         options.cloning = 1;
123                 else if (!strcmp(value, "false"))
124                         options.cloning = 0;
125                 else
126                         return -1;
127                 return 0;
128         } else if (!strcmp(name, "update-shallow")) {
129                 if (!strcmp(value, "true"))
130                         options.update_shallow = 1;
131                 else if (!strcmp(value, "false"))
132                         options.update_shallow = 0;
133                 else
134                         return -1;
135                 return 0;
136         } else if (!strcmp(name, "pushcert")) {
137                 if (!strcmp(value, "true"))
138                         options.push_cert = SEND_PACK_PUSH_CERT_ALWAYS;
139                 else if (!strcmp(value, "false"))
140                         options.push_cert = SEND_PACK_PUSH_CERT_NEVER;
141                 else if (!strcmp(value, "if-asked"))
142                         options.push_cert = SEND_PACK_PUSH_CERT_IF_ASKED;
143                 else
144                         return -1;
145                 return 0;
146         } else if (!strcmp(name, "push-option")) {
147                 string_list_append(&options.push_options, value);
148                 return 0;
149
150 #if LIBCURL_VERSION_NUM >= 0x070a08
151         } else if (!strcmp(name, "family")) {
152                 if (!strcmp(value, "ipv4"))
153                         git_curl_ipresolve = CURL_IPRESOLVE_V4;
154                 else if (!strcmp(value, "ipv6"))
155                         git_curl_ipresolve = CURL_IPRESOLVE_V6;
156                 else if (!strcmp(value, "all"))
157                         git_curl_ipresolve = CURL_IPRESOLVE_WHATEVER;
158                 else
159                         return -1;
160                 return 0;
161 #endif /* LIBCURL_VERSION_NUM >= 0x070a08 */
162         } else {
163                 return 1 /* unsupported */;
164         }
165 }
166
167 struct discovery {
168         char *service;
169         char *buf_alloc;
170         char *buf;
171         size_t len;
172         struct ref *refs;
173         struct oid_array shallow;
174         enum protocol_version version;
175         unsigned proto_git : 1;
176 };
177 static struct discovery *last_discovery;
178
179 static struct ref *parse_git_refs(struct discovery *heads, int for_push)
180 {
181         struct ref *list = NULL;
182         struct packet_reader reader;
183
184         packet_reader_init(&reader, -1, heads->buf, heads->len,
185                            PACKET_READ_CHOMP_NEWLINE |
186                            PACKET_READ_GENTLE_ON_EOF);
187
188         heads->version = discover_version(&reader);
189         switch (heads->version) {
190         case protocol_v2:
191                 /*
192                  * Do nothing.  This isn't a list of refs but rather a
193                  * capability advertisement.  Client would have run
194                  * 'stateless-connect' so we'll dump this capability listing
195                  * and let them request the refs themselves.
196                  */
197                 break;
198         case protocol_v1:
199         case protocol_v0:
200                 get_remote_heads(&reader, &list, for_push ? REF_NORMAL : 0,
201                                  NULL, &heads->shallow);
202                 break;
203         case protocol_unknown_version:
204                 BUG("unknown protocol version");
205         }
206
207         return list;
208 }
209
210 static struct ref *parse_info_refs(struct discovery *heads)
211 {
212         char *data, *start, *mid;
213         char *ref_name;
214         int i = 0;
215
216         struct ref *refs = NULL;
217         struct ref *ref = NULL;
218         struct ref *last_ref = NULL;
219
220         data = heads->buf;
221         start = NULL;
222         mid = data;
223         while (i < heads->len) {
224                 if (!start) {
225                         start = &data[i];
226                 }
227                 if (data[i] == '\t')
228                         mid = &data[i];
229                 if (data[i] == '\n') {
230                         if (mid - start != 40)
231                                 die("%sinfo/refs not valid: is this a git repository?",
232                                     url.buf);
233                         data[i] = 0;
234                         ref_name = mid + 1;
235                         ref = alloc_ref(ref_name);
236                         get_oid_hex(start, &ref->old_oid);
237                         if (!refs)
238                                 refs = ref;
239                         if (last_ref)
240                                 last_ref->next = ref;
241                         last_ref = ref;
242                         start = NULL;
243                 }
244                 i++;
245         }
246
247         ref = alloc_ref("HEAD");
248         if (!http_fetch_ref(url.buf, ref) &&
249             !resolve_remote_symref(ref, refs)) {
250                 ref->next = refs;
251                 refs = ref;
252         } else {
253                 free(ref);
254         }
255
256         return refs;
257 }
258
259 static void free_discovery(struct discovery *d)
260 {
261         if (d) {
262                 if (d == last_discovery)
263                         last_discovery = NULL;
264                 free(d->shallow.oid);
265                 free(d->buf_alloc);
266                 free_refs(d->refs);
267                 free(d->service);
268                 free(d);
269         }
270 }
271
272 static int show_http_message(struct strbuf *type, struct strbuf *charset,
273                              struct strbuf *msg)
274 {
275         const char *p, *eol;
276
277         /*
278          * We only show text/plain parts, as other types are likely
279          * to be ugly to look at on the user's terminal.
280          */
281         if (strcmp(type->buf, "text/plain"))
282                 return -1;
283         if (charset->len)
284                 strbuf_reencode(msg, charset->buf, get_log_output_encoding());
285
286         strbuf_trim(msg);
287         if (!msg->len)
288                 return -1;
289
290         p = msg->buf;
291         do {
292                 eol = strchrnul(p, '\n');
293                 fprintf(stderr, "remote: %.*s\n", (int)(eol - p), p);
294                 p = eol + 1;
295         } while(*eol);
296         return 0;
297 }
298
299 static int get_protocol_http_header(enum protocol_version version,
300                                     struct strbuf *header)
301 {
302         if (version > 0) {
303                 strbuf_addf(header, GIT_PROTOCOL_HEADER ": version=%d",
304                             version);
305
306                 return 1;
307         }
308
309         return 0;
310 }
311
312 static struct discovery *discover_refs(const char *service, int for_push)
313 {
314         struct strbuf exp = STRBUF_INIT;
315         struct strbuf type = STRBUF_INIT;
316         struct strbuf charset = STRBUF_INIT;
317         struct strbuf buffer = STRBUF_INIT;
318         struct strbuf refs_url = STRBUF_INIT;
319         struct strbuf effective_url = STRBUF_INIT;
320         struct strbuf protocol_header = STRBUF_INIT;
321         struct string_list extra_headers = STRING_LIST_INIT_DUP;
322         struct discovery *last = last_discovery;
323         int http_ret, maybe_smart = 0;
324         struct http_get_options http_options;
325         enum protocol_version version = get_protocol_version_config();
326
327         if (last && !strcmp(service, last->service))
328                 return last;
329         free_discovery(last);
330
331         strbuf_addf(&refs_url, "%sinfo/refs", url.buf);
332         if ((starts_with(url.buf, "http://") || starts_with(url.buf, "https://")) &&
333              git_env_bool("GIT_SMART_HTTP", 1)) {
334                 maybe_smart = 1;
335                 if (!strchr(url.buf, '?'))
336                         strbuf_addch(&refs_url, '?');
337                 else
338                         strbuf_addch(&refs_url, '&');
339                 strbuf_addf(&refs_url, "service=%s", service);
340         }
341
342         /*
343          * NEEDSWORK: If we are trying to use protocol v2 and we are planning
344          * to perform a push, then fallback to v0 since the client doesn't know
345          * how to push yet using v2.
346          */
347         if (version == protocol_v2 && !strcmp("git-receive-pack", service))
348                 version = protocol_v0;
349
350         /* Add the extra Git-Protocol header */
351         if (get_protocol_http_header(version, &protocol_header))
352                 string_list_append(&extra_headers, protocol_header.buf);
353
354         memset(&http_options, 0, sizeof(http_options));
355         http_options.content_type = &type;
356         http_options.charset = &charset;
357         http_options.effective_url = &effective_url;
358         http_options.base_url = &url;
359         http_options.extra_headers = &extra_headers;
360         http_options.initial_request = 1;
361         http_options.no_cache = 1;
362         http_options.keep_error = 1;
363
364         http_ret = http_get_strbuf(refs_url.buf, &buffer, &http_options);
365         switch (http_ret) {
366         case HTTP_OK:
367                 break;
368         case HTTP_MISSING_TARGET:
369                 show_http_message(&type, &charset, &buffer);
370                 die("repository '%s' not found", url.buf);
371         case HTTP_NOAUTH:
372                 show_http_message(&type, &charset, &buffer);
373                 die("Authentication failed for '%s'", url.buf);
374         default:
375                 show_http_message(&type, &charset, &buffer);
376                 die("unable to access '%s': %s", url.buf, curl_errorstr);
377         }
378
379         if (options.verbosity && !starts_with(refs_url.buf, url.buf))
380                 warning(_("redirecting to %s"), url.buf);
381
382         last= xcalloc(1, sizeof(*last_discovery));
383         last->service = xstrdup(service);
384         last->buf_alloc = strbuf_detach(&buffer, &last->len);
385         last->buf = last->buf_alloc;
386
387         strbuf_addf(&exp, "application/x-%s-advertisement", service);
388         if (maybe_smart &&
389             (5 <= last->len && last->buf[4] == '#') &&
390             !strbuf_cmp(&exp, &type)) {
391                 char *line;
392
393                 /*
394                  * smart HTTP response; validate that the service
395                  * pkt-line matches our request.
396                  */
397                 line = packet_read_line_buf(&last->buf, &last->len, NULL);
398
399                 strbuf_reset(&exp);
400                 strbuf_addf(&exp, "# service=%s", service);
401                 if (strcmp(line, exp.buf))
402                         die("invalid server response; got '%s'", line);
403                 strbuf_release(&exp);
404
405                 /* The header can include additional metadata lines, up
406                  * until a packet flush marker.  Ignore these now, but
407                  * in the future we might start to scan them.
408                  */
409                 while (packet_read_line_buf(&last->buf, &last->len, NULL))
410                         ;
411
412                 last->proto_git = 1;
413         } else if (maybe_smart &&
414                    last->len > 5 && starts_with(last->buf + 4, "version 2")) {
415                 last->proto_git = 1;
416         }
417
418         if (last->proto_git)
419                 last->refs = parse_git_refs(last, for_push);
420         else
421                 last->refs = parse_info_refs(last);
422
423         strbuf_release(&refs_url);
424         strbuf_release(&exp);
425         strbuf_release(&type);
426         strbuf_release(&charset);
427         strbuf_release(&effective_url);
428         strbuf_release(&buffer);
429         strbuf_release(&protocol_header);
430         string_list_clear(&extra_headers, 0);
431         last_discovery = last;
432         return last;
433 }
434
435 static struct ref *get_refs(int for_push)
436 {
437         struct discovery *heads;
438
439         if (for_push)
440                 heads = discover_refs("git-receive-pack", for_push);
441         else
442                 heads = discover_refs("git-upload-pack", for_push);
443
444         return heads->refs;
445 }
446
447 static void output_refs(struct ref *refs)
448 {
449         struct ref *posn;
450         for (posn = refs; posn; posn = posn->next) {
451                 if (posn->symref)
452                         printf("@%s %s\n", posn->symref, posn->name);
453                 else
454                         printf("%s %s\n", oid_to_hex(&posn->old_oid), posn->name);
455         }
456         printf("\n");
457         fflush(stdout);
458 }
459
460 struct rpc_state {
461         const char *service_name;
462         const char **argv;
463         struct strbuf *stdin_preamble;
464         char *service_url;
465         char *hdr_content_type;
466         char *hdr_accept;
467         char *protocol_header;
468         char *buf;
469         size_t alloc;
470         size_t len;
471         size_t pos;
472         int in;
473         int out;
474         int any_written;
475         struct strbuf result;
476         unsigned gzip_request : 1;
477         unsigned initial_buffer : 1;
478 };
479
480 static size_t rpc_out(void *ptr, size_t eltsize,
481                 size_t nmemb, void *buffer_)
482 {
483         size_t max = eltsize * nmemb;
484         struct rpc_state *rpc = buffer_;
485         size_t avail = rpc->len - rpc->pos;
486
487         if (!avail) {
488                 rpc->initial_buffer = 0;
489                 avail = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0);
490                 if (!avail)
491                         return 0;
492                 rpc->pos = 0;
493                 rpc->len = avail;
494         }
495
496         if (max < avail)
497                 avail = max;
498         memcpy(ptr, rpc->buf + rpc->pos, avail);
499         rpc->pos += avail;
500         return avail;
501 }
502
503 #ifndef NO_CURL_IOCTL
504 static curlioerr rpc_ioctl(CURL *handle, int cmd, void *clientp)
505 {
506         struct rpc_state *rpc = clientp;
507
508         switch (cmd) {
509         case CURLIOCMD_NOP:
510                 return CURLIOE_OK;
511
512         case CURLIOCMD_RESTARTREAD:
513                 if (rpc->initial_buffer) {
514                         rpc->pos = 0;
515                         return CURLIOE_OK;
516                 }
517                 error("unable to rewind rpc post data - try increasing http.postBuffer");
518                 return CURLIOE_FAILRESTART;
519
520         default:
521                 return CURLIOE_UNKNOWNCMD;
522         }
523 }
524 #endif
525
526 static size_t rpc_in(char *ptr, size_t eltsize,
527                 size_t nmemb, void *buffer_)
528 {
529         size_t size = eltsize * nmemb;
530         struct rpc_state *rpc = buffer_;
531         if (size)
532                 rpc->any_written = 1;
533         write_or_die(rpc->in, ptr, size);
534         return size;
535 }
536
537 static int run_slot(struct active_request_slot *slot,
538                     struct slot_results *results)
539 {
540         int err;
541         struct slot_results results_buf;
542
543         if (!results)
544                 results = &results_buf;
545
546         err = run_one_slot(slot, results);
547
548         if (err != HTTP_OK && err != HTTP_REAUTH) {
549                 struct strbuf msg = STRBUF_INIT;
550                 if (results->http_code && results->http_code != 200)
551                         strbuf_addf(&msg, "HTTP %ld", results->http_code);
552                 if (results->curl_result != CURLE_OK) {
553                         if (msg.len)
554                                 strbuf_addch(&msg, ' ');
555                         strbuf_addf(&msg, "curl %d", results->curl_result);
556                         if (curl_errorstr[0]) {
557                                 strbuf_addch(&msg, ' ');
558                                 strbuf_addstr(&msg, curl_errorstr);
559                         }
560                 }
561                 error("RPC failed; %s", msg.buf);
562                 strbuf_release(&msg);
563         }
564
565         return err;
566 }
567
568 static int probe_rpc(struct rpc_state *rpc, struct slot_results *results)
569 {
570         struct active_request_slot *slot;
571         struct curl_slist *headers = http_copy_default_headers();
572         struct strbuf buf = STRBUF_INIT;
573         int err;
574
575         slot = get_active_slot();
576
577         headers = curl_slist_append(headers, rpc->hdr_content_type);
578         headers = curl_slist_append(headers, rpc->hdr_accept);
579
580         curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
581         curl_easy_setopt(slot->curl, CURLOPT_POST, 1);
582         curl_easy_setopt(slot->curl, CURLOPT_URL, rpc->service_url);
583         curl_easy_setopt(slot->curl, CURLOPT_ENCODING, NULL);
584         curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, "0000");
585         curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE, 4);
586         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
587         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
588         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buf);
589
590         err = run_slot(slot, results);
591
592         curl_slist_free_all(headers);
593         strbuf_release(&buf);
594         return err;
595 }
596
597 static curl_off_t xcurl_off_t(ssize_t len) {
598         if (len > maximum_signed_value_of_type(curl_off_t))
599                 die("cannot handle pushes this big");
600         return (curl_off_t) len;
601 }
602
603 static int post_rpc(struct rpc_state *rpc)
604 {
605         struct active_request_slot *slot;
606         struct curl_slist *headers = http_copy_default_headers();
607         int use_gzip = rpc->gzip_request;
608         char *gzip_body = NULL;
609         size_t gzip_size = 0;
610         int err, large_request = 0;
611         int needs_100_continue = 0;
612
613         /* Try to load the entire request, if we can fit it into the
614          * allocated buffer space we can use HTTP/1.0 and avoid the
615          * chunked encoding mess.
616          */
617         while (1) {
618                 size_t left = rpc->alloc - rpc->len;
619                 char *buf = rpc->buf + rpc->len;
620                 int n;
621
622                 if (left < LARGE_PACKET_MAX) {
623                         large_request = 1;
624                         use_gzip = 0;
625                         break;
626                 }
627
628                 n = packet_read(rpc->out, NULL, NULL, buf, left, 0);
629                 if (!n)
630                         break;
631                 rpc->len += n;
632         }
633
634         if (large_request) {
635                 struct slot_results results;
636
637                 do {
638                         err = probe_rpc(rpc, &results);
639                         if (err == HTTP_REAUTH)
640                                 credential_fill(&http_auth);
641                 } while (err == HTTP_REAUTH);
642                 if (err != HTTP_OK)
643                         return -1;
644
645                 if (results.auth_avail & CURLAUTH_GSSNEGOTIATE)
646                         needs_100_continue = 1;
647         }
648
649         headers = curl_slist_append(headers, rpc->hdr_content_type);
650         headers = curl_slist_append(headers, rpc->hdr_accept);
651         headers = curl_slist_append(headers, needs_100_continue ?
652                 "Expect: 100-continue" : "Expect:");
653
654         /* Add the extra Git-Protocol header */
655         if (rpc->protocol_header)
656                 headers = curl_slist_append(headers, rpc->protocol_header);
657
658 retry:
659         slot = get_active_slot();
660
661         curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
662         curl_easy_setopt(slot->curl, CURLOPT_POST, 1);
663         curl_easy_setopt(slot->curl, CURLOPT_URL, rpc->service_url);
664         curl_easy_setopt(slot->curl, CURLOPT_ENCODING, "gzip");
665
666         if (large_request) {
667                 /* The request body is large and the size cannot be predicted.
668                  * We must use chunked encoding to send it.
669                  */
670                 headers = curl_slist_append(headers, "Transfer-Encoding: chunked");
671                 rpc->initial_buffer = 1;
672                 curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, rpc_out);
673                 curl_easy_setopt(slot->curl, CURLOPT_INFILE, rpc);
674 #ifndef NO_CURL_IOCTL
675                 curl_easy_setopt(slot->curl, CURLOPT_IOCTLFUNCTION, rpc_ioctl);
676                 curl_easy_setopt(slot->curl, CURLOPT_IOCTLDATA, rpc);
677 #endif
678                 if (options.verbosity > 1) {
679                         fprintf(stderr, "POST %s (chunked)\n", rpc->service_name);
680                         fflush(stderr);
681                 }
682
683         } else if (gzip_body) {
684                 /*
685                  * If we are looping to retry authentication, then the previous
686                  * run will have set up the headers and gzip buffer already,
687                  * and we just need to send it.
688                  */
689                 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);
690                 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, xcurl_off_t(gzip_size));
691
692         } else if (use_gzip && 1024 < rpc->len) {
693                 /* The client backend isn't giving us compressed data so
694                  * we can try to deflate it ourselves, this may save on.
695                  * the transfer time.
696                  */
697                 git_zstream stream;
698                 int ret;
699
700                 git_deflate_init_gzip(&stream, Z_BEST_COMPRESSION);
701                 gzip_size = git_deflate_bound(&stream, rpc->len);
702                 gzip_body = xmalloc(gzip_size);
703
704                 stream.next_in = (unsigned char *)rpc->buf;
705                 stream.avail_in = rpc->len;
706                 stream.next_out = (unsigned char *)gzip_body;
707                 stream.avail_out = gzip_size;
708
709                 ret = git_deflate(&stream, Z_FINISH);
710                 if (ret != Z_STREAM_END)
711                         die("cannot deflate request; zlib deflate error %d", ret);
712
713                 ret = git_deflate_end_gently(&stream);
714                 if (ret != Z_OK)
715                         die("cannot deflate request; zlib end error %d", ret);
716
717                 gzip_size = stream.total_out;
718
719                 headers = curl_slist_append(headers, "Content-Encoding: gzip");
720                 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, gzip_body);
721                 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, xcurl_off_t(gzip_size));
722
723                 if (options.verbosity > 1) {
724                         fprintf(stderr, "POST %s (gzip %lu to %lu bytes)\n",
725                                 rpc->service_name,
726                                 (unsigned long)rpc->len, (unsigned long)gzip_size);
727                         fflush(stderr);
728                 }
729         } else {
730                 /* We know the complete request size in advance, use the
731                  * more normal Content-Length approach.
732                  */
733                 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, rpc->buf);
734                 curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDSIZE_LARGE, xcurl_off_t(rpc->len));
735                 if (options.verbosity > 1) {
736                         fprintf(stderr, "POST %s (%lu bytes)\n",
737                                 rpc->service_name, (unsigned long)rpc->len);
738                         fflush(stderr);
739                 }
740         }
741
742         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
743         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, rpc_in);
744         curl_easy_setopt(slot->curl, CURLOPT_FILE, rpc);
745
746
747         rpc->any_written = 0;
748         err = run_slot(slot, NULL);
749         if (err == HTTP_REAUTH && !large_request) {
750                 credential_fill(&http_auth);
751                 goto retry;
752         }
753         if (err != HTTP_OK)
754                 err = -1;
755
756         if (!rpc->any_written)
757                 err = -1;
758
759         curl_slist_free_all(headers);
760         free(gzip_body);
761         return err;
762 }
763
764 static int rpc_service(struct rpc_state *rpc, struct discovery *heads)
765 {
766         const char *svc = rpc->service_name;
767         struct strbuf buf = STRBUF_INIT;
768         struct strbuf *preamble = rpc->stdin_preamble;
769         struct child_process client = CHILD_PROCESS_INIT;
770         int err = 0;
771
772         client.in = -1;
773         client.out = -1;
774         client.git_cmd = 1;
775         client.argv = rpc->argv;
776         if (start_command(&client))
777                 exit(1);
778         if (preamble)
779                 write_or_die(client.in, preamble->buf, preamble->len);
780         if (heads)
781                 write_or_die(client.in, heads->buf, heads->len);
782
783         rpc->alloc = http_post_buffer;
784         rpc->buf = xmalloc(rpc->alloc);
785         rpc->in = client.in;
786         rpc->out = client.out;
787         strbuf_init(&rpc->result, 0);
788
789         strbuf_addf(&buf, "%s%s", url.buf, svc);
790         rpc->service_url = strbuf_detach(&buf, NULL);
791
792         strbuf_addf(&buf, "Content-Type: application/x-%s-request", svc);
793         rpc->hdr_content_type = strbuf_detach(&buf, NULL);
794
795         strbuf_addf(&buf, "Accept: application/x-%s-result", svc);
796         rpc->hdr_accept = strbuf_detach(&buf, NULL);
797
798         if (get_protocol_http_header(heads->version, &buf))
799                 rpc->protocol_header = strbuf_detach(&buf, NULL);
800         else
801                 rpc->protocol_header = NULL;
802
803         while (!err) {
804                 int n = packet_read(rpc->out, NULL, NULL, rpc->buf, rpc->alloc, 0);
805                 if (!n)
806                         break;
807                 rpc->pos = 0;
808                 rpc->len = n;
809                 err |= post_rpc(rpc);
810         }
811
812         close(client.in);
813         client.in = -1;
814         if (!err) {
815                 strbuf_read(&rpc->result, client.out, 0);
816         } else {
817                 char buf[4096];
818                 for (;;)
819                         if (xread(client.out, buf, sizeof(buf)) <= 0)
820                                 break;
821         }
822
823         close(client.out);
824         client.out = -1;
825
826         err |= finish_command(&client);
827         free(rpc->service_url);
828         free(rpc->hdr_content_type);
829         free(rpc->hdr_accept);
830         free(rpc->protocol_header);
831         free(rpc->buf);
832         strbuf_release(&buf);
833         return err;
834 }
835
836 static int fetch_dumb(int nr_heads, struct ref **to_fetch)
837 {
838         struct walker *walker;
839         char **targets;
840         int ret, i;
841
842         ALLOC_ARRAY(targets, nr_heads);
843         if (options.depth || options.deepen_since)
844                 die("dumb http transport does not support shallow capabilities");
845         for (i = 0; i < nr_heads; i++)
846                 targets[i] = xstrdup(oid_to_hex(&to_fetch[i]->old_oid));
847
848         walker = get_http_walker(url.buf);
849         walker->get_all = 1;
850         walker->get_tree = 1;
851         walker->get_history = 1;
852         walker->get_verbosely = options.verbosity >= 3;
853         walker->get_recover = 0;
854         ret = walker_fetch(walker, nr_heads, targets, NULL, NULL);
855         walker_free(walker);
856
857         for (i = 0; i < nr_heads; i++)
858                 free(targets[i]);
859         free(targets);
860
861         return ret ? error("fetch failed.") : 0;
862 }
863
864 static int fetch_git(struct discovery *heads,
865         int nr_heads, struct ref **to_fetch)
866 {
867         struct rpc_state rpc;
868         struct strbuf preamble = STRBUF_INIT;
869         int i, err;
870         struct argv_array args = ARGV_ARRAY_INIT;
871
872         argv_array_pushl(&args, "fetch-pack", "--stateless-rpc",
873                          "--stdin", "--lock-pack", NULL);
874         if (options.followtags)
875                 argv_array_push(&args, "--include-tag");
876         if (options.thin)
877                 argv_array_push(&args, "--thin");
878         if (options.verbosity >= 3)
879                 argv_array_pushl(&args, "-v", "-v", NULL);
880         if (options.check_self_contained_and_connected)
881                 argv_array_push(&args, "--check-self-contained-and-connected");
882         if (options.cloning)
883                 argv_array_push(&args, "--cloning");
884         if (options.update_shallow)
885                 argv_array_push(&args, "--update-shallow");
886         if (!options.progress)
887                 argv_array_push(&args, "--no-progress");
888         if (options.depth)
889                 argv_array_pushf(&args, "--depth=%lu", options.depth);
890         if (options.deepen_since)
891                 argv_array_pushf(&args, "--shallow-since=%s", options.deepen_since);
892         for (i = 0; i < options.deepen_not.nr; i++)
893                 argv_array_pushf(&args, "--shallow-exclude=%s",
894                                  options.deepen_not.items[i].string);
895         if (options.deepen_relative && options.depth)
896                 argv_array_push(&args, "--deepen-relative");
897         argv_array_push(&args, url.buf);
898
899         for (i = 0; i < nr_heads; i++) {
900                 struct ref *ref = to_fetch[i];
901                 if (!*ref->name)
902                         die("cannot fetch by sha1 over smart http");
903                 packet_buf_write(&preamble, "%s %s\n",
904                                  oid_to_hex(&ref->old_oid), ref->name);
905         }
906         packet_buf_flush(&preamble);
907
908         memset(&rpc, 0, sizeof(rpc));
909         rpc.service_name = "git-upload-pack",
910         rpc.argv = args.argv;
911         rpc.stdin_preamble = &preamble;
912         rpc.gzip_request = 1;
913
914         err = rpc_service(&rpc, heads);
915         if (rpc.result.len)
916                 write_or_die(1, rpc.result.buf, rpc.result.len);
917         strbuf_release(&rpc.result);
918         strbuf_release(&preamble);
919         argv_array_clear(&args);
920         return err;
921 }
922
923 static int fetch(int nr_heads, struct ref **to_fetch)
924 {
925         struct discovery *d = discover_refs("git-upload-pack", 0);
926         if (d->proto_git)
927                 return fetch_git(d, nr_heads, to_fetch);
928         else
929                 return fetch_dumb(nr_heads, to_fetch);
930 }
931
932 static void parse_fetch(struct strbuf *buf)
933 {
934         struct ref **to_fetch = NULL;
935         struct ref *list_head = NULL;
936         struct ref **list = &list_head;
937         int alloc_heads = 0, nr_heads = 0;
938
939         do {
940                 const char *p;
941                 if (skip_prefix(buf->buf, "fetch ", &p)) {
942                         const char *name;
943                         struct ref *ref;
944                         struct object_id old_oid;
945
946                         if (get_oid_hex(p, &old_oid))
947                                 die("protocol error: expected sha/ref, got %s'", p);
948                         if (p[GIT_SHA1_HEXSZ] == ' ')
949                                 name = p + GIT_SHA1_HEXSZ + 1;
950                         else if (!p[GIT_SHA1_HEXSZ])
951                                 name = "";
952                         else
953                                 die("protocol error: expected sha/ref, got %s'", p);
954
955                         ref = alloc_ref(name);
956                         oidcpy(&ref->old_oid, &old_oid);
957
958                         *list = ref;
959                         list = &ref->next;
960
961                         ALLOC_GROW(to_fetch, nr_heads + 1, alloc_heads);
962                         to_fetch[nr_heads++] = ref;
963                 }
964                 else
965                         die("http transport does not support %s", buf->buf);
966
967                 strbuf_reset(buf);
968                 if (strbuf_getline_lf(buf, stdin) == EOF)
969                         return;
970                 if (!*buf->buf)
971                         break;
972         } while (1);
973
974         if (fetch(nr_heads, to_fetch))
975                 exit(128); /* error already reported */
976         free_refs(list_head);
977         free(to_fetch);
978
979         printf("\n");
980         fflush(stdout);
981         strbuf_reset(buf);
982 }
983
984 static int push_dav(int nr_spec, char **specs)
985 {
986         struct child_process child = CHILD_PROCESS_INIT;
987         size_t i;
988
989         child.git_cmd = 1;
990         argv_array_push(&child.args, "http-push");
991         argv_array_push(&child.args, "--helper-status");
992         if (options.dry_run)
993                 argv_array_push(&child.args, "--dry-run");
994         if (options.verbosity > 1)
995                 argv_array_push(&child.args, "--verbose");
996         argv_array_push(&child.args, url.buf);
997         for (i = 0; i < nr_spec; i++)
998                 argv_array_push(&child.args, specs[i]);
999
1000         if (run_command(&child))
1001                 die("git-http-push failed");
1002         return 0;
1003 }
1004
1005 static int push_git(struct discovery *heads, int nr_spec, char **specs)
1006 {
1007         struct rpc_state rpc;
1008         int i, err;
1009         struct argv_array args;
1010         struct string_list_item *cas_option;
1011         struct strbuf preamble = STRBUF_INIT;
1012
1013         argv_array_init(&args);
1014         argv_array_pushl(&args, "send-pack", "--stateless-rpc", "--helper-status",
1015                          NULL);
1016
1017         if (options.thin)
1018                 argv_array_push(&args, "--thin");
1019         if (options.dry_run)
1020                 argv_array_push(&args, "--dry-run");
1021         if (options.push_cert == SEND_PACK_PUSH_CERT_ALWAYS)
1022                 argv_array_push(&args, "--signed=yes");
1023         else if (options.push_cert == SEND_PACK_PUSH_CERT_IF_ASKED)
1024                 argv_array_push(&args, "--signed=if-asked");
1025         if (options.verbosity == 0)
1026                 argv_array_push(&args, "--quiet");
1027         else if (options.verbosity > 1)
1028                 argv_array_push(&args, "--verbose");
1029         for (i = 0; i < options.push_options.nr; i++)
1030                 argv_array_pushf(&args, "--push-option=%s",
1031                                  options.push_options.items[i].string);
1032         argv_array_push(&args, options.progress ? "--progress" : "--no-progress");
1033         for_each_string_list_item(cas_option, &cas_options)
1034                 argv_array_push(&args, cas_option->string);
1035         argv_array_push(&args, url.buf);
1036
1037         argv_array_push(&args, "--stdin");
1038         for (i = 0; i < nr_spec; i++)
1039                 packet_buf_write(&preamble, "%s\n", specs[i]);
1040         packet_buf_flush(&preamble);
1041
1042         memset(&rpc, 0, sizeof(rpc));
1043         rpc.service_name = "git-receive-pack",
1044         rpc.argv = args.argv;
1045         rpc.stdin_preamble = &preamble;
1046
1047         err = rpc_service(&rpc, heads);
1048         if (rpc.result.len)
1049                 write_or_die(1, rpc.result.buf, rpc.result.len);
1050         strbuf_release(&rpc.result);
1051         strbuf_release(&preamble);
1052         argv_array_clear(&args);
1053         return err;
1054 }
1055
1056 static int push(int nr_spec, char **specs)
1057 {
1058         struct discovery *heads = discover_refs("git-receive-pack", 1);
1059         int ret;
1060
1061         if (heads->proto_git)
1062                 ret = push_git(heads, nr_spec, specs);
1063         else
1064                 ret = push_dav(nr_spec, specs);
1065         free_discovery(heads);
1066         return ret;
1067 }
1068
1069 static void parse_push(struct strbuf *buf)
1070 {
1071         char **specs = NULL;
1072         int alloc_spec = 0, nr_spec = 0, i, ret;
1073
1074         do {
1075                 if (starts_with(buf->buf, "push ")) {
1076                         ALLOC_GROW(specs, nr_spec + 1, alloc_spec);
1077                         specs[nr_spec++] = xstrdup(buf->buf + 5);
1078                 }
1079                 else
1080                         die("http transport does not support %s", buf->buf);
1081
1082                 strbuf_reset(buf);
1083                 if (strbuf_getline_lf(buf, stdin) == EOF)
1084                         goto free_specs;
1085                 if (!*buf->buf)
1086                         break;
1087         } while (1);
1088
1089         ret = push(nr_spec, specs);
1090         printf("\n");
1091         fflush(stdout);
1092
1093         if (ret)
1094                 exit(128); /* error already reported */
1095
1096  free_specs:
1097         for (i = 0; i < nr_spec; i++)
1098                 free(specs[i]);
1099         free(specs);
1100 }
1101
1102 /*
1103  * Used to represent the state of a connection to an HTTP server when
1104  * communicating using git's wire-protocol version 2.
1105  */
1106 struct proxy_state {
1107         char *service_name;
1108         char *service_url;
1109         struct curl_slist *headers;
1110         struct strbuf request_buffer;
1111         int in;
1112         int out;
1113         struct packet_reader reader;
1114         size_t pos;
1115         int seen_flush;
1116 };
1117
1118 static void proxy_state_init(struct proxy_state *p, const char *service_name,
1119                              enum protocol_version version)
1120 {
1121         struct strbuf buf = STRBUF_INIT;
1122
1123         memset(p, 0, sizeof(*p));
1124         p->service_name = xstrdup(service_name);
1125
1126         p->in = 0;
1127         p->out = 1;
1128         strbuf_init(&p->request_buffer, 0);
1129
1130         strbuf_addf(&buf, "%s%s", url.buf, p->service_name);
1131         p->service_url = strbuf_detach(&buf, NULL);
1132
1133         p->headers = http_copy_default_headers();
1134
1135         strbuf_addf(&buf, "Content-Type: application/x-%s-request", p->service_name);
1136         p->headers = curl_slist_append(p->headers, buf.buf);
1137         strbuf_reset(&buf);
1138
1139         strbuf_addf(&buf, "Accept: application/x-%s-result", p->service_name);
1140         p->headers = curl_slist_append(p->headers, buf.buf);
1141         strbuf_reset(&buf);
1142
1143         p->headers = curl_slist_append(p->headers, "Transfer-Encoding: chunked");
1144
1145         /* Add the Git-Protocol header */
1146         if (get_protocol_http_header(version, &buf))
1147                 p->headers = curl_slist_append(p->headers, buf.buf);
1148
1149         packet_reader_init(&p->reader, p->in, NULL, 0,
1150                            PACKET_READ_GENTLE_ON_EOF);
1151
1152         strbuf_release(&buf);
1153 }
1154
1155 static void proxy_state_clear(struct proxy_state *p)
1156 {
1157         free(p->service_name);
1158         free(p->service_url);
1159         curl_slist_free_all(p->headers);
1160         strbuf_release(&p->request_buffer);
1161 }
1162
1163 /*
1164  * CURLOPT_READFUNCTION callback function.
1165  * Attempts to copy over a single packet-line at a time into the
1166  * curl provided buffer.
1167  */
1168 static size_t proxy_in(char *buffer, size_t eltsize,
1169                        size_t nmemb, void *userdata)
1170 {
1171         size_t max;
1172         struct proxy_state *p = userdata;
1173         size_t avail = p->request_buffer.len - p->pos;
1174
1175
1176         if (eltsize != 1)
1177                 BUG("curl read callback called with size = %"PRIuMAX" != 1",
1178                     (uintmax_t)eltsize);
1179         max = nmemb;
1180
1181         if (!avail) {
1182                 if (p->seen_flush) {
1183                         p->seen_flush = 0;
1184                         return 0;
1185                 }
1186
1187                 strbuf_reset(&p->request_buffer);
1188                 switch (packet_reader_read(&p->reader)) {
1189                 case PACKET_READ_EOF:
1190                         die("unexpected EOF when reading from parent process");
1191                 case PACKET_READ_NORMAL:
1192                         packet_buf_write_len(&p->request_buffer, p->reader.line,
1193                                              p->reader.pktlen);
1194                         break;
1195                 case PACKET_READ_DELIM:
1196                         packet_buf_delim(&p->request_buffer);
1197                         break;
1198                 case PACKET_READ_FLUSH:
1199                         packet_buf_flush(&p->request_buffer);
1200                         p->seen_flush = 1;
1201                         break;
1202                 }
1203                 p->pos = 0;
1204                 avail = p->request_buffer.len;
1205         }
1206
1207         if (max < avail)
1208                 avail = max;
1209         memcpy(buffer, p->request_buffer.buf + p->pos, avail);
1210         p->pos += avail;
1211         return avail;
1212 }
1213
1214 static size_t proxy_out(char *buffer, size_t eltsize,
1215                         size_t nmemb, void *userdata)
1216 {
1217         size_t size;
1218         struct proxy_state *p = userdata;
1219
1220         if (eltsize != 1)
1221                 BUG("curl read callback called with size = %"PRIuMAX" != 1",
1222                     (uintmax_t)eltsize);
1223         size = nmemb;
1224
1225         write_or_die(p->out, buffer, size);
1226         return size;
1227 }
1228
1229 /* Issues a request to the HTTP server configured in `p` */
1230 static int proxy_request(struct proxy_state *p)
1231 {
1232         struct active_request_slot *slot;
1233
1234         slot = get_active_slot();
1235
1236         curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
1237         curl_easy_setopt(slot->curl, CURLOPT_POST, 1);
1238         curl_easy_setopt(slot->curl, CURLOPT_URL, p->service_url);
1239         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, p->headers);
1240
1241         /* Setup function to read request from client */
1242         curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, proxy_in);
1243         curl_easy_setopt(slot->curl, CURLOPT_READDATA, p);
1244
1245         /* Setup function to write server response to client */
1246         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, proxy_out);
1247         curl_easy_setopt(slot->curl, CURLOPT_WRITEDATA, p);
1248
1249         if (run_slot(slot, NULL) != HTTP_OK)
1250                 return -1;
1251
1252         return 0;
1253 }
1254
1255 static int stateless_connect(const char *service_name)
1256 {
1257         struct discovery *discover;
1258         struct proxy_state p;
1259
1260         /*
1261          * Run the info/refs request and see if the server supports protocol
1262          * v2.  If and only if the server supports v2 can we successfully
1263          * establish a stateless connection, otherwise we need to tell the
1264          * client to fallback to using other transport helper functions to
1265          * complete their request.
1266          */
1267         discover = discover_refs(service_name, 0);
1268         if (discover->version != protocol_v2) {
1269                 printf("fallback\n");
1270                 fflush(stdout);
1271                 return -1;
1272         } else {
1273                 /* Stateless Connection established */
1274                 printf("\n");
1275                 fflush(stdout);
1276         }
1277
1278         proxy_state_init(&p, service_name, discover->version);
1279
1280         /*
1281          * Dump the capability listing that we got from the server earlier
1282          * during the info/refs request.
1283          */
1284         write_or_die(p.out, discover->buf, discover->len);
1285
1286         /* Peek the next packet line.  Until we see EOF keep sending POSTs */
1287         while (packet_reader_peek(&p.reader) != PACKET_READ_EOF) {
1288                 if (proxy_request(&p)) {
1289                         /* We would have an err here */
1290                         break;
1291                 }
1292         }
1293
1294         proxy_state_clear(&p);
1295         return 0;
1296 }
1297
1298 int cmd_main(int argc, const char **argv)
1299 {
1300         struct strbuf buf = STRBUF_INIT;
1301         int nongit;
1302
1303         setup_git_directory_gently(&nongit);
1304         if (argc < 2) {
1305                 error("remote-curl: usage: git remote-curl <remote> [<url>]");
1306                 return 1;
1307         }
1308
1309         options.verbosity = 1;
1310         options.progress = !!isatty(2);
1311         options.thin = 1;
1312         string_list_init(&options.deepen_not, 1);
1313         string_list_init(&options.push_options, 1);
1314
1315         remote = remote_get(argv[1]);
1316
1317         if (argc > 2) {
1318                 end_url_with_slash(&url, argv[2]);
1319         } else {
1320                 end_url_with_slash(&url, remote->url[0]);
1321         }
1322
1323         http_init(remote, url.buf, 0);
1324
1325         do {
1326                 const char *arg;
1327
1328                 if (strbuf_getline_lf(&buf, stdin) == EOF) {
1329                         if (ferror(stdin))
1330                                 error("remote-curl: error reading command stream from git");
1331                         return 1;
1332                 }
1333                 if (buf.len == 0)
1334                         break;
1335                 if (starts_with(buf.buf, "fetch ")) {
1336                         if (nongit)
1337                                 die("remote-curl: fetch attempted without a local repo");
1338                         parse_fetch(&buf);
1339
1340                 } else if (!strcmp(buf.buf, "list") || starts_with(buf.buf, "list ")) {
1341                         int for_push = !!strstr(buf.buf + 4, "for-push");
1342                         output_refs(get_refs(for_push));
1343
1344                 } else if (starts_with(buf.buf, "push ")) {
1345                         parse_push(&buf);
1346
1347                 } else if (skip_prefix(buf.buf, "option ", &arg)) {
1348                         char *value = strchr(arg, ' ');
1349                         int result;
1350
1351                         if (value)
1352                                 *value++ = '\0';
1353                         else
1354                                 value = "true";
1355
1356                         result = set_option(arg, value);
1357                         if (!result)
1358                                 printf("ok\n");
1359                         else if (result < 0)
1360                                 printf("error invalid value\n");
1361                         else
1362                                 printf("unsupported\n");
1363                         fflush(stdout);
1364
1365                 } else if (!strcmp(buf.buf, "capabilities")) {
1366                         printf("stateless-connect\n");
1367                         printf("fetch\n");
1368                         printf("option\n");
1369                         printf("push\n");
1370                         printf("check-connectivity\n");
1371                         printf("\n");
1372                         fflush(stdout);
1373                 } else if (skip_prefix(buf.buf, "stateless-connect ", &arg)) {
1374                         if (!stateless_connect(arg))
1375                                 break;
1376                 } else {
1377                         error("remote-curl: unknown command '%s' from git", buf.buf);
1378                         return 1;
1379                 }
1380                 strbuf_reset(&buf);
1381         } while (1);
1382
1383         http_cleanup();
1384
1385         return 0;
1386 }