http: clear POSTFIELDS when initializing a slot
[git] / http.c
1 #include "http.h"
2 #include "pack.h"
3 #include "sideband.h"
4
5 int data_received;
6 int active_requests;
7 int http_is_verbose;
8 size_t http_post_buffer = 16 * LARGE_PACKET_MAX;
9
10 #ifdef USE_CURL_MULTI
11 static int max_requests = -1;
12 static CURLM *curlm;
13 #endif
14 #ifndef NO_CURL_EASY_DUPHANDLE
15 static CURL *curl_default;
16 #endif
17
18 #define PREV_BUF_SIZE 4096
19 #define RANGE_HEADER_SIZE 30
20
21 char curl_errorstr[CURL_ERROR_SIZE];
22
23 static int curl_ssl_verify = -1;
24 static const char *ssl_cert;
25 #if LIBCURL_VERSION_NUM >= 0x070903
26 static const char *ssl_key;
27 #endif
28 #if LIBCURL_VERSION_NUM >= 0x070908
29 static const char *ssl_capath;
30 #endif
31 static const char *ssl_cainfo;
32 static long curl_low_speed_limit = -1;
33 static long curl_low_speed_time = -1;
34 static int curl_ftp_no_epsv;
35 static const char *curl_http_proxy;
36 static char *user_name, *user_pass;
37
38 #if LIBCURL_VERSION_NUM >= 0x071700
39 /* Use CURLOPT_KEYPASSWD as is */
40 #elif LIBCURL_VERSION_NUM >= 0x070903
41 #define CURLOPT_KEYPASSWD CURLOPT_SSLKEYPASSWD
42 #else
43 #define CURLOPT_KEYPASSWD CURLOPT_SSLCERTPASSWD
44 #endif
45
46 static char *ssl_cert_password;
47 static int ssl_cert_password_required;
48
49 static struct curl_slist *pragma_header;
50 static struct curl_slist *no_pragma_header;
51
52 static struct active_request_slot *active_queue_head;
53
54 size_t fread_buffer(void *ptr, size_t eltsize, size_t nmemb, void *buffer_)
55 {
56         size_t size = eltsize * nmemb;
57         struct buffer *buffer = buffer_;
58
59         if (size > buffer->buf.len - buffer->posn)
60                 size = buffer->buf.len - buffer->posn;
61         memcpy(ptr, buffer->buf.buf + buffer->posn, size);
62         buffer->posn += size;
63
64         return size;
65 }
66
67 #ifndef NO_CURL_IOCTL
68 curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp)
69 {
70         struct buffer *buffer = clientp;
71
72         switch (cmd) {
73         case CURLIOCMD_NOP:
74                 return CURLIOE_OK;
75
76         case CURLIOCMD_RESTARTREAD:
77                 buffer->posn = 0;
78                 return CURLIOE_OK;
79
80         default:
81                 return CURLIOE_UNKNOWNCMD;
82         }
83 }
84 #endif
85
86 size_t fwrite_buffer(const void *ptr, size_t eltsize, size_t nmemb, void *buffer_)
87 {
88         size_t size = eltsize * nmemb;
89         struct strbuf *buffer = buffer_;
90
91         strbuf_add(buffer, ptr, size);
92         data_received++;
93         return size;
94 }
95
96 size_t fwrite_null(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf)
97 {
98         data_received++;
99         return eltsize * nmemb;
100 }
101
102 #ifdef USE_CURL_MULTI
103 static void process_curl_messages(void)
104 {
105         int num_messages;
106         struct active_request_slot *slot;
107         CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
108
109         while (curl_message != NULL) {
110                 if (curl_message->msg == CURLMSG_DONE) {
111                         int curl_result = curl_message->data.result;
112                         slot = active_queue_head;
113                         while (slot != NULL &&
114                                slot->curl != curl_message->easy_handle)
115                                 slot = slot->next;
116                         if (slot != NULL) {
117                                 curl_multi_remove_handle(curlm, slot->curl);
118                                 slot->curl_result = curl_result;
119                                 finish_active_slot(slot);
120                         } else {
121                                 fprintf(stderr, "Received DONE message for unknown request!\n");
122                         }
123                 } else {
124                         fprintf(stderr, "Unknown CURL message received: %d\n",
125                                 (int)curl_message->msg);
126                 }
127                 curl_message = curl_multi_info_read(curlm, &num_messages);
128         }
129 }
130 #endif
131
132 static int http_options(const char *var, const char *value, void *cb)
133 {
134         if (!strcmp("http.sslverify", var)) {
135                 curl_ssl_verify = git_config_bool(var, value);
136                 return 0;
137         }
138         if (!strcmp("http.sslcert", var))
139                 return git_config_string(&ssl_cert, var, value);
140 #if LIBCURL_VERSION_NUM >= 0x070903
141         if (!strcmp("http.sslkey", var))
142                 return git_config_string(&ssl_key, var, value);
143 #endif
144 #if LIBCURL_VERSION_NUM >= 0x070908
145         if (!strcmp("http.sslcapath", var))
146                 return git_config_string(&ssl_capath, var, value);
147 #endif
148         if (!strcmp("http.sslcainfo", var))
149                 return git_config_string(&ssl_cainfo, var, value);
150         if (!strcmp("http.sslcertpasswordprotected", var)) {
151                 if (git_config_bool(var, value))
152                         ssl_cert_password_required = 1;
153                 return 0;
154         }
155 #ifdef USE_CURL_MULTI
156         if (!strcmp("http.maxrequests", var)) {
157                 max_requests = git_config_int(var, value);
158                 return 0;
159         }
160 #endif
161         if (!strcmp("http.lowspeedlimit", var)) {
162                 curl_low_speed_limit = (long)git_config_int(var, value);
163                 return 0;
164         }
165         if (!strcmp("http.lowspeedtime", var)) {
166                 curl_low_speed_time = (long)git_config_int(var, value);
167                 return 0;
168         }
169
170         if (!strcmp("http.noepsv", var)) {
171                 curl_ftp_no_epsv = git_config_bool(var, value);
172                 return 0;
173         }
174         if (!strcmp("http.proxy", var))
175                 return git_config_string(&curl_http_proxy, var, value);
176
177         if (!strcmp("http.postbuffer", var)) {
178                 http_post_buffer = git_config_int(var, value);
179                 if (http_post_buffer < LARGE_PACKET_MAX)
180                         http_post_buffer = LARGE_PACKET_MAX;
181                 return 0;
182         }
183
184         /* Fall back on the default ones */
185         return git_default_config(var, value, cb);
186 }
187
188 static void init_curl_http_auth(CURL *result)
189 {
190         if (user_name) {
191                 struct strbuf up = STRBUF_INIT;
192                 if (!user_pass)
193                         user_pass = xstrdup(getpass("Password: "));
194                 strbuf_addf(&up, "%s:%s", user_name, user_pass);
195                 curl_easy_setopt(result, CURLOPT_USERPWD,
196                                  strbuf_detach(&up, NULL));
197         }
198 }
199
200 static int has_cert_password(void)
201 {
202         if (ssl_cert_password != NULL)
203                 return 1;
204         if (ssl_cert == NULL || ssl_cert_password_required != 1)
205                 return 0;
206         /* Only prompt the user once. */
207         ssl_cert_password_required = -1;
208         ssl_cert_password = getpass("Certificate Password: ");
209         if (ssl_cert_password != NULL) {
210                 ssl_cert_password = xstrdup(ssl_cert_password);
211                 return 1;
212         } else
213                 return 0;
214 }
215
216 static CURL *get_curl_handle(void)
217 {
218         CURL *result = curl_easy_init();
219
220         if (!curl_ssl_verify) {
221                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
222                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
223         } else {
224                 /* Verify authenticity of the peer's certificate */
225                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1);
226                 /* The name in the cert must match whom we tried to connect */
227                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2);
228         }
229
230 #if LIBCURL_VERSION_NUM >= 0x070907
231         curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
232 #endif
233
234         init_curl_http_auth(result);
235
236         if (ssl_cert != NULL)
237                 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
238         if (has_cert_password())
239                 curl_easy_setopt(result, CURLOPT_KEYPASSWD, ssl_cert_password);
240 #if LIBCURL_VERSION_NUM >= 0x070903
241         if (ssl_key != NULL)
242                 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
243 #endif
244 #if LIBCURL_VERSION_NUM >= 0x070908
245         if (ssl_capath != NULL)
246                 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
247 #endif
248         if (ssl_cainfo != NULL)
249                 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
250         curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
251
252         if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
253                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
254                                  curl_low_speed_limit);
255                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
256                                  curl_low_speed_time);
257         }
258
259         curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
260
261         if (getenv("GIT_CURL_VERBOSE"))
262                 curl_easy_setopt(result, CURLOPT_VERBOSE, 1);
263
264         curl_easy_setopt(result, CURLOPT_USERAGENT, GIT_USER_AGENT);
265
266         if (curl_ftp_no_epsv)
267                 curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0);
268
269         if (curl_http_proxy)
270                 curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy);
271
272         return result;
273 }
274
275 static void http_auth_init(const char *url)
276 {
277         char *at, *colon, *cp, *slash;
278         int len;
279
280         cp = strstr(url, "://");
281         if (!cp)
282                 return;
283
284         /*
285          * Ok, the URL looks like "proto://something".  Which one?
286          * "proto://<user>:<pass>@<host>/...",
287          * "proto://<user>@<host>/...", or just
288          * "proto://<host>/..."?
289          */
290         cp += 3;
291         at = strchr(cp, '@');
292         colon = strchr(cp, ':');
293         slash = strchrnul(cp, '/');
294         if (!at || slash <= at)
295                 return; /* No credentials */
296         if (!colon || at <= colon) {
297                 /* Only username */
298                 len = at - cp;
299                 user_name = xmalloc(len + 1);
300                 memcpy(user_name, cp, len);
301                 user_name[len] = '\0';
302                 user_pass = NULL;
303         } else {
304                 len = colon - cp;
305                 user_name = xmalloc(len + 1);
306                 memcpy(user_name, cp, len);
307                 user_name[len] = '\0';
308                 len = at - (colon + 1);
309                 user_pass = xmalloc(len + 1);
310                 memcpy(user_pass, colon + 1, len);
311                 user_pass[len] = '\0';
312         }
313 }
314
315 static void set_from_env(const char **var, const char *envname)
316 {
317         const char *val = getenv(envname);
318         if (val)
319                 *var = val;
320 }
321
322 void http_init(struct remote *remote)
323 {
324         char *low_speed_limit;
325         char *low_speed_time;
326
327         http_is_verbose = 0;
328
329         git_config(http_options, NULL);
330
331         curl_global_init(CURL_GLOBAL_ALL);
332
333         if (remote && remote->http_proxy)
334                 curl_http_proxy = xstrdup(remote->http_proxy);
335
336         pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
337         no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
338
339 #ifdef USE_CURL_MULTI
340         {
341                 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
342                 if (http_max_requests != NULL)
343                         max_requests = atoi(http_max_requests);
344         }
345
346         curlm = curl_multi_init();
347         if (curlm == NULL) {
348                 fprintf(stderr, "Error creating curl multi handle.\n");
349                 exit(1);
350         }
351 #endif
352
353         if (getenv("GIT_SSL_NO_VERIFY"))
354                 curl_ssl_verify = 0;
355
356         set_from_env(&ssl_cert, "GIT_SSL_CERT");
357 #if LIBCURL_VERSION_NUM >= 0x070903
358         set_from_env(&ssl_key, "GIT_SSL_KEY");
359 #endif
360 #if LIBCURL_VERSION_NUM >= 0x070908
361         set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
362 #endif
363         set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
364
365         low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
366         if (low_speed_limit != NULL)
367                 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
368         low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
369         if (low_speed_time != NULL)
370                 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
371
372         if (curl_ssl_verify == -1)
373                 curl_ssl_verify = 1;
374
375 #ifdef USE_CURL_MULTI
376         if (max_requests < 1)
377                 max_requests = DEFAULT_MAX_REQUESTS;
378 #endif
379
380         if (getenv("GIT_CURL_FTP_NO_EPSV"))
381                 curl_ftp_no_epsv = 1;
382
383         if (remote && remote->url && remote->url[0]) {
384                 http_auth_init(remote->url[0]);
385                 if (!ssl_cert_password_required &&
386                     getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
387                     !prefixcmp(remote->url[0], "https://"))
388                         ssl_cert_password_required = 1;
389         }
390
391 #ifndef NO_CURL_EASY_DUPHANDLE
392         curl_default = get_curl_handle();
393 #endif
394 }
395
396 void http_cleanup(void)
397 {
398         struct active_request_slot *slot = active_queue_head;
399
400         while (slot != NULL) {
401                 struct active_request_slot *next = slot->next;
402                 if (slot->curl != NULL) {
403 #ifdef USE_CURL_MULTI
404                         curl_multi_remove_handle(curlm, slot->curl);
405 #endif
406                         curl_easy_cleanup(slot->curl);
407                 }
408                 free(slot);
409                 slot = next;
410         }
411         active_queue_head = NULL;
412
413 #ifndef NO_CURL_EASY_DUPHANDLE
414         curl_easy_cleanup(curl_default);
415 #endif
416
417 #ifdef USE_CURL_MULTI
418         curl_multi_cleanup(curlm);
419 #endif
420         curl_global_cleanup();
421
422         curl_slist_free_all(pragma_header);
423         pragma_header = NULL;
424
425         curl_slist_free_all(no_pragma_header);
426         no_pragma_header = NULL;
427
428         if (curl_http_proxy) {
429                 free((void *)curl_http_proxy);
430                 curl_http_proxy = NULL;
431         }
432
433         if (ssl_cert_password != NULL) {
434                 memset(ssl_cert_password, 0, strlen(ssl_cert_password));
435                 free(ssl_cert_password);
436                 ssl_cert_password = NULL;
437         }
438         ssl_cert_password_required = 0;
439 }
440
441 struct active_request_slot *get_active_slot(void)
442 {
443         struct active_request_slot *slot = active_queue_head;
444         struct active_request_slot *newslot;
445
446 #ifdef USE_CURL_MULTI
447         int num_transfers;
448
449         /* Wait for a slot to open up if the queue is full */
450         while (active_requests >= max_requests) {
451                 curl_multi_perform(curlm, &num_transfers);
452                 if (num_transfers < active_requests)
453                         process_curl_messages();
454         }
455 #endif
456
457         while (slot != NULL && slot->in_use)
458                 slot = slot->next;
459
460         if (slot == NULL) {
461                 newslot = xmalloc(sizeof(*newslot));
462                 newslot->curl = NULL;
463                 newslot->in_use = 0;
464                 newslot->next = NULL;
465
466                 slot = active_queue_head;
467                 if (slot == NULL) {
468                         active_queue_head = newslot;
469                 } else {
470                         while (slot->next != NULL)
471                                 slot = slot->next;
472                         slot->next = newslot;
473                 }
474                 slot = newslot;
475         }
476
477         if (slot->curl == NULL) {
478 #ifdef NO_CURL_EASY_DUPHANDLE
479                 slot->curl = get_curl_handle();
480 #else
481                 slot->curl = curl_easy_duphandle(curl_default);
482 #endif
483         }
484
485         active_requests++;
486         slot->in_use = 1;
487         slot->local = NULL;
488         slot->results = NULL;
489         slot->finished = NULL;
490         slot->callback_data = NULL;
491         slot->callback_func = NULL;
492         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
493         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
494         curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
495         curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
496         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
497         curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL);
498         curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
499         curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
500
501         return slot;
502 }
503
504 int start_active_slot(struct active_request_slot *slot)
505 {
506 #ifdef USE_CURL_MULTI
507         CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
508         int num_transfers;
509
510         if (curlm_result != CURLM_OK &&
511             curlm_result != CURLM_CALL_MULTI_PERFORM) {
512                 active_requests--;
513                 slot->in_use = 0;
514                 return 0;
515         }
516
517         /*
518          * We know there must be something to do, since we just added
519          * something.
520          */
521         curl_multi_perform(curlm, &num_transfers);
522 #endif
523         return 1;
524 }
525
526 #ifdef USE_CURL_MULTI
527 struct fill_chain {
528         void *data;
529         int (*fill)(void *);
530         struct fill_chain *next;
531 };
532
533 static struct fill_chain *fill_cfg;
534
535 void add_fill_function(void *data, int (*fill)(void *))
536 {
537         struct fill_chain *new = xmalloc(sizeof(*new));
538         struct fill_chain **linkp = &fill_cfg;
539         new->data = data;
540         new->fill = fill;
541         new->next = NULL;
542         while (*linkp)
543                 linkp = &(*linkp)->next;
544         *linkp = new;
545 }
546
547 void fill_active_slots(void)
548 {
549         struct active_request_slot *slot = active_queue_head;
550
551         while (active_requests < max_requests) {
552                 struct fill_chain *fill;
553                 for (fill = fill_cfg; fill; fill = fill->next)
554                         if (fill->fill(fill->data))
555                                 break;
556
557                 if (!fill)
558                         break;
559         }
560
561         while (slot != NULL) {
562                 if (!slot->in_use && slot->curl != NULL) {
563                         curl_easy_cleanup(slot->curl);
564                         slot->curl = NULL;
565                 }
566                 slot = slot->next;
567         }
568 }
569
570 void step_active_slots(void)
571 {
572         int num_transfers;
573         CURLMcode curlm_result;
574
575         do {
576                 curlm_result = curl_multi_perform(curlm, &num_transfers);
577         } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
578         if (num_transfers < active_requests) {
579                 process_curl_messages();
580                 fill_active_slots();
581         }
582 }
583 #endif
584
585 void run_active_slot(struct active_request_slot *slot)
586 {
587 #ifdef USE_CURL_MULTI
588         long last_pos = 0;
589         long current_pos;
590         fd_set readfds;
591         fd_set writefds;
592         fd_set excfds;
593         int max_fd;
594         struct timeval select_timeout;
595         int finished = 0;
596
597         slot->finished = &finished;
598         while (!finished) {
599                 data_received = 0;
600                 step_active_slots();
601
602                 if (!data_received && slot->local != NULL) {
603                         current_pos = ftell(slot->local);
604                         if (current_pos > last_pos)
605                                 data_received++;
606                         last_pos = current_pos;
607                 }
608
609                 if (slot->in_use && !data_received) {
610                         max_fd = 0;
611                         FD_ZERO(&readfds);
612                         FD_ZERO(&writefds);
613                         FD_ZERO(&excfds);
614                         select_timeout.tv_sec = 0;
615                         select_timeout.tv_usec = 50000;
616                         select(max_fd, &readfds, &writefds,
617                                &excfds, &select_timeout);
618                 }
619         }
620 #else
621         while (slot->in_use) {
622                 slot->curl_result = curl_easy_perform(slot->curl);
623                 finish_active_slot(slot);
624         }
625 #endif
626 }
627
628 static void closedown_active_slot(struct active_request_slot *slot)
629 {
630         active_requests--;
631         slot->in_use = 0;
632 }
633
634 void release_active_slot(struct active_request_slot *slot)
635 {
636         closedown_active_slot(slot);
637         if (slot->curl) {
638 #ifdef USE_CURL_MULTI
639                 curl_multi_remove_handle(curlm, slot->curl);
640 #endif
641                 curl_easy_cleanup(slot->curl);
642                 slot->curl = NULL;
643         }
644 #ifdef USE_CURL_MULTI
645         fill_active_slots();
646 #endif
647 }
648
649 void finish_active_slot(struct active_request_slot *slot)
650 {
651         closedown_active_slot(slot);
652         curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
653
654         if (slot->finished != NULL)
655                 (*slot->finished) = 1;
656
657         /* Store slot results so they can be read after the slot is reused */
658         if (slot->results != NULL) {
659                 slot->results->curl_result = slot->curl_result;
660                 slot->results->http_code = slot->http_code;
661         }
662
663         /* Run callback if appropriate */
664         if (slot->callback_func != NULL)
665                 slot->callback_func(slot->callback_data);
666 }
667
668 void finish_all_active_slots(void)
669 {
670         struct active_request_slot *slot = active_queue_head;
671
672         while (slot != NULL)
673                 if (slot->in_use) {
674                         run_active_slot(slot);
675                         slot = active_queue_head;
676                 } else {
677                         slot = slot->next;
678                 }
679 }
680
681 /* Helpers for modifying and creating URLs */
682 static inline int needs_quote(int ch)
683 {
684         if (((ch >= 'A') && (ch <= 'Z'))
685                         || ((ch >= 'a') && (ch <= 'z'))
686                         || ((ch >= '0') && (ch <= '9'))
687                         || (ch == '/')
688                         || (ch == '-')
689                         || (ch == '.'))
690                 return 0;
691         return 1;
692 }
693
694 static inline int hex(int v)
695 {
696         if (v < 10)
697                 return '0' + v;
698         else
699                 return 'A' + v - 10;
700 }
701
702 static void end_url_with_slash(struct strbuf *buf, const char *url)
703 {
704         strbuf_addstr(buf, url);
705         if (buf->len && buf->buf[buf->len - 1] != '/')
706                 strbuf_addstr(buf, "/");
707 }
708
709 static char *quote_ref_url(const char *base, const char *ref)
710 {
711         struct strbuf buf = STRBUF_INIT;
712         const char *cp;
713         int ch;
714
715         end_url_with_slash(&buf, base);
716
717         for (cp = ref; (ch = *cp) != 0; cp++)
718                 if (needs_quote(ch))
719                         strbuf_addf(&buf, "%%%02x", ch);
720                 else
721                         strbuf_addch(&buf, *cp);
722
723         return strbuf_detach(&buf, NULL);
724 }
725
726 void append_remote_object_url(struct strbuf *buf, const char *url,
727                               const char *hex,
728                               int only_two_digit_prefix)
729 {
730         end_url_with_slash(buf, url);
731
732         strbuf_addf(buf, "objects/%.*s/", 2, hex);
733         if (!only_two_digit_prefix)
734                 strbuf_addf(buf, "%s", hex+2);
735 }
736
737 char *get_remote_object_url(const char *url, const char *hex,
738                             int only_two_digit_prefix)
739 {
740         struct strbuf buf = STRBUF_INIT;
741         append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
742         return strbuf_detach(&buf, NULL);
743 }
744
745 /* http_request() targets */
746 #define HTTP_REQUEST_STRBUF     0
747 #define HTTP_REQUEST_FILE       1
748
749 static int http_request(const char *url, void *result, int target, int options)
750 {
751         struct active_request_slot *slot;
752         struct slot_results results;
753         struct curl_slist *headers = NULL;
754         struct strbuf buf = STRBUF_INIT;
755         int ret;
756
757         slot = get_active_slot();
758         slot->results = &results;
759         curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
760
761         if (result == NULL) {
762                 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
763         } else {
764                 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
765                 curl_easy_setopt(slot->curl, CURLOPT_FILE, result);
766
767                 if (target == HTTP_REQUEST_FILE) {
768                         long posn = ftell(result);
769                         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
770                                          fwrite);
771                         if (posn > 0) {
772                                 strbuf_addf(&buf, "Range: bytes=%ld-", posn);
773                                 headers = curl_slist_append(headers, buf.buf);
774                                 strbuf_reset(&buf);
775                         }
776                         slot->local = result;
777                 } else
778                         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
779                                          fwrite_buffer);
780         }
781
782         strbuf_addstr(&buf, "Pragma:");
783         if (options & HTTP_NO_CACHE)
784                 strbuf_addstr(&buf, " no-cache");
785
786         headers = curl_slist_append(headers, buf.buf);
787
788         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
789         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
790
791         if (start_active_slot(slot)) {
792                 run_active_slot(slot);
793                 if (results.curl_result == CURLE_OK)
794                         ret = HTTP_OK;
795                 else if (missing_target(&results))
796                         ret = HTTP_MISSING_TARGET;
797                 else
798                         ret = HTTP_ERROR;
799         } else {
800                 error("Unable to start HTTP request for %s", url);
801                 ret = HTTP_START_FAILED;
802         }
803
804         slot->local = NULL;
805         curl_slist_free_all(headers);
806         strbuf_release(&buf);
807
808         return ret;
809 }
810
811 int http_get_strbuf(const char *url, struct strbuf *result, int options)
812 {
813         return http_request(url, result, HTTP_REQUEST_STRBUF, options);
814 }
815
816 int http_get_file(const char *url, const char *filename, int options)
817 {
818         int ret;
819         struct strbuf tmpfile = STRBUF_INIT;
820         FILE *result;
821
822         strbuf_addf(&tmpfile, "%s.temp", filename);
823         result = fopen(tmpfile.buf, "a");
824         if (! result) {
825                 error("Unable to open local file %s", tmpfile.buf);
826                 ret = HTTP_ERROR;
827                 goto cleanup;
828         }
829
830         ret = http_request(url, result, HTTP_REQUEST_FILE, options);
831         fclose(result);
832
833         if ((ret == HTTP_OK) && move_temp_to_file(tmpfile.buf, filename))
834                 ret = HTTP_ERROR;
835 cleanup:
836         strbuf_release(&tmpfile);
837         return ret;
838 }
839
840 int http_error(const char *url, int ret)
841 {
842         /* http_request has already handled HTTP_START_FAILED. */
843         if (ret != HTTP_START_FAILED)
844                 error("%s while accessing %s\n", curl_errorstr, url);
845
846         return ret;
847 }
848
849 int http_fetch_ref(const char *base, struct ref *ref)
850 {
851         char *url;
852         struct strbuf buffer = STRBUF_INIT;
853         int ret = -1;
854
855         url = quote_ref_url(base, ref->name);
856         if (http_get_strbuf(url, &buffer, HTTP_NO_CACHE) == HTTP_OK) {
857                 strbuf_rtrim(&buffer);
858                 if (buffer.len == 40)
859                         ret = get_sha1_hex(buffer.buf, ref->old_sha1);
860                 else if (!prefixcmp(buffer.buf, "ref: ")) {
861                         ref->symref = xstrdup(buffer.buf + 5);
862                         ret = 0;
863                 }
864         }
865
866         strbuf_release(&buffer);
867         free(url);
868         return ret;
869 }
870
871 /* Helpers for fetching packs */
872 static int fetch_pack_index(unsigned char *sha1, const char *base_url)
873 {
874         int ret = 0;
875         char *hex = xstrdup(sha1_to_hex(sha1));
876         char *filename;
877         char *url = NULL;
878         struct strbuf buf = STRBUF_INIT;
879
880         if (has_pack_index(sha1)) {
881                 ret = 0;
882                 goto cleanup;
883         }
884
885         if (http_is_verbose)
886                 fprintf(stderr, "Getting index for pack %s\n", hex);
887
888         end_url_with_slash(&buf, base_url);
889         strbuf_addf(&buf, "objects/pack/pack-%s.idx", hex);
890         url = strbuf_detach(&buf, NULL);
891
892         filename = sha1_pack_index_name(sha1);
893         if (http_get_file(url, filename, 0) != HTTP_OK)
894                 ret = error("Unable to get pack index %s\n", url);
895
896 cleanup:
897         free(hex);
898         free(url);
899         return ret;
900 }
901
902 static int fetch_and_setup_pack_index(struct packed_git **packs_head,
903         unsigned char *sha1, const char *base_url)
904 {
905         struct packed_git *new_pack;
906
907         if (fetch_pack_index(sha1, base_url))
908                 return -1;
909
910         new_pack = parse_pack_index(sha1);
911         if (!new_pack)
912                 return -1; /* parse_pack_index() already issued error message */
913         new_pack->next = *packs_head;
914         *packs_head = new_pack;
915         return 0;
916 }
917
918 int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
919 {
920         int ret = 0, i = 0;
921         char *url, *data;
922         struct strbuf buf = STRBUF_INIT;
923         unsigned char sha1[20];
924
925         end_url_with_slash(&buf, base_url);
926         strbuf_addstr(&buf, "objects/info/packs");
927         url = strbuf_detach(&buf, NULL);
928
929         ret = http_get_strbuf(url, &buf, HTTP_NO_CACHE);
930         if (ret != HTTP_OK)
931                 goto cleanup;
932
933         data = buf.buf;
934         while (i < buf.len) {
935                 switch (data[i]) {
936                 case 'P':
937                         i++;
938                         if (i + 52 <= buf.len &&
939                             !prefixcmp(data + i, " pack-") &&
940                             !prefixcmp(data + i + 46, ".pack\n")) {
941                                 get_sha1_hex(data + i + 6, sha1);
942                                 fetch_and_setup_pack_index(packs_head, sha1,
943                                                       base_url);
944                                 i += 51;
945                                 break;
946                         }
947                 default:
948                         while (i < buf.len && data[i] != '\n')
949                                 i++;
950                 }
951                 i++;
952         }
953
954 cleanup:
955         free(url);
956         return ret;
957 }
958
959 void release_http_pack_request(struct http_pack_request *preq)
960 {
961         if (preq->packfile != NULL) {
962                 fclose(preq->packfile);
963                 preq->packfile = NULL;
964                 preq->slot->local = NULL;
965         }
966         if (preq->range_header != NULL) {
967                 curl_slist_free_all(preq->range_header);
968                 preq->range_header = NULL;
969         }
970         preq->slot = NULL;
971         free(preq->url);
972 }
973
974 int finish_http_pack_request(struct http_pack_request *preq)
975 {
976         int ret;
977         struct packed_git **lst;
978
979         preq->target->pack_size = ftell(preq->packfile);
980
981         if (preq->packfile != NULL) {
982                 fclose(preq->packfile);
983                 preq->packfile = NULL;
984                 preq->slot->local = NULL;
985         }
986
987         ret = move_temp_to_file(preq->tmpfile, preq->filename);
988         if (ret)
989                 return ret;
990
991         lst = preq->lst;
992         while (*lst != preq->target)
993                 lst = &((*lst)->next);
994         *lst = (*lst)->next;
995
996         if (verify_pack(preq->target))
997                 return -1;
998         install_packed_git(preq->target);
999
1000         return 0;
1001 }
1002
1003 struct http_pack_request *new_http_pack_request(
1004         struct packed_git *target, const char *base_url)
1005 {
1006         char *filename;
1007         long prev_posn = 0;
1008         char range[RANGE_HEADER_SIZE];
1009         struct strbuf buf = STRBUF_INIT;
1010         struct http_pack_request *preq;
1011
1012         preq = xmalloc(sizeof(*preq));
1013         preq->target = target;
1014         preq->range_header = NULL;
1015
1016         end_url_with_slash(&buf, base_url);
1017         strbuf_addf(&buf, "objects/pack/pack-%s.pack",
1018                 sha1_to_hex(target->sha1));
1019         preq->url = strbuf_detach(&buf, NULL);
1020
1021         filename = sha1_pack_name(target->sha1);
1022         snprintf(preq->filename, sizeof(preq->filename), "%s", filename);
1023         snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp", filename);
1024         preq->packfile = fopen(preq->tmpfile, "a");
1025         if (!preq->packfile) {
1026                 error("Unable to open local file %s for pack",
1027                       preq->tmpfile);
1028                 goto abort;
1029         }
1030
1031         preq->slot = get_active_slot();
1032         preq->slot->local = preq->packfile;
1033         curl_easy_setopt(preq->slot->curl, CURLOPT_FILE, preq->packfile);
1034         curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
1035         curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
1036         curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
1037                 no_pragma_header);
1038
1039         /*
1040          * If there is data present from a previous transfer attempt,
1041          * resume where it left off
1042          */
1043         prev_posn = ftell(preq->packfile);
1044         if (prev_posn>0) {
1045                 if (http_is_verbose)
1046                         fprintf(stderr,
1047                                 "Resuming fetch of pack %s at byte %ld\n",
1048                                 sha1_to_hex(target->sha1), prev_posn);
1049                 sprintf(range, "Range: bytes=%ld-", prev_posn);
1050                 preq->range_header = curl_slist_append(NULL, range);
1051                 curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
1052                         preq->range_header);
1053         }
1054
1055         return preq;
1056
1057 abort:
1058         free(filename);
1059         free(preq->url);
1060         free(preq);
1061         return NULL;
1062 }
1063
1064 /* Helpers for fetching objects (loose) */
1065 static size_t fwrite_sha1_file(void *ptr, size_t eltsize, size_t nmemb,
1066                                void *data)
1067 {
1068         unsigned char expn[4096];
1069         size_t size = eltsize * nmemb;
1070         int posn = 0;
1071         struct http_object_request *freq =
1072                 (struct http_object_request *)data;
1073         do {
1074                 ssize_t retval = xwrite(freq->localfile,
1075                                         (char *) ptr + posn, size - posn);
1076                 if (retval < 0)
1077                         return posn;
1078                 posn += retval;
1079         } while (posn < size);
1080
1081         freq->stream.avail_in = size;
1082         freq->stream.next_in = ptr;
1083         do {
1084                 freq->stream.next_out = expn;
1085                 freq->stream.avail_out = sizeof(expn);
1086                 freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
1087                 git_SHA1_Update(&freq->c, expn,
1088                                 sizeof(expn) - freq->stream.avail_out);
1089         } while (freq->stream.avail_in && freq->zret == Z_OK);
1090         data_received++;
1091         return size;
1092 }
1093
1094 struct http_object_request *new_http_object_request(const char *base_url,
1095         unsigned char *sha1)
1096 {
1097         char *hex = sha1_to_hex(sha1);
1098         char *filename;
1099         char prevfile[PATH_MAX];
1100         int prevlocal;
1101         unsigned char prev_buf[PREV_BUF_SIZE];
1102         ssize_t prev_read = 0;
1103         long prev_posn = 0;
1104         char range[RANGE_HEADER_SIZE];
1105         struct curl_slist *range_header = NULL;
1106         struct http_object_request *freq;
1107
1108         freq = xmalloc(sizeof(*freq));
1109         hashcpy(freq->sha1, sha1);
1110         freq->localfile = -1;
1111
1112         filename = sha1_file_name(sha1);
1113         snprintf(freq->filename, sizeof(freq->filename), "%s", filename);
1114         snprintf(freq->tmpfile, sizeof(freq->tmpfile),
1115                  "%s.temp", filename);
1116
1117         snprintf(prevfile, sizeof(prevfile), "%s.prev", filename);
1118         unlink_or_warn(prevfile);
1119         rename(freq->tmpfile, prevfile);
1120         unlink_or_warn(freq->tmpfile);
1121
1122         if (freq->localfile != -1)
1123                 error("fd leakage in start: %d", freq->localfile);
1124         freq->localfile = open(freq->tmpfile,
1125                                O_WRONLY | O_CREAT | O_EXCL, 0666);
1126         /*
1127          * This could have failed due to the "lazy directory creation";
1128          * try to mkdir the last path component.
1129          */
1130         if (freq->localfile < 0 && errno == ENOENT) {
1131                 char *dir = strrchr(freq->tmpfile, '/');
1132                 if (dir) {
1133                         *dir = 0;
1134                         mkdir(freq->tmpfile, 0777);
1135                         *dir = '/';
1136                 }
1137                 freq->localfile = open(freq->tmpfile,
1138                                        O_WRONLY | O_CREAT | O_EXCL, 0666);
1139         }
1140
1141         if (freq->localfile < 0) {
1142                 error("Couldn't create temporary file %s for %s: %s",
1143                       freq->tmpfile, freq->filename, strerror(errno));
1144                 goto abort;
1145         }
1146
1147         memset(&freq->stream, 0, sizeof(freq->stream));
1148
1149         git_inflate_init(&freq->stream);
1150
1151         git_SHA1_Init(&freq->c);
1152
1153         freq->url = get_remote_object_url(base_url, hex, 0);
1154
1155         /*
1156          * If a previous temp file is present, process what was already
1157          * fetched.
1158          */
1159         prevlocal = open(prevfile, O_RDONLY);
1160         if (prevlocal != -1) {
1161                 do {
1162                         prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
1163                         if (prev_read>0) {
1164                                 if (fwrite_sha1_file(prev_buf,
1165                                                      1,
1166                                                      prev_read,
1167                                                      freq) == prev_read) {
1168                                         prev_posn += prev_read;
1169                                 } else {
1170                                         prev_read = -1;
1171                                 }
1172                         }
1173                 } while (prev_read > 0);
1174                 close(prevlocal);
1175         }
1176         unlink_or_warn(prevfile);
1177
1178         /*
1179          * Reset inflate/SHA1 if there was an error reading the previous temp
1180          * file; also rewind to the beginning of the local file.
1181          */
1182         if (prev_read == -1) {
1183                 memset(&freq->stream, 0, sizeof(freq->stream));
1184                 git_inflate_init(&freq->stream);
1185                 git_SHA1_Init(&freq->c);
1186                 if (prev_posn>0) {
1187                         prev_posn = 0;
1188                         lseek(freq->localfile, 0, SEEK_SET);
1189                         if (ftruncate(freq->localfile, 0) < 0) {
1190                                 error("Couldn't truncate temporary file %s for %s: %s",
1191                                           freq->tmpfile, freq->filename, strerror(errno));
1192                                 goto abort;
1193                         }
1194                 }
1195         }
1196
1197         freq->slot = get_active_slot();
1198
1199         curl_easy_setopt(freq->slot->curl, CURLOPT_FILE, freq);
1200         curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
1201         curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
1202         curl_easy_setopt(freq->slot->curl, CURLOPT_URL, freq->url);
1203         curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1204
1205         /*
1206          * If we have successfully processed data from a previous fetch
1207          * attempt, only fetch the data we don't already have.
1208          */
1209         if (prev_posn>0) {
1210                 if (http_is_verbose)
1211                         fprintf(stderr,
1212                                 "Resuming fetch of object %s at byte %ld\n",
1213                                 hex, prev_posn);
1214                 sprintf(range, "Range: bytes=%ld-", prev_posn);
1215                 range_header = curl_slist_append(range_header, range);
1216                 curl_easy_setopt(freq->slot->curl,
1217                                  CURLOPT_HTTPHEADER, range_header);
1218         }
1219
1220         return freq;
1221
1222 abort:
1223         free(filename);
1224         free(freq->url);
1225         free(freq);
1226         return NULL;
1227 }
1228
1229 void process_http_object_request(struct http_object_request *freq)
1230 {
1231         if (freq->slot == NULL)
1232                 return;
1233         freq->curl_result = freq->slot->curl_result;
1234         freq->http_code = freq->slot->http_code;
1235         freq->slot = NULL;
1236 }
1237
1238 int finish_http_object_request(struct http_object_request *freq)
1239 {
1240         struct stat st;
1241
1242         close(freq->localfile);
1243         freq->localfile = -1;
1244
1245         process_http_object_request(freq);
1246
1247         if (freq->http_code == 416) {
1248                 fprintf(stderr, "Warning: requested range invalid; we may already have all the data.\n");
1249         } else if (freq->curl_result != CURLE_OK) {
1250                 if (stat(freq->tmpfile, &st) == 0)
1251                         if (st.st_size == 0)
1252                                 unlink_or_warn(freq->tmpfile);
1253                 return -1;
1254         }
1255
1256         git_inflate_end(&freq->stream);
1257         git_SHA1_Final(freq->real_sha1, &freq->c);
1258         if (freq->zret != Z_STREAM_END) {
1259                 unlink_or_warn(freq->tmpfile);
1260                 return -1;
1261         }
1262         if (hashcmp(freq->sha1, freq->real_sha1)) {
1263                 unlink_or_warn(freq->tmpfile);
1264                 return -1;
1265         }
1266         freq->rename =
1267                 move_temp_to_file(freq->tmpfile, freq->filename);
1268
1269         return freq->rename;
1270 }
1271
1272 void abort_http_object_request(struct http_object_request *freq)
1273 {
1274         unlink_or_warn(freq->tmpfile);
1275
1276         release_http_object_request(freq);
1277 }
1278
1279 void release_http_object_request(struct http_object_request *freq)
1280 {
1281         if (freq->localfile != -1) {
1282                 close(freq->localfile);
1283                 freq->localfile = -1;
1284         }
1285         if (freq->url != NULL) {
1286                 free(freq->url);
1287                 freq->url = NULL;
1288         }
1289         if (freq->slot != NULL) {
1290                 freq->slot->callback_func = NULL;
1291                 freq->slot->callback_data = NULL;
1292                 release_active_slot(freq->slot);
1293                 freq->slot = NULL;
1294         }
1295 }