Merge branch 'jl/maint-submodule-recurse-fetch' into maint
[git] / http.c
1 #include "http.h"
2 #include "pack.h"
3 #include "sideband.h"
4 #include "run-command.h"
5 #include "url.h"
6 #include "credential.h"
7
8 int active_requests;
9 int http_is_verbose;
10 size_t http_post_buffer = 16 * LARGE_PACKET_MAX;
11
12 #if LIBCURL_VERSION_NUM >= 0x070a06
13 #define LIBCURL_CAN_HANDLE_AUTH_ANY
14 #endif
15
16 static int min_curl_sessions = 1;
17 static int curl_session_count;
18 #ifdef USE_CURL_MULTI
19 static int max_requests = -1;
20 static CURLM *curlm;
21 #endif
22 #ifndef NO_CURL_EASY_DUPHANDLE
23 static CURL *curl_default;
24 #endif
25
26 #define PREV_BUF_SIZE 4096
27 #define RANGE_HEADER_SIZE 30
28
29 char curl_errorstr[CURL_ERROR_SIZE];
30
31 static int curl_ssl_verify = -1;
32 static const char *ssl_cert;
33 #if LIBCURL_VERSION_NUM >= 0x070903
34 static const char *ssl_key;
35 #endif
36 #if LIBCURL_VERSION_NUM >= 0x070908
37 static const char *ssl_capath;
38 #endif
39 static const char *ssl_cainfo;
40 static long curl_low_speed_limit = -1;
41 static long curl_low_speed_time = -1;
42 static int curl_ftp_no_epsv;
43 static const char *curl_http_proxy;
44 static const char *curl_cookie_file;
45 static struct credential http_auth = CREDENTIAL_INIT;
46 static int http_proactive_auth;
47 static const char *user_agent;
48
49 #if LIBCURL_VERSION_NUM >= 0x071700
50 /* Use CURLOPT_KEYPASSWD as is */
51 #elif LIBCURL_VERSION_NUM >= 0x070903
52 #define CURLOPT_KEYPASSWD CURLOPT_SSLKEYPASSWD
53 #else
54 #define CURLOPT_KEYPASSWD CURLOPT_SSLCERTPASSWD
55 #endif
56
57 static struct credential cert_auth = CREDENTIAL_INIT;
58 static int ssl_cert_password_required;
59
60 static struct curl_slist *pragma_header;
61 static struct curl_slist *no_pragma_header;
62
63 static struct active_request_slot *active_queue_head;
64
65 size_t fread_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
66 {
67         size_t size = eltsize * nmemb;
68         struct buffer *buffer = buffer_;
69
70         if (size > buffer->buf.len - buffer->posn)
71                 size = buffer->buf.len - buffer->posn;
72         memcpy(ptr, buffer->buf.buf + buffer->posn, size);
73         buffer->posn += size;
74
75         return size;
76 }
77
78 #ifndef NO_CURL_IOCTL
79 curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp)
80 {
81         struct buffer *buffer = clientp;
82
83         switch (cmd) {
84         case CURLIOCMD_NOP:
85                 return CURLIOE_OK;
86
87         case CURLIOCMD_RESTARTREAD:
88                 buffer->posn = 0;
89                 return CURLIOE_OK;
90
91         default:
92                 return CURLIOE_UNKNOWNCMD;
93         }
94 }
95 #endif
96
97 size_t fwrite_buffer(char *ptr, size_t eltsize, size_t nmemb, void *buffer_)
98 {
99         size_t size = eltsize * nmemb;
100         struct strbuf *buffer = buffer_;
101
102         strbuf_add(buffer, ptr, size);
103         return size;
104 }
105
106 size_t fwrite_null(char *ptr, size_t eltsize, size_t nmemb, void *strbuf)
107 {
108         return eltsize * nmemb;
109 }
110
111 #ifdef USE_CURL_MULTI
112 static void process_curl_messages(void)
113 {
114         int num_messages;
115         struct active_request_slot *slot;
116         CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
117
118         while (curl_message != NULL) {
119                 if (curl_message->msg == CURLMSG_DONE) {
120                         int curl_result = curl_message->data.result;
121                         slot = active_queue_head;
122                         while (slot != NULL &&
123                                slot->curl != curl_message->easy_handle)
124                                 slot = slot->next;
125                         if (slot != NULL) {
126                                 curl_multi_remove_handle(curlm, slot->curl);
127                                 slot->curl_result = curl_result;
128                                 finish_active_slot(slot);
129                         } else {
130                                 fprintf(stderr, "Received DONE message for unknown request!\n");
131                         }
132                 } else {
133                         fprintf(stderr, "Unknown CURL message received: %d\n",
134                                 (int)curl_message->msg);
135                 }
136                 curl_message = curl_multi_info_read(curlm, &num_messages);
137         }
138 }
139 #endif
140
141 static int http_options(const char *var, const char *value, void *cb)
142 {
143         if (!strcmp("http.sslverify", var)) {
144                 curl_ssl_verify = git_config_bool(var, value);
145                 return 0;
146         }
147         if (!strcmp("http.sslcert", var))
148                 return git_config_string(&ssl_cert, var, value);
149 #if LIBCURL_VERSION_NUM >= 0x070903
150         if (!strcmp("http.sslkey", var))
151                 return git_config_string(&ssl_key, var, value);
152 #endif
153 #if LIBCURL_VERSION_NUM >= 0x070908
154         if (!strcmp("http.sslcapath", var))
155                 return git_config_string(&ssl_capath, var, value);
156 #endif
157         if (!strcmp("http.sslcainfo", var))
158                 return git_config_string(&ssl_cainfo, var, value);
159         if (!strcmp("http.sslcertpasswordprotected", var)) {
160                 if (git_config_bool(var, value))
161                         ssl_cert_password_required = 1;
162                 return 0;
163         }
164         if (!strcmp("http.minsessions", var)) {
165                 min_curl_sessions = git_config_int(var, value);
166 #ifndef USE_CURL_MULTI
167                 if (min_curl_sessions > 1)
168                         min_curl_sessions = 1;
169 #endif
170                 return 0;
171         }
172 #ifdef USE_CURL_MULTI
173         if (!strcmp("http.maxrequests", var)) {
174                 max_requests = git_config_int(var, value);
175                 return 0;
176         }
177 #endif
178         if (!strcmp("http.lowspeedlimit", var)) {
179                 curl_low_speed_limit = (long)git_config_int(var, value);
180                 return 0;
181         }
182         if (!strcmp("http.lowspeedtime", var)) {
183                 curl_low_speed_time = (long)git_config_int(var, value);
184                 return 0;
185         }
186
187         if (!strcmp("http.noepsv", var)) {
188                 curl_ftp_no_epsv = git_config_bool(var, value);
189                 return 0;
190         }
191         if (!strcmp("http.proxy", var))
192                 return git_config_string(&curl_http_proxy, var, value);
193
194         if (!strcmp("http.cookiefile", var))
195                 return git_config_string(&curl_cookie_file, var, value);
196
197         if (!strcmp("http.postbuffer", var)) {
198                 http_post_buffer = git_config_int(var, value);
199                 if (http_post_buffer < LARGE_PACKET_MAX)
200                         http_post_buffer = LARGE_PACKET_MAX;
201                 return 0;
202         }
203
204         if (!strcmp("http.useragent", var))
205                 return git_config_string(&user_agent, var, value);
206
207         /* Fall back on the default ones */
208         return git_default_config(var, value, cb);
209 }
210
211 static void init_curl_http_auth(CURL *result)
212 {
213         if (http_auth.username) {
214                 struct strbuf up = STRBUF_INIT;
215                 credential_fill(&http_auth);
216                 strbuf_addf(&up, "%s:%s",
217                             http_auth.username, http_auth.password);
218                 curl_easy_setopt(result, CURLOPT_USERPWD,
219                                  strbuf_detach(&up, NULL));
220         }
221 }
222
223 static int has_cert_password(void)
224 {
225         if (ssl_cert == NULL || ssl_cert_password_required != 1)
226                 return 0;
227         if (!cert_auth.password) {
228                 cert_auth.protocol = xstrdup("cert");
229                 cert_auth.path = xstrdup(ssl_cert);
230                 credential_fill(&cert_auth);
231         }
232         return 1;
233 }
234
235 static CURL *get_curl_handle(void)
236 {
237         CURL *result = curl_easy_init();
238
239         if (!curl_ssl_verify) {
240                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
241                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
242         } else {
243                 /* Verify authenticity of the peer's certificate */
244                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1);
245                 /* The name in the cert must match whom we tried to connect */
246                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2);
247         }
248
249 #if LIBCURL_VERSION_NUM >= 0x070907
250         curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
251 #endif
252 #ifdef LIBCURL_CAN_HANDLE_AUTH_ANY
253         curl_easy_setopt(result, CURLOPT_HTTPAUTH, CURLAUTH_ANY);
254 #endif
255
256         if (http_proactive_auth)
257                 init_curl_http_auth(result);
258
259         if (ssl_cert != NULL)
260                 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
261         if (has_cert_password())
262                 curl_easy_setopt(result, CURLOPT_KEYPASSWD, cert_auth.password);
263 #if LIBCURL_VERSION_NUM >= 0x070903
264         if (ssl_key != NULL)
265                 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
266 #endif
267 #if LIBCURL_VERSION_NUM >= 0x070908
268         if (ssl_capath != NULL)
269                 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
270 #endif
271         if (ssl_cainfo != NULL)
272                 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
273         curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
274
275         if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
276                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
277                                  curl_low_speed_limit);
278                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
279                                  curl_low_speed_time);
280         }
281
282         curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
283 #if LIBCURL_VERSION_NUM >= 0x071301
284         curl_easy_setopt(result, CURLOPT_POSTREDIR, CURL_REDIR_POST_ALL);
285 #elif LIBCURL_VERSION_NUM >= 0x071101
286         curl_easy_setopt(result, CURLOPT_POST301, 1);
287 #endif
288
289         if (getenv("GIT_CURL_VERBOSE"))
290                 curl_easy_setopt(result, CURLOPT_VERBOSE, 1);
291
292         curl_easy_setopt(result, CURLOPT_USERAGENT,
293                 user_agent ? user_agent : GIT_HTTP_USER_AGENT);
294
295         if (curl_ftp_no_epsv)
296                 curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0);
297
298         if (curl_http_proxy) {
299                 curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy);
300                 curl_easy_setopt(result, CURLOPT_PROXYAUTH, CURLAUTH_ANY);
301         }
302
303         return result;
304 }
305
306 static void set_from_env(const char **var, const char *envname)
307 {
308         const char *val = getenv(envname);
309         if (val)
310                 *var = val;
311 }
312
313 void http_init(struct remote *remote, const char *url, int proactive_auth)
314 {
315         char *low_speed_limit;
316         char *low_speed_time;
317
318         http_is_verbose = 0;
319
320         git_config(http_options, NULL);
321
322         curl_global_init(CURL_GLOBAL_ALL);
323
324         http_proactive_auth = proactive_auth;
325
326         if (remote && remote->http_proxy)
327                 curl_http_proxy = xstrdup(remote->http_proxy);
328
329         pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
330         no_pragma_header = curl_slist_append(no_pragma_header, "Pragma:");
331
332 #ifdef USE_CURL_MULTI
333         {
334                 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
335                 if (http_max_requests != NULL)
336                         max_requests = atoi(http_max_requests);
337         }
338
339         curlm = curl_multi_init();
340         if (curlm == NULL) {
341                 fprintf(stderr, "Error creating curl multi handle.\n");
342                 exit(1);
343         }
344 #endif
345
346         if (getenv("GIT_SSL_NO_VERIFY"))
347                 curl_ssl_verify = 0;
348
349         set_from_env(&ssl_cert, "GIT_SSL_CERT");
350 #if LIBCURL_VERSION_NUM >= 0x070903
351         set_from_env(&ssl_key, "GIT_SSL_KEY");
352 #endif
353 #if LIBCURL_VERSION_NUM >= 0x070908
354         set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
355 #endif
356         set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
357
358         set_from_env(&user_agent, "GIT_HTTP_USER_AGENT");
359
360         low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
361         if (low_speed_limit != NULL)
362                 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
363         low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
364         if (low_speed_time != NULL)
365                 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
366
367         if (curl_ssl_verify == -1)
368                 curl_ssl_verify = 1;
369
370         curl_session_count = 0;
371 #ifdef USE_CURL_MULTI
372         if (max_requests < 1)
373                 max_requests = DEFAULT_MAX_REQUESTS;
374 #endif
375
376         if (getenv("GIT_CURL_FTP_NO_EPSV"))
377                 curl_ftp_no_epsv = 1;
378
379         if (url) {
380                 credential_from_url(&http_auth, url);
381                 if (!ssl_cert_password_required &&
382                     getenv("GIT_SSL_CERT_PASSWORD_PROTECTED") &&
383                     !prefixcmp(url, "https://"))
384                         ssl_cert_password_required = 1;
385         }
386
387 #ifndef NO_CURL_EASY_DUPHANDLE
388         curl_default = get_curl_handle();
389 #endif
390 }
391
392 void http_cleanup(void)
393 {
394         struct active_request_slot *slot = active_queue_head;
395
396         while (slot != NULL) {
397                 struct active_request_slot *next = slot->next;
398                 if (slot->curl != NULL) {
399 #ifdef USE_CURL_MULTI
400                         curl_multi_remove_handle(curlm, slot->curl);
401 #endif
402                         curl_easy_cleanup(slot->curl);
403                 }
404                 free(slot);
405                 slot = next;
406         }
407         active_queue_head = NULL;
408
409 #ifndef NO_CURL_EASY_DUPHANDLE
410         curl_easy_cleanup(curl_default);
411 #endif
412
413 #ifdef USE_CURL_MULTI
414         curl_multi_cleanup(curlm);
415 #endif
416         curl_global_cleanup();
417
418         curl_slist_free_all(pragma_header);
419         pragma_header = NULL;
420
421         curl_slist_free_all(no_pragma_header);
422         no_pragma_header = NULL;
423
424         if (curl_http_proxy) {
425                 free((void *)curl_http_proxy);
426                 curl_http_proxy = NULL;
427         }
428
429         if (cert_auth.password != NULL) {
430                 memset(cert_auth.password, 0, strlen(cert_auth.password));
431                 free(cert_auth.password);
432                 cert_auth.password = NULL;
433         }
434         ssl_cert_password_required = 0;
435 }
436
437 struct active_request_slot *get_active_slot(void)
438 {
439         struct active_request_slot *slot = active_queue_head;
440         struct active_request_slot *newslot;
441
442 #ifdef USE_CURL_MULTI
443         int num_transfers;
444
445         /* Wait for a slot to open up if the queue is full */
446         while (active_requests >= max_requests) {
447                 curl_multi_perform(curlm, &num_transfers);
448                 if (num_transfers < active_requests)
449                         process_curl_messages();
450         }
451 #endif
452
453         while (slot != NULL && slot->in_use)
454                 slot = slot->next;
455
456         if (slot == NULL) {
457                 newslot = xmalloc(sizeof(*newslot));
458                 newslot->curl = NULL;
459                 newslot->in_use = 0;
460                 newslot->next = NULL;
461
462                 slot = active_queue_head;
463                 if (slot == NULL) {
464                         active_queue_head = newslot;
465                 } else {
466                         while (slot->next != NULL)
467                                 slot = slot->next;
468                         slot->next = newslot;
469                 }
470                 slot = newslot;
471         }
472
473         if (slot->curl == NULL) {
474 #ifdef NO_CURL_EASY_DUPHANDLE
475                 slot->curl = get_curl_handle();
476 #else
477                 slot->curl = curl_easy_duphandle(curl_default);
478 #endif
479                 curl_session_count++;
480         }
481
482         active_requests++;
483         slot->in_use = 1;
484         slot->results = NULL;
485         slot->finished = NULL;
486         slot->callback_data = NULL;
487         slot->callback_func = NULL;
488         curl_easy_setopt(slot->curl, CURLOPT_COOKIEFILE, curl_cookie_file);
489         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
490         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
491         curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
492         curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
493         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
494         curl_easy_setopt(slot->curl, CURLOPT_POSTFIELDS, NULL);
495         curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
496         curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
497
498         return slot;
499 }
500
501 int start_active_slot(struct active_request_slot *slot)
502 {
503 #ifdef USE_CURL_MULTI
504         CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
505         int num_transfers;
506
507         if (curlm_result != CURLM_OK &&
508             curlm_result != CURLM_CALL_MULTI_PERFORM) {
509                 active_requests--;
510                 slot->in_use = 0;
511                 return 0;
512         }
513
514         /*
515          * We know there must be something to do, since we just added
516          * something.
517          */
518         curl_multi_perform(curlm, &num_transfers);
519 #endif
520         return 1;
521 }
522
523 #ifdef USE_CURL_MULTI
524 struct fill_chain {
525         void *data;
526         int (*fill)(void *);
527         struct fill_chain *next;
528 };
529
530 static struct fill_chain *fill_cfg;
531
532 void add_fill_function(void *data, int (*fill)(void *))
533 {
534         struct fill_chain *new = xmalloc(sizeof(*new));
535         struct fill_chain **linkp = &fill_cfg;
536         new->data = data;
537         new->fill = fill;
538         new->next = NULL;
539         while (*linkp)
540                 linkp = &(*linkp)->next;
541         *linkp = new;
542 }
543
544 void fill_active_slots(void)
545 {
546         struct active_request_slot *slot = active_queue_head;
547
548         while (active_requests < max_requests) {
549                 struct fill_chain *fill;
550                 for (fill = fill_cfg; fill; fill = fill->next)
551                         if (fill->fill(fill->data))
552                                 break;
553
554                 if (!fill)
555                         break;
556         }
557
558         while (slot != NULL) {
559                 if (!slot->in_use && slot->curl != NULL
560                         && curl_session_count > min_curl_sessions) {
561                         curl_easy_cleanup(slot->curl);
562                         slot->curl = NULL;
563                         curl_session_count--;
564                 }
565                 slot = slot->next;
566         }
567 }
568
569 void step_active_slots(void)
570 {
571         int num_transfers;
572         CURLMcode curlm_result;
573
574         do {
575                 curlm_result = curl_multi_perform(curlm, &num_transfers);
576         } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
577         if (num_transfers < active_requests) {
578                 process_curl_messages();
579                 fill_active_slots();
580         }
581 }
582 #endif
583
584 void run_active_slot(struct active_request_slot *slot)
585 {
586 #ifdef USE_CURL_MULTI
587         fd_set readfds;
588         fd_set writefds;
589         fd_set excfds;
590         int max_fd;
591         struct timeval select_timeout;
592         int finished = 0;
593
594         slot->finished = &finished;
595         while (!finished) {
596                 step_active_slots();
597
598                 if (slot->in_use) {
599 #if LIBCURL_VERSION_NUM >= 0x070f04
600                         long curl_timeout;
601                         curl_multi_timeout(curlm, &curl_timeout);
602                         if (curl_timeout == 0) {
603                                 continue;
604                         } else if (curl_timeout == -1) {
605                                 select_timeout.tv_sec  = 0;
606                                 select_timeout.tv_usec = 50000;
607                         } else {
608                                 select_timeout.tv_sec  =  curl_timeout / 1000;
609                                 select_timeout.tv_usec = (curl_timeout % 1000) * 1000;
610                         }
611 #else
612                         select_timeout.tv_sec  = 0;
613                         select_timeout.tv_usec = 50000;
614 #endif
615
616                         max_fd = -1;
617                         FD_ZERO(&readfds);
618                         FD_ZERO(&writefds);
619                         FD_ZERO(&excfds);
620                         curl_multi_fdset(curlm, &readfds, &writefds, &excfds, &max_fd);
621
622                         select(max_fd+1, &readfds, &writefds, &excfds, &select_timeout);
623                 }
624         }
625 #else
626         while (slot->in_use) {
627                 slot->curl_result = curl_easy_perform(slot->curl);
628                 finish_active_slot(slot);
629         }
630 #endif
631 }
632
633 static void closedown_active_slot(struct active_request_slot *slot)
634 {
635         active_requests--;
636         slot->in_use = 0;
637 }
638
639 static void release_active_slot(struct active_request_slot *slot)
640 {
641         closedown_active_slot(slot);
642         if (slot->curl && curl_session_count > min_curl_sessions) {
643 #ifdef USE_CURL_MULTI
644                 curl_multi_remove_handle(curlm, slot->curl);
645 #endif
646                 curl_easy_cleanup(slot->curl);
647                 slot->curl = NULL;
648                 curl_session_count--;
649         }
650 #ifdef USE_CURL_MULTI
651         fill_active_slots();
652 #endif
653 }
654
655 void finish_active_slot(struct active_request_slot *slot)
656 {
657         closedown_active_slot(slot);
658         curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
659
660         if (slot->finished != NULL)
661                 (*slot->finished) = 1;
662
663         /* Store slot results so they can be read after the slot is reused */
664         if (slot->results != NULL) {
665                 slot->results->curl_result = slot->curl_result;
666                 slot->results->http_code = slot->http_code;
667         }
668
669         /* Run callback if appropriate */
670         if (slot->callback_func != NULL)
671                 slot->callback_func(slot->callback_data);
672 }
673
674 void finish_all_active_slots(void)
675 {
676         struct active_request_slot *slot = active_queue_head;
677
678         while (slot != NULL)
679                 if (slot->in_use) {
680                         run_active_slot(slot);
681                         slot = active_queue_head;
682                 } else {
683                         slot = slot->next;
684                 }
685 }
686
687 /* Helpers for modifying and creating URLs */
688 static inline int needs_quote(int ch)
689 {
690         if (((ch >= 'A') && (ch <= 'Z'))
691                         || ((ch >= 'a') && (ch <= 'z'))
692                         || ((ch >= '0') && (ch <= '9'))
693                         || (ch == '/')
694                         || (ch == '-')
695                         || (ch == '.'))
696                 return 0;
697         return 1;
698 }
699
700 static char *quote_ref_url(const char *base, const char *ref)
701 {
702         struct strbuf buf = STRBUF_INIT;
703         const char *cp;
704         int ch;
705
706         end_url_with_slash(&buf, base);
707
708         for (cp = ref; (ch = *cp) != 0; cp++)
709                 if (needs_quote(ch))
710                         strbuf_addf(&buf, "%%%02x", ch);
711                 else
712                         strbuf_addch(&buf, *cp);
713
714         return strbuf_detach(&buf, NULL);
715 }
716
717 void append_remote_object_url(struct strbuf *buf, const char *url,
718                               const char *hex,
719                               int only_two_digit_prefix)
720 {
721         end_url_with_slash(buf, url);
722
723         strbuf_addf(buf, "objects/%.*s/", 2, hex);
724         if (!only_two_digit_prefix)
725                 strbuf_addf(buf, "%s", hex+2);
726 }
727
728 char *get_remote_object_url(const char *url, const char *hex,
729                             int only_two_digit_prefix)
730 {
731         struct strbuf buf = STRBUF_INIT;
732         append_remote_object_url(&buf, url, hex, only_two_digit_prefix);
733         return strbuf_detach(&buf, NULL);
734 }
735
736 /* http_request() targets */
737 #define HTTP_REQUEST_STRBUF     0
738 #define HTTP_REQUEST_FILE       1
739
740 static int http_request(const char *url, void *result, int target, int options)
741 {
742         struct active_request_slot *slot;
743         struct slot_results results;
744         struct curl_slist *headers = NULL;
745         struct strbuf buf = STRBUF_INIT;
746         int ret;
747
748         slot = get_active_slot();
749         slot->results = &results;
750         curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
751
752         if (result == NULL) {
753                 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 1);
754         } else {
755                 curl_easy_setopt(slot->curl, CURLOPT_NOBODY, 0);
756                 curl_easy_setopt(slot->curl, CURLOPT_FILE, result);
757
758                 if (target == HTTP_REQUEST_FILE) {
759                         long posn = ftell(result);
760                         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
761                                          fwrite);
762                         if (posn > 0) {
763                                 strbuf_addf(&buf, "Range: bytes=%ld-", posn);
764                                 headers = curl_slist_append(headers, buf.buf);
765                                 strbuf_reset(&buf);
766                         }
767                 } else
768                         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION,
769                                          fwrite_buffer);
770         }
771
772         strbuf_addstr(&buf, "Pragma:");
773         if (options & HTTP_NO_CACHE)
774                 strbuf_addstr(&buf, " no-cache");
775
776         headers = curl_slist_append(headers, buf.buf);
777
778         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
779         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, headers);
780
781         if (start_active_slot(slot)) {
782                 run_active_slot(slot);
783                 if (results.curl_result == CURLE_OK)
784                         ret = HTTP_OK;
785                 else if (missing_target(&results))
786                         ret = HTTP_MISSING_TARGET;
787                 else if (results.http_code == 401) {
788                         if (http_auth.username && http_auth.password) {
789                                 credential_reject(&http_auth);
790                                 ret = HTTP_NOAUTH;
791                         } else {
792                                 credential_fill(&http_auth);
793                                 init_curl_http_auth(slot->curl);
794                                 ret = HTTP_REAUTH;
795                         }
796                 } else {
797                         if (!curl_errorstr[0])
798                                 strlcpy(curl_errorstr,
799                                         curl_easy_strerror(results.curl_result),
800                                         sizeof(curl_errorstr));
801                         ret = HTTP_ERROR;
802                 }
803         } else {
804                 error("Unable to start HTTP request for %s", url);
805                 ret = HTTP_START_FAILED;
806         }
807
808         curl_slist_free_all(headers);
809         strbuf_release(&buf);
810
811         if (ret == HTTP_OK)
812                 credential_approve(&http_auth);
813
814         return ret;
815 }
816
817 static int http_request_reauth(const char *url, void *result, int target,
818                                int options)
819 {
820         int ret = http_request(url, result, target, options);
821         if (ret != HTTP_REAUTH)
822                 return ret;
823         return http_request(url, result, target, options);
824 }
825
826 int http_get_strbuf(const char *url, struct strbuf *result, int options)
827 {
828         return http_request_reauth(url, result, HTTP_REQUEST_STRBUF, options);
829 }
830
831 /*
832  * Downloads a URL and stores the result in the given file.
833  *
834  * If a previous interrupted download is detected (i.e. a previous temporary
835  * file is still around) the download is resumed.
836  */
837 static int http_get_file(const char *url, const char *filename, int options)
838 {
839         int ret;
840         struct strbuf tmpfile = STRBUF_INIT;
841         FILE *result;
842
843         strbuf_addf(&tmpfile, "%s.temp", filename);
844         result = fopen(tmpfile.buf, "a");
845         if (! result) {
846                 error("Unable to open local file %s", tmpfile.buf);
847                 ret = HTTP_ERROR;
848                 goto cleanup;
849         }
850
851         ret = http_request_reauth(url, result, HTTP_REQUEST_FILE, options);
852         fclose(result);
853
854         if ((ret == HTTP_OK) && move_temp_to_file(tmpfile.buf, filename))
855                 ret = HTTP_ERROR;
856 cleanup:
857         strbuf_release(&tmpfile);
858         return ret;
859 }
860
861 int http_error(const char *url, int ret)
862 {
863         /* http_request has already handled HTTP_START_FAILED. */
864         if (ret != HTTP_START_FAILED)
865                 error("%s while accessing %s", curl_errorstr, url);
866
867         return ret;
868 }
869
870 int http_fetch_ref(const char *base, struct ref *ref)
871 {
872         char *url;
873         struct strbuf buffer = STRBUF_INIT;
874         int ret = -1;
875
876         url = quote_ref_url(base, ref->name);
877         if (http_get_strbuf(url, &buffer, HTTP_NO_CACHE) == HTTP_OK) {
878                 strbuf_rtrim(&buffer);
879                 if (buffer.len == 40)
880                         ret = get_sha1_hex(buffer.buf, ref->old_sha1);
881                 else if (!prefixcmp(buffer.buf, "ref: ")) {
882                         ref->symref = xstrdup(buffer.buf + 5);
883                         ret = 0;
884                 }
885         }
886
887         strbuf_release(&buffer);
888         free(url);
889         return ret;
890 }
891
892 /* Helpers for fetching packs */
893 static char *fetch_pack_index(unsigned char *sha1, const char *base_url)
894 {
895         char *url, *tmp;
896         struct strbuf buf = STRBUF_INIT;
897
898         if (http_is_verbose)
899                 fprintf(stderr, "Getting index for pack %s\n", sha1_to_hex(sha1));
900
901         end_url_with_slash(&buf, base_url);
902         strbuf_addf(&buf, "objects/pack/pack-%s.idx", sha1_to_hex(sha1));
903         url = strbuf_detach(&buf, NULL);
904
905         strbuf_addf(&buf, "%s.temp", sha1_pack_index_name(sha1));
906         tmp = strbuf_detach(&buf, NULL);
907
908         if (http_get_file(url, tmp, 0) != HTTP_OK) {
909                 error("Unable to get pack index %s\n", url);
910                 free(tmp);
911                 tmp = NULL;
912         }
913
914         free(url);
915         return tmp;
916 }
917
918 static int fetch_and_setup_pack_index(struct packed_git **packs_head,
919         unsigned char *sha1, const char *base_url)
920 {
921         struct packed_git *new_pack;
922         char *tmp_idx = NULL;
923         int ret;
924
925         if (has_pack_index(sha1)) {
926                 new_pack = parse_pack_index(sha1, NULL);
927                 if (!new_pack)
928                         return -1; /* parse_pack_index() already issued error message */
929                 goto add_pack;
930         }
931
932         tmp_idx = fetch_pack_index(sha1, base_url);
933         if (!tmp_idx)
934                 return -1;
935
936         new_pack = parse_pack_index(sha1, tmp_idx);
937         if (!new_pack) {
938                 unlink(tmp_idx);
939                 free(tmp_idx);
940
941                 return -1; /* parse_pack_index() already issued error message */
942         }
943
944         ret = verify_pack_index(new_pack);
945         if (!ret) {
946                 close_pack_index(new_pack);
947                 ret = move_temp_to_file(tmp_idx, sha1_pack_index_name(sha1));
948         }
949         free(tmp_idx);
950         if (ret)
951                 return -1;
952
953 add_pack:
954         new_pack->next = *packs_head;
955         *packs_head = new_pack;
956         return 0;
957 }
958
959 int http_get_info_packs(const char *base_url, struct packed_git **packs_head)
960 {
961         int ret = 0, i = 0;
962         char *url, *data;
963         struct strbuf buf = STRBUF_INIT;
964         unsigned char sha1[20];
965
966         end_url_with_slash(&buf, base_url);
967         strbuf_addstr(&buf, "objects/info/packs");
968         url = strbuf_detach(&buf, NULL);
969
970         ret = http_get_strbuf(url, &buf, HTTP_NO_CACHE);
971         if (ret != HTTP_OK)
972                 goto cleanup;
973
974         data = buf.buf;
975         while (i < buf.len) {
976                 switch (data[i]) {
977                 case 'P':
978                         i++;
979                         if (i + 52 <= buf.len &&
980                             !prefixcmp(data + i, " pack-") &&
981                             !prefixcmp(data + i + 46, ".pack\n")) {
982                                 get_sha1_hex(data + i + 6, sha1);
983                                 fetch_and_setup_pack_index(packs_head, sha1,
984                                                       base_url);
985                                 i += 51;
986                                 break;
987                         }
988                 default:
989                         while (i < buf.len && data[i] != '\n')
990                                 i++;
991                 }
992                 i++;
993         }
994
995 cleanup:
996         free(url);
997         return ret;
998 }
999
1000 void release_http_pack_request(struct http_pack_request *preq)
1001 {
1002         if (preq->packfile != NULL) {
1003                 fclose(preq->packfile);
1004                 preq->packfile = NULL;
1005         }
1006         if (preq->range_header != NULL) {
1007                 curl_slist_free_all(preq->range_header);
1008                 preq->range_header = NULL;
1009         }
1010         preq->slot = NULL;
1011         free(preq->url);
1012 }
1013
1014 int finish_http_pack_request(struct http_pack_request *preq)
1015 {
1016         struct packed_git **lst;
1017         struct packed_git *p = preq->target;
1018         char *tmp_idx;
1019         struct child_process ip;
1020         const char *ip_argv[8];
1021
1022         close_pack_index(p);
1023
1024         fclose(preq->packfile);
1025         preq->packfile = NULL;
1026
1027         lst = preq->lst;
1028         while (*lst != p)
1029                 lst = &((*lst)->next);
1030         *lst = (*lst)->next;
1031
1032         tmp_idx = xstrdup(preq->tmpfile);
1033         strcpy(tmp_idx + strlen(tmp_idx) - strlen(".pack.temp"),
1034                ".idx.temp");
1035
1036         ip_argv[0] = "index-pack";
1037         ip_argv[1] = "-o";
1038         ip_argv[2] = tmp_idx;
1039         ip_argv[3] = preq->tmpfile;
1040         ip_argv[4] = NULL;
1041
1042         memset(&ip, 0, sizeof(ip));
1043         ip.argv = ip_argv;
1044         ip.git_cmd = 1;
1045         ip.no_stdin = 1;
1046         ip.no_stdout = 1;
1047
1048         if (run_command(&ip)) {
1049                 unlink(preq->tmpfile);
1050                 unlink(tmp_idx);
1051                 free(tmp_idx);
1052                 return -1;
1053         }
1054
1055         unlink(sha1_pack_index_name(p->sha1));
1056
1057         if (move_temp_to_file(preq->tmpfile, sha1_pack_name(p->sha1))
1058          || move_temp_to_file(tmp_idx, sha1_pack_index_name(p->sha1))) {
1059                 free(tmp_idx);
1060                 return -1;
1061         }
1062
1063         install_packed_git(p);
1064         free(tmp_idx);
1065         return 0;
1066 }
1067
1068 struct http_pack_request *new_http_pack_request(
1069         struct packed_git *target, const char *base_url)
1070 {
1071         long prev_posn = 0;
1072         char range[RANGE_HEADER_SIZE];
1073         struct strbuf buf = STRBUF_INIT;
1074         struct http_pack_request *preq;
1075
1076         preq = xcalloc(1, sizeof(*preq));
1077         preq->target = target;
1078
1079         end_url_with_slash(&buf, base_url);
1080         strbuf_addf(&buf, "objects/pack/pack-%s.pack",
1081                 sha1_to_hex(target->sha1));
1082         preq->url = strbuf_detach(&buf, NULL);
1083
1084         snprintf(preq->tmpfile, sizeof(preq->tmpfile), "%s.temp",
1085                 sha1_pack_name(target->sha1));
1086         preq->packfile = fopen(preq->tmpfile, "a");
1087         if (!preq->packfile) {
1088                 error("Unable to open local file %s for pack",
1089                       preq->tmpfile);
1090                 goto abort;
1091         }
1092
1093         preq->slot = get_active_slot();
1094         curl_easy_setopt(preq->slot->curl, CURLOPT_FILE, preq->packfile);
1095         curl_easy_setopt(preq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite);
1096         curl_easy_setopt(preq->slot->curl, CURLOPT_URL, preq->url);
1097         curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
1098                 no_pragma_header);
1099
1100         /*
1101          * If there is data present from a previous transfer attempt,
1102          * resume where it left off
1103          */
1104         prev_posn = ftell(preq->packfile);
1105         if (prev_posn>0) {
1106                 if (http_is_verbose)
1107                         fprintf(stderr,
1108                                 "Resuming fetch of pack %s at byte %ld\n",
1109                                 sha1_to_hex(target->sha1), prev_posn);
1110                 sprintf(range, "Range: bytes=%ld-", prev_posn);
1111                 preq->range_header = curl_slist_append(NULL, range);
1112                 curl_easy_setopt(preq->slot->curl, CURLOPT_HTTPHEADER,
1113                         preq->range_header);
1114         }
1115
1116         return preq;
1117
1118 abort:
1119         free(preq->url);
1120         free(preq);
1121         return NULL;
1122 }
1123
1124 /* Helpers for fetching objects (loose) */
1125 static size_t fwrite_sha1_file(char *ptr, size_t eltsize, size_t nmemb,
1126                                void *data)
1127 {
1128         unsigned char expn[4096];
1129         size_t size = eltsize * nmemb;
1130         int posn = 0;
1131         struct http_object_request *freq =
1132                 (struct http_object_request *)data;
1133         do {
1134                 ssize_t retval = xwrite(freq->localfile,
1135                                         (char *) ptr + posn, size - posn);
1136                 if (retval < 0)
1137                         return posn;
1138                 posn += retval;
1139         } while (posn < size);
1140
1141         freq->stream.avail_in = size;
1142         freq->stream.next_in = (void *)ptr;
1143         do {
1144                 freq->stream.next_out = expn;
1145                 freq->stream.avail_out = sizeof(expn);
1146                 freq->zret = git_inflate(&freq->stream, Z_SYNC_FLUSH);
1147                 git_SHA1_Update(&freq->c, expn,
1148                                 sizeof(expn) - freq->stream.avail_out);
1149         } while (freq->stream.avail_in && freq->zret == Z_OK);
1150         return size;
1151 }
1152
1153 struct http_object_request *new_http_object_request(const char *base_url,
1154         unsigned char *sha1)
1155 {
1156         char *hex = sha1_to_hex(sha1);
1157         char *filename;
1158         char prevfile[PATH_MAX];
1159         int prevlocal;
1160         char prev_buf[PREV_BUF_SIZE];
1161         ssize_t prev_read = 0;
1162         long prev_posn = 0;
1163         char range[RANGE_HEADER_SIZE];
1164         struct curl_slist *range_header = NULL;
1165         struct http_object_request *freq;
1166
1167         freq = xcalloc(1, sizeof(*freq));
1168         hashcpy(freq->sha1, sha1);
1169         freq->localfile = -1;
1170
1171         filename = sha1_file_name(sha1);
1172         snprintf(freq->tmpfile, sizeof(freq->tmpfile),
1173                  "%s.temp", filename);
1174
1175         snprintf(prevfile, sizeof(prevfile), "%s.prev", filename);
1176         unlink_or_warn(prevfile);
1177         rename(freq->tmpfile, prevfile);
1178         unlink_or_warn(freq->tmpfile);
1179
1180         if (freq->localfile != -1)
1181                 error("fd leakage in start: %d", freq->localfile);
1182         freq->localfile = open(freq->tmpfile,
1183                                O_WRONLY | O_CREAT | O_EXCL, 0666);
1184         /*
1185          * This could have failed due to the "lazy directory creation";
1186          * try to mkdir the last path component.
1187          */
1188         if (freq->localfile < 0 && errno == ENOENT) {
1189                 char *dir = strrchr(freq->tmpfile, '/');
1190                 if (dir) {
1191                         *dir = 0;
1192                         mkdir(freq->tmpfile, 0777);
1193                         *dir = '/';
1194                 }
1195                 freq->localfile = open(freq->tmpfile,
1196                                        O_WRONLY | O_CREAT | O_EXCL, 0666);
1197         }
1198
1199         if (freq->localfile < 0) {
1200                 error("Couldn't create temporary file %s: %s",
1201                       freq->tmpfile, strerror(errno));
1202                 goto abort;
1203         }
1204
1205         git_inflate_init(&freq->stream);
1206
1207         git_SHA1_Init(&freq->c);
1208
1209         freq->url = get_remote_object_url(base_url, hex, 0);
1210
1211         /*
1212          * If a previous temp file is present, process what was already
1213          * fetched.
1214          */
1215         prevlocal = open(prevfile, O_RDONLY);
1216         if (prevlocal != -1) {
1217                 do {
1218                         prev_read = xread(prevlocal, prev_buf, PREV_BUF_SIZE);
1219                         if (prev_read>0) {
1220                                 if (fwrite_sha1_file(prev_buf,
1221                                                      1,
1222                                                      prev_read,
1223                                                      freq) == prev_read) {
1224                                         prev_posn += prev_read;
1225                                 } else {
1226                                         prev_read = -1;
1227                                 }
1228                         }
1229                 } while (prev_read > 0);
1230                 close(prevlocal);
1231         }
1232         unlink_or_warn(prevfile);
1233
1234         /*
1235          * Reset inflate/SHA1 if there was an error reading the previous temp
1236          * file; also rewind to the beginning of the local file.
1237          */
1238         if (prev_read == -1) {
1239                 memset(&freq->stream, 0, sizeof(freq->stream));
1240                 git_inflate_init(&freq->stream);
1241                 git_SHA1_Init(&freq->c);
1242                 if (prev_posn>0) {
1243                         prev_posn = 0;
1244                         lseek(freq->localfile, 0, SEEK_SET);
1245                         if (ftruncate(freq->localfile, 0) < 0) {
1246                                 error("Couldn't truncate temporary file %s: %s",
1247                                           freq->tmpfile, strerror(errno));
1248                                 goto abort;
1249                         }
1250                 }
1251         }
1252
1253         freq->slot = get_active_slot();
1254
1255         curl_easy_setopt(freq->slot->curl, CURLOPT_FILE, freq);
1256         curl_easy_setopt(freq->slot->curl, CURLOPT_WRITEFUNCTION, fwrite_sha1_file);
1257         curl_easy_setopt(freq->slot->curl, CURLOPT_ERRORBUFFER, freq->errorstr);
1258         curl_easy_setopt(freq->slot->curl, CURLOPT_URL, freq->url);
1259         curl_easy_setopt(freq->slot->curl, CURLOPT_HTTPHEADER, no_pragma_header);
1260
1261         /*
1262          * If we have successfully processed data from a previous fetch
1263          * attempt, only fetch the data we don't already have.
1264          */
1265         if (prev_posn>0) {
1266                 if (http_is_verbose)
1267                         fprintf(stderr,
1268                                 "Resuming fetch of object %s at byte %ld\n",
1269                                 hex, prev_posn);
1270                 sprintf(range, "Range: bytes=%ld-", prev_posn);
1271                 range_header = curl_slist_append(range_header, range);
1272                 curl_easy_setopt(freq->slot->curl,
1273                                  CURLOPT_HTTPHEADER, range_header);
1274         }
1275
1276         return freq;
1277
1278 abort:
1279         free(freq->url);
1280         free(freq);
1281         return NULL;
1282 }
1283
1284 void process_http_object_request(struct http_object_request *freq)
1285 {
1286         if (freq->slot == NULL)
1287                 return;
1288         freq->curl_result = freq->slot->curl_result;
1289         freq->http_code = freq->slot->http_code;
1290         freq->slot = NULL;
1291 }
1292
1293 int finish_http_object_request(struct http_object_request *freq)
1294 {
1295         struct stat st;
1296
1297         close(freq->localfile);
1298         freq->localfile = -1;
1299
1300         process_http_object_request(freq);
1301
1302         if (freq->http_code == 416) {
1303                 warning("requested range invalid; we may already have all the data.");
1304         } else if (freq->curl_result != CURLE_OK) {
1305                 if (stat(freq->tmpfile, &st) == 0)
1306                         if (st.st_size == 0)
1307                                 unlink_or_warn(freq->tmpfile);
1308                 return -1;
1309         }
1310
1311         git_inflate_end(&freq->stream);
1312         git_SHA1_Final(freq->real_sha1, &freq->c);
1313         if (freq->zret != Z_STREAM_END) {
1314                 unlink_or_warn(freq->tmpfile);
1315                 return -1;
1316         }
1317         if (hashcmp(freq->sha1, freq->real_sha1)) {
1318                 unlink_or_warn(freq->tmpfile);
1319                 return -1;
1320         }
1321         freq->rename =
1322                 move_temp_to_file(freq->tmpfile, sha1_file_name(freq->sha1));
1323
1324         return freq->rename;
1325 }
1326
1327 void abort_http_object_request(struct http_object_request *freq)
1328 {
1329         unlink_or_warn(freq->tmpfile);
1330
1331         release_http_object_request(freq);
1332 }
1333
1334 void release_http_object_request(struct http_object_request *freq)
1335 {
1336         if (freq->localfile != -1) {
1337                 close(freq->localfile);
1338                 freq->localfile = -1;
1339         }
1340         if (freq->url != NULL) {
1341                 free(freq->url);
1342                 freq->url = NULL;
1343         }
1344         if (freq->slot != NULL) {
1345                 freq->slot->callback_func = NULL;
1346                 freq->slot->callback_data = NULL;
1347                 release_active_slot(freq->slot);
1348                 freq->slot = NULL;
1349         }
1350 }