Makefile: test-parse-options depends on parse-options.h
[git] / http.c
1 #include "http.h"
2
3 int data_received;
4 int active_requests;
5
6 #ifdef USE_CURL_MULTI
7 static int max_requests = -1;
8 static CURLM *curlm;
9 #endif
10 #ifndef NO_CURL_EASY_DUPHANDLE
11 static CURL *curl_default;
12 #endif
13 char curl_errorstr[CURL_ERROR_SIZE];
14
15 static int curl_ssl_verify = -1;
16 static const char *ssl_cert;
17 #if LIBCURL_VERSION_NUM >= 0x070902
18 static const char *ssl_key;
19 #endif
20 #if LIBCURL_VERSION_NUM >= 0x070908
21 static const char *ssl_capath;
22 #endif
23 static const char *ssl_cainfo;
24 static long curl_low_speed_limit = -1;
25 static long curl_low_speed_time = -1;
26 static int curl_ftp_no_epsv;
27 static const char *curl_http_proxy;
28 static char *user_name, *user_pass;
29
30 static struct curl_slist *pragma_header;
31
32 static struct active_request_slot *active_queue_head;
33
34 size_t fread_buffer(void *ptr, size_t eltsize, size_t nmemb, void *buffer_)
35 {
36         size_t size = eltsize * nmemb;
37         struct buffer *buffer = buffer_;
38
39         if (size > buffer->buf.len - buffer->posn)
40                 size = buffer->buf.len - buffer->posn;
41         memcpy(ptr, buffer->buf.buf + buffer->posn, size);
42         buffer->posn += size;
43
44         return size;
45 }
46
47 #ifndef NO_CURL_IOCTL
48 curlioerr ioctl_buffer(CURL *handle, int cmd, void *clientp)
49 {
50         struct buffer *buffer = clientp;
51
52         switch (cmd) {
53         case CURLIOCMD_NOP:
54                 return CURLIOE_OK;
55
56         case CURLIOCMD_RESTARTREAD:
57                 buffer->posn = 0;
58                 return CURLIOE_OK;
59
60         default:
61                 return CURLIOE_UNKNOWNCMD;
62         }
63 }
64 #endif
65
66 size_t fwrite_buffer(const void *ptr, size_t eltsize, size_t nmemb, void *buffer_)
67 {
68         size_t size = eltsize * nmemb;
69         struct strbuf *buffer = buffer_;
70
71         strbuf_add(buffer, ptr, size);
72         data_received++;
73         return size;
74 }
75
76 size_t fwrite_null(const void *ptr, size_t eltsize, size_t nmemb, void *strbuf)
77 {
78         data_received++;
79         return eltsize * nmemb;
80 }
81
82 static void finish_active_slot(struct active_request_slot *slot);
83
84 #ifdef USE_CURL_MULTI
85 static void process_curl_messages(void)
86 {
87         int num_messages;
88         struct active_request_slot *slot;
89         CURLMsg *curl_message = curl_multi_info_read(curlm, &num_messages);
90
91         while (curl_message != NULL) {
92                 if (curl_message->msg == CURLMSG_DONE) {
93                         int curl_result = curl_message->data.result;
94                         slot = active_queue_head;
95                         while (slot != NULL &&
96                                slot->curl != curl_message->easy_handle)
97                                 slot = slot->next;
98                         if (slot != NULL) {
99                                 curl_multi_remove_handle(curlm, slot->curl);
100                                 slot->curl_result = curl_result;
101                                 finish_active_slot(slot);
102                         } else {
103                                 fprintf(stderr, "Received DONE message for unknown request!\n");
104                         }
105                 } else {
106                         fprintf(stderr, "Unknown CURL message received: %d\n",
107                                 (int)curl_message->msg);
108                 }
109                 curl_message = curl_multi_info_read(curlm, &num_messages);
110         }
111 }
112 #endif
113
114 static int http_options(const char *var, const char *value, void *cb)
115 {
116         if (!strcmp("http.sslverify", var)) {
117                 curl_ssl_verify = git_config_bool(var, value);
118                 return 0;
119         }
120         if (!strcmp("http.sslcert", var))
121                 return git_config_string(&ssl_cert, var, value);
122 #if LIBCURL_VERSION_NUM >= 0x070902
123         if (!strcmp("http.sslkey", var))
124                 return git_config_string(&ssl_key, var, value);
125 #endif
126 #if LIBCURL_VERSION_NUM >= 0x070908
127         if (!strcmp("http.sslcapath", var))
128                 return git_config_string(&ssl_capath, var, value);
129 #endif
130         if (!strcmp("http.sslcainfo", var))
131                 return git_config_string(&ssl_cainfo, var, value);
132 #ifdef USE_CURL_MULTI
133         if (!strcmp("http.maxrequests", var)) {
134                 max_requests = git_config_int(var, value);
135                 return 0;
136         }
137 #endif
138         if (!strcmp("http.lowspeedlimit", var)) {
139                 curl_low_speed_limit = (long)git_config_int(var, value);
140                 return 0;
141         }
142         if (!strcmp("http.lowspeedtime", var)) {
143                 curl_low_speed_time = (long)git_config_int(var, value);
144                 return 0;
145         }
146
147         if (!strcmp("http.noepsv", var)) {
148                 curl_ftp_no_epsv = git_config_bool(var, value);
149                 return 0;
150         }
151         if (!strcmp("http.proxy", var))
152                 return git_config_string(&curl_http_proxy, var, value);
153
154         /* Fall back on the default ones */
155         return git_default_config(var, value, cb);
156 }
157
158 static void init_curl_http_auth(CURL *result)
159 {
160         if (user_name) {
161                 struct strbuf up = STRBUF_INIT;
162                 if (!user_pass)
163                         user_pass = xstrdup(getpass("Password: "));
164                 strbuf_addf(&up, "%s:%s", user_name, user_pass);
165                 curl_easy_setopt(result, CURLOPT_USERPWD,
166                                  strbuf_detach(&up, NULL));
167         }
168 }
169
170 static CURL *get_curl_handle(void)
171 {
172         CURL *result = curl_easy_init();
173
174         if (!curl_ssl_verify) {
175                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 0);
176                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 0);
177         } else {
178                 /* Verify authenticity of the peer's certificate */
179                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYPEER, 1);
180                 /* The name in the cert must match whom we tried to connect */
181                 curl_easy_setopt(result, CURLOPT_SSL_VERIFYHOST, 2);
182         }
183
184 #if LIBCURL_VERSION_NUM >= 0x070907
185         curl_easy_setopt(result, CURLOPT_NETRC, CURL_NETRC_OPTIONAL);
186 #endif
187
188         init_curl_http_auth(result);
189
190         if (ssl_cert != NULL)
191                 curl_easy_setopt(result, CURLOPT_SSLCERT, ssl_cert);
192 #if LIBCURL_VERSION_NUM >= 0x070902
193         if (ssl_key != NULL)
194                 curl_easy_setopt(result, CURLOPT_SSLKEY, ssl_key);
195 #endif
196 #if LIBCURL_VERSION_NUM >= 0x070908
197         if (ssl_capath != NULL)
198                 curl_easy_setopt(result, CURLOPT_CAPATH, ssl_capath);
199 #endif
200         if (ssl_cainfo != NULL)
201                 curl_easy_setopt(result, CURLOPT_CAINFO, ssl_cainfo);
202         curl_easy_setopt(result, CURLOPT_FAILONERROR, 1);
203
204         if (curl_low_speed_limit > 0 && curl_low_speed_time > 0) {
205                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_LIMIT,
206                                  curl_low_speed_limit);
207                 curl_easy_setopt(result, CURLOPT_LOW_SPEED_TIME,
208                                  curl_low_speed_time);
209         }
210
211         curl_easy_setopt(result, CURLOPT_FOLLOWLOCATION, 1);
212
213         if (getenv("GIT_CURL_VERBOSE"))
214                 curl_easy_setopt(result, CURLOPT_VERBOSE, 1);
215
216         curl_easy_setopt(result, CURLOPT_USERAGENT, GIT_USER_AGENT);
217
218         if (curl_ftp_no_epsv)
219                 curl_easy_setopt(result, CURLOPT_FTP_USE_EPSV, 0);
220
221         if (curl_http_proxy)
222                 curl_easy_setopt(result, CURLOPT_PROXY, curl_http_proxy);
223
224         return result;
225 }
226
227 static void http_auth_init(const char *url)
228 {
229         char *at, *colon, *cp, *slash;
230         int len;
231
232         cp = strstr(url, "://");
233         if (!cp)
234                 return;
235
236         /*
237          * Ok, the URL looks like "proto://something".  Which one?
238          * "proto://<user>:<pass>@<host>/...",
239          * "proto://<user>@<host>/...", or just
240          * "proto://<host>/..."?
241          */
242         cp += 3;
243         at = strchr(cp, '@');
244         colon = strchr(cp, ':');
245         slash = strchrnul(cp, '/');
246         if (!at || slash <= at)
247                 return; /* No credentials */
248         if (!colon || at <= colon) {
249                 /* Only username */
250                 len = at - cp;
251                 user_name = xmalloc(len + 1);
252                 memcpy(user_name, cp, len);
253                 user_name[len] = '\0';
254                 user_pass = NULL;
255         } else {
256                 len = colon - cp;
257                 user_name = xmalloc(len + 1);
258                 memcpy(user_name, cp, len);
259                 user_name[len] = '\0';
260                 len = at - (colon + 1);
261                 user_pass = xmalloc(len + 1);
262                 memcpy(user_pass, colon + 1, len);
263                 user_pass[len] = '\0';
264         }
265 }
266
267 static void set_from_env(const char **var, const char *envname)
268 {
269         const char *val = getenv(envname);
270         if (val)
271                 *var = val;
272 }
273
274 void http_init(struct remote *remote)
275 {
276         char *low_speed_limit;
277         char *low_speed_time;
278
279         git_config(http_options, NULL);
280
281         curl_global_init(CURL_GLOBAL_ALL);
282
283         if (remote && remote->http_proxy)
284                 curl_http_proxy = xstrdup(remote->http_proxy);
285
286         pragma_header = curl_slist_append(pragma_header, "Pragma: no-cache");
287
288 #ifdef USE_CURL_MULTI
289         {
290                 char *http_max_requests = getenv("GIT_HTTP_MAX_REQUESTS");
291                 if (http_max_requests != NULL)
292                         max_requests = atoi(http_max_requests);
293         }
294
295         curlm = curl_multi_init();
296         if (curlm == NULL) {
297                 fprintf(stderr, "Error creating curl multi handle.\n");
298                 exit(1);
299         }
300 #endif
301
302         if (getenv("GIT_SSL_NO_VERIFY"))
303                 curl_ssl_verify = 0;
304
305         set_from_env(&ssl_cert, "GIT_SSL_CERT");
306 #if LIBCURL_VERSION_NUM >= 0x070902
307         set_from_env(&ssl_key, "GIT_SSL_KEY");
308 #endif
309 #if LIBCURL_VERSION_NUM >= 0x070908
310         set_from_env(&ssl_capath, "GIT_SSL_CAPATH");
311 #endif
312         set_from_env(&ssl_cainfo, "GIT_SSL_CAINFO");
313
314         low_speed_limit = getenv("GIT_HTTP_LOW_SPEED_LIMIT");
315         if (low_speed_limit != NULL)
316                 curl_low_speed_limit = strtol(low_speed_limit, NULL, 10);
317         low_speed_time = getenv("GIT_HTTP_LOW_SPEED_TIME");
318         if (low_speed_time != NULL)
319                 curl_low_speed_time = strtol(low_speed_time, NULL, 10);
320
321         if (curl_ssl_verify == -1)
322                 curl_ssl_verify = 1;
323
324 #ifdef USE_CURL_MULTI
325         if (max_requests < 1)
326                 max_requests = DEFAULT_MAX_REQUESTS;
327 #endif
328
329         if (getenv("GIT_CURL_FTP_NO_EPSV"))
330                 curl_ftp_no_epsv = 1;
331
332         if (remote && remote->url && remote->url[0])
333                 http_auth_init(remote->url[0]);
334
335 #ifndef NO_CURL_EASY_DUPHANDLE
336         curl_default = get_curl_handle();
337 #endif
338 }
339
340 void http_cleanup(void)
341 {
342         struct active_request_slot *slot = active_queue_head;
343
344         while (slot != NULL) {
345                 struct active_request_slot *next = slot->next;
346                 if (slot->curl != NULL) {
347 #ifdef USE_CURL_MULTI
348                         curl_multi_remove_handle(curlm, slot->curl);
349 #endif
350                         curl_easy_cleanup(slot->curl);
351                 }
352                 free(slot);
353                 slot = next;
354         }
355         active_queue_head = NULL;
356
357 #ifndef NO_CURL_EASY_DUPHANDLE
358         curl_easy_cleanup(curl_default);
359 #endif
360
361 #ifdef USE_CURL_MULTI
362         curl_multi_cleanup(curlm);
363 #endif
364         curl_global_cleanup();
365
366         curl_slist_free_all(pragma_header);
367         pragma_header = NULL;
368
369         if (curl_http_proxy) {
370                 free((void *)curl_http_proxy);
371                 curl_http_proxy = NULL;
372         }
373 }
374
375 struct active_request_slot *get_active_slot(void)
376 {
377         struct active_request_slot *slot = active_queue_head;
378         struct active_request_slot *newslot;
379
380 #ifdef USE_CURL_MULTI
381         int num_transfers;
382
383         /* Wait for a slot to open up if the queue is full */
384         while (active_requests >= max_requests) {
385                 curl_multi_perform(curlm, &num_transfers);
386                 if (num_transfers < active_requests)
387                         process_curl_messages();
388         }
389 #endif
390
391         while (slot != NULL && slot->in_use)
392                 slot = slot->next;
393
394         if (slot == NULL) {
395                 newslot = xmalloc(sizeof(*newslot));
396                 newslot->curl = NULL;
397                 newslot->in_use = 0;
398                 newslot->next = NULL;
399
400                 slot = active_queue_head;
401                 if (slot == NULL) {
402                         active_queue_head = newslot;
403                 } else {
404                         while (slot->next != NULL)
405                                 slot = slot->next;
406                         slot->next = newslot;
407                 }
408                 slot = newslot;
409         }
410
411         if (slot->curl == NULL) {
412 #ifdef NO_CURL_EASY_DUPHANDLE
413                 slot->curl = get_curl_handle();
414 #else
415                 slot->curl = curl_easy_duphandle(curl_default);
416 #endif
417         }
418
419         active_requests++;
420         slot->in_use = 1;
421         slot->local = NULL;
422         slot->results = NULL;
423         slot->finished = NULL;
424         slot->callback_data = NULL;
425         slot->callback_func = NULL;
426         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, pragma_header);
427         curl_easy_setopt(slot->curl, CURLOPT_ERRORBUFFER, curl_errorstr);
428         curl_easy_setopt(slot->curl, CURLOPT_CUSTOMREQUEST, NULL);
429         curl_easy_setopt(slot->curl, CURLOPT_READFUNCTION, NULL);
430         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, NULL);
431         curl_easy_setopt(slot->curl, CURLOPT_UPLOAD, 0);
432         curl_easy_setopt(slot->curl, CURLOPT_HTTPGET, 1);
433
434         return slot;
435 }
436
437 int start_active_slot(struct active_request_slot *slot)
438 {
439 #ifdef USE_CURL_MULTI
440         CURLMcode curlm_result = curl_multi_add_handle(curlm, slot->curl);
441         int num_transfers;
442
443         if (curlm_result != CURLM_OK &&
444             curlm_result != CURLM_CALL_MULTI_PERFORM) {
445                 active_requests--;
446                 slot->in_use = 0;
447                 return 0;
448         }
449
450         /*
451          * We know there must be something to do, since we just added
452          * something.
453          */
454         curl_multi_perform(curlm, &num_transfers);
455 #endif
456         return 1;
457 }
458
459 #ifdef USE_CURL_MULTI
460 struct fill_chain {
461         void *data;
462         int (*fill)(void *);
463         struct fill_chain *next;
464 };
465
466 static struct fill_chain *fill_cfg;
467
468 void add_fill_function(void *data, int (*fill)(void *))
469 {
470         struct fill_chain *new = xmalloc(sizeof(*new));
471         struct fill_chain **linkp = &fill_cfg;
472         new->data = data;
473         new->fill = fill;
474         new->next = NULL;
475         while (*linkp)
476                 linkp = &(*linkp)->next;
477         *linkp = new;
478 }
479
480 void fill_active_slots(void)
481 {
482         struct active_request_slot *slot = active_queue_head;
483
484         while (active_requests < max_requests) {
485                 struct fill_chain *fill;
486                 for (fill = fill_cfg; fill; fill = fill->next)
487                         if (fill->fill(fill->data))
488                                 break;
489
490                 if (!fill)
491                         break;
492         }
493
494         while (slot != NULL) {
495                 if (!slot->in_use && slot->curl != NULL) {
496                         curl_easy_cleanup(slot->curl);
497                         slot->curl = NULL;
498                 }
499                 slot = slot->next;
500         }
501 }
502
503 void step_active_slots(void)
504 {
505         int num_transfers;
506         CURLMcode curlm_result;
507
508         do {
509                 curlm_result = curl_multi_perform(curlm, &num_transfers);
510         } while (curlm_result == CURLM_CALL_MULTI_PERFORM);
511         if (num_transfers < active_requests) {
512                 process_curl_messages();
513                 fill_active_slots();
514         }
515 }
516 #endif
517
518 void run_active_slot(struct active_request_slot *slot)
519 {
520 #ifdef USE_CURL_MULTI
521         long last_pos = 0;
522         long current_pos;
523         fd_set readfds;
524         fd_set writefds;
525         fd_set excfds;
526         int max_fd;
527         struct timeval select_timeout;
528         int finished = 0;
529
530         slot->finished = &finished;
531         while (!finished) {
532                 data_received = 0;
533                 step_active_slots();
534
535                 if (!data_received && slot->local != NULL) {
536                         current_pos = ftell(slot->local);
537                         if (current_pos > last_pos)
538                                 data_received++;
539                         last_pos = current_pos;
540                 }
541
542                 if (slot->in_use && !data_received) {
543                         max_fd = 0;
544                         FD_ZERO(&readfds);
545                         FD_ZERO(&writefds);
546                         FD_ZERO(&excfds);
547                         select_timeout.tv_sec = 0;
548                         select_timeout.tv_usec = 50000;
549                         select(max_fd, &readfds, &writefds,
550                                &excfds, &select_timeout);
551                 }
552         }
553 #else
554         while (slot->in_use) {
555                 slot->curl_result = curl_easy_perform(slot->curl);
556                 finish_active_slot(slot);
557         }
558 #endif
559 }
560
561 static void closedown_active_slot(struct active_request_slot *slot)
562 {
563         active_requests--;
564         slot->in_use = 0;
565 }
566
567 void release_active_slot(struct active_request_slot *slot)
568 {
569         closedown_active_slot(slot);
570         if (slot->curl) {
571 #ifdef USE_CURL_MULTI
572                 curl_multi_remove_handle(curlm, slot->curl);
573 #endif
574                 curl_easy_cleanup(slot->curl);
575                 slot->curl = NULL;
576         }
577 #ifdef USE_CURL_MULTI
578         fill_active_slots();
579 #endif
580 }
581
582 static void finish_active_slot(struct active_request_slot *slot)
583 {
584         closedown_active_slot(slot);
585         curl_easy_getinfo(slot->curl, CURLINFO_HTTP_CODE, &slot->http_code);
586
587         if (slot->finished != NULL)
588                 (*slot->finished) = 1;
589
590         /* Store slot results so they can be read after the slot is reused */
591         if (slot->results != NULL) {
592                 slot->results->curl_result = slot->curl_result;
593                 slot->results->http_code = slot->http_code;
594         }
595
596         /* Run callback if appropriate */
597         if (slot->callback_func != NULL)
598                 slot->callback_func(slot->callback_data);
599 }
600
601 void finish_all_active_slots(void)
602 {
603         struct active_request_slot *slot = active_queue_head;
604
605         while (slot != NULL)
606                 if (slot->in_use) {
607                         run_active_slot(slot);
608                         slot = active_queue_head;
609                 } else {
610                         slot = slot->next;
611                 }
612 }
613
614 static inline int needs_quote(int ch)
615 {
616         if (((ch >= 'A') && (ch <= 'Z'))
617                         || ((ch >= 'a') && (ch <= 'z'))
618                         || ((ch >= '0') && (ch <= '9'))
619                         || (ch == '/')
620                         || (ch == '-')
621                         || (ch == '.'))
622                 return 0;
623         return 1;
624 }
625
626 static inline int hex(int v)
627 {
628         if (v < 10)
629                 return '0' + v;
630         else
631                 return 'A' + v - 10;
632 }
633
634 static char *quote_ref_url(const char *base, const char *ref)
635 {
636         struct strbuf buf = STRBUF_INIT;
637         const char *cp;
638         int ch;
639
640         strbuf_addstr(&buf, base);
641         if (buf.len && buf.buf[buf.len - 1] != '/' && *ref != '/')
642                 strbuf_addstr(&buf, "/");
643
644         for (cp = ref; (ch = *cp) != 0; cp++)
645                 if (needs_quote(ch))
646                         strbuf_addf(&buf, "%%%02x", ch);
647                 else
648                         strbuf_addch(&buf, *cp);
649
650         return strbuf_detach(&buf, NULL);
651 }
652
653 int http_fetch_ref(const char *base, struct ref *ref)
654 {
655         char *url;
656         struct strbuf buffer = STRBUF_INIT;
657         struct active_request_slot *slot;
658         struct slot_results results;
659         int ret;
660
661         url = quote_ref_url(base, ref->name);
662         slot = get_active_slot();
663         slot->results = &results;
664         curl_easy_setopt(slot->curl, CURLOPT_FILE, &buffer);
665         curl_easy_setopt(slot->curl, CURLOPT_WRITEFUNCTION, fwrite_buffer);
666         curl_easy_setopt(slot->curl, CURLOPT_HTTPHEADER, NULL);
667         curl_easy_setopt(slot->curl, CURLOPT_URL, url);
668         if (start_active_slot(slot)) {
669                 run_active_slot(slot);
670                 if (results.curl_result == CURLE_OK) {
671                         strbuf_rtrim(&buffer);
672                         if (buffer.len == 40)
673                                 ret = get_sha1_hex(buffer.buf, ref->old_sha1);
674                         else if (!prefixcmp(buffer.buf, "ref: ")) {
675                                 ref->symref = xstrdup(buffer.buf + 5);
676                                 ret = 0;
677                         } else
678                                 ret = 1;
679                 } else {
680                         ret = error("Couldn't get %s for %s\n%s",
681                                     url, ref->name, curl_errorstr);
682                 }
683         } else {
684                 ret = error("Unable to start request");
685         }
686
687         strbuf_release(&buffer);
688         free(url);
689         return ret;
690 }