Merge branch 'maint'
[git] / fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9
10 static int keep_pack;
11 static int transfer_unpack_limit = -1;
12 static int fetch_unpack_limit = -1;
13 static int unpack_limit = 100;
14 static int quiet;
15 static int verbose;
16 static int fetch_all;
17 static int depth;
18 static const char fetch_pack_usage[] =
19 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [-v] [<host>:]<directory> [<refs>...]";
20 static const char *uploadpack = "git-upload-pack";
21
22 #define COMPLETE        (1U << 0)
23 #define COMMON          (1U << 1)
24 #define COMMON_REF      (1U << 2)
25 #define SEEN            (1U << 3)
26 #define POPPED          (1U << 4)
27
28 /*
29  * After sending this many "have"s if we do not get any new ACK , we
30  * give up traversing our history.
31  */
32 #define MAX_IN_VAIN 256
33
34 static struct commit_list *rev_list;
35 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
36
37 static void rev_list_push(struct commit *commit, int mark)
38 {
39         if (!(commit->object.flags & mark)) {
40                 commit->object.flags |= mark;
41
42                 if (!(commit->object.parsed))
43                         parse_commit(commit);
44
45                 insert_by_date(commit, &rev_list);
46
47                 if (!(commit->object.flags & COMMON))
48                         non_common_revs++;
49         }
50 }
51
52 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
53 {
54         struct object *o = deref_tag(parse_object(sha1), path, 0);
55
56         if (o && o->type == OBJ_COMMIT)
57                 rev_list_push((struct commit *)o, SEEN);
58
59         return 0;
60 }
61
62 /*
63    This function marks a rev and its ancestors as common.
64    In some cases, it is desirable to mark only the ancestors (for example
65    when only the server does not yet know that they are common).
66 */
67
68 static void mark_common(struct commit *commit,
69                 int ancestors_only, int dont_parse)
70 {
71         if (commit != NULL && !(commit->object.flags & COMMON)) {
72                 struct object *o = (struct object *)commit;
73
74                 if (!ancestors_only)
75                         o->flags |= COMMON;
76
77                 if (!(o->flags & SEEN))
78                         rev_list_push(commit, SEEN);
79                 else {
80                         struct commit_list *parents;
81
82                         if (!ancestors_only && !(o->flags & POPPED))
83                                 non_common_revs--;
84                         if (!o->parsed && !dont_parse)
85                                 parse_commit(commit);
86
87                         for (parents = commit->parents;
88                                         parents;
89                                         parents = parents->next)
90                                 mark_common(parents->item, 0, dont_parse);
91                 }
92         }
93 }
94
95 /*
96   Get the next rev to send, ignoring the common.
97 */
98
99 static const unsigned char* get_rev(void)
100 {
101         struct commit *commit = NULL;
102
103         while (commit == NULL) {
104                 unsigned int mark;
105                 struct commit_list* parents;
106
107                 if (rev_list == NULL || non_common_revs == 0)
108                         return NULL;
109
110                 commit = rev_list->item;
111                 if (!(commit->object.parsed))
112                         parse_commit(commit);
113                 commit->object.flags |= POPPED;
114                 if (!(commit->object.flags & COMMON))
115                         non_common_revs--;
116         
117                 parents = commit->parents;
118
119                 if (commit->object.flags & COMMON) {
120                         /* do not send "have", and ignore ancestors */
121                         commit = NULL;
122                         mark = COMMON | SEEN;
123                 } else if (commit->object.flags & COMMON_REF)
124                         /* send "have", and ignore ancestors */
125                         mark = COMMON | SEEN;
126                 else
127                         /* send "have", also for its ancestors */
128                         mark = SEEN;
129
130                 while (parents) {
131                         if (!(parents->item->object.flags & SEEN))
132                                 rev_list_push(parents->item, mark);
133                         if (mark & COMMON)
134                                 mark_common(parents->item, 1, 0);
135                         parents = parents->next;
136                 }
137
138                 rev_list = rev_list->next;
139         }
140
141         return commit->object.sha1;
142 }
143
144 static int find_common(int fd[2], unsigned char *result_sha1,
145                        struct ref *refs)
146 {
147         int fetching;
148         int count = 0, flushes = 0, retval;
149         const unsigned char *sha1;
150         unsigned in_vain = 0;
151         int got_continue = 0;
152
153         for_each_ref(rev_list_insert_ref, NULL);
154
155         fetching = 0;
156         for ( ; refs ; refs = refs->next) {
157                 unsigned char *remote = refs->old_sha1;
158                 struct object *o;
159
160                 /*
161                  * If that object is complete (i.e. it is an ancestor of a
162                  * local ref), we tell them we have it but do not have to
163                  * tell them about its ancestors, which they already know
164                  * about.
165                  *
166                  * We use lookup_object here because we are only
167                  * interested in the case we *know* the object is
168                  * reachable and we have already scanned it.
169                  */
170                 if (((o = lookup_object(remote)) != NULL) &&
171                                 (o->flags & COMPLETE)) {
172                         continue;
173                 }
174
175                 if (!fetching)
176                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
177                                      sha1_to_hex(remote),
178                                      (multi_ack ? " multi_ack" : ""),
179                                      (use_sideband == 2 ? " side-band-64k" : ""),
180                                      (use_sideband == 1 ? " side-band" : ""),
181                                      (use_thin_pack ? " thin-pack" : ""),
182                                      " ofs-delta");
183                 else
184                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
185                 fetching++;
186         }
187         if (is_repository_shallow())
188                 write_shallow_commits(fd[1], 1);
189         if (depth > 0)
190                 packet_write(fd[1], "deepen %d", depth);
191         packet_flush(fd[1]);
192         if (!fetching)
193                 return 1;
194
195         if (depth > 0) {
196                 char line[1024];
197                 unsigned char sha1[20];
198                 int len;
199
200                 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
201                         if (!prefixcmp(line, "shallow ")) {
202                                 if (get_sha1_hex(line + 8, sha1))
203                                         die("invalid shallow line: %s", line);
204                                 register_shallow(sha1);
205                                 continue;
206                         }
207                         if (!prefixcmp(line, "unshallow ")) {
208                                 if (get_sha1_hex(line + 10, sha1))
209                                         die("invalid unshallow line: %s", line);
210                                 if (!lookup_object(sha1))
211                                         die("object not found: %s", line);
212                                 /* make sure that it is parsed as shallow */
213                                 parse_object(sha1);
214                                 if (unregister_shallow(sha1))
215                                         die("no shallow found: %s", line);
216                                 continue;
217                         }
218                         die("expected shallow/unshallow, got %s", line);
219                 }
220         }
221
222         flushes = 0;
223         retval = -1;
224         while ((sha1 = get_rev())) {
225                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
226                 if (verbose)
227                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
228                 in_vain++;
229                 if (!(31 & ++count)) {
230                         int ack;
231
232                         packet_flush(fd[1]);
233                         flushes++;
234
235                         /*
236                          * We keep one window "ahead" of the other side, and
237                          * will wait for an ACK only on the next one
238                          */
239                         if (count == 32)
240                                 continue;
241
242                         do {
243                                 ack = get_ack(fd[0], result_sha1);
244                                 if (verbose && ack)
245                                         fprintf(stderr, "got ack %d %s\n", ack,
246                                                         sha1_to_hex(result_sha1));
247                                 if (ack == 1) {
248                                         flushes = 0;
249                                         multi_ack = 0;
250                                         retval = 0;
251                                         goto done;
252                                 } else if (ack == 2) {
253                                         struct commit *commit =
254                                                 lookup_commit(result_sha1);
255                                         mark_common(commit, 0, 1);
256                                         retval = 0;
257                                         in_vain = 0;
258                                         got_continue = 1;
259                                 }
260                         } while (ack);
261                         flushes--;
262                         if (got_continue && MAX_IN_VAIN < in_vain) {
263                                 if (verbose)
264                                         fprintf(stderr, "giving up\n");
265                                 break; /* give up */
266                         }
267                 }
268         }
269 done:
270         packet_write(fd[1], "done\n");
271         if (verbose)
272                 fprintf(stderr, "done\n");
273         if (retval != 0) {
274                 multi_ack = 0;
275                 flushes++;
276         }
277         while (flushes || multi_ack) {
278                 int ack = get_ack(fd[0], result_sha1);
279                 if (ack) {
280                         if (verbose)
281                                 fprintf(stderr, "got ack (%d) %s\n", ack,
282                                         sha1_to_hex(result_sha1));
283                         if (ack == 1)
284                                 return 0;
285                         multi_ack = 1;
286                         continue;
287                 }
288                 flushes--;
289         }
290         return retval;
291 }
292
293 static struct commit_list *complete;
294
295 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
296 {
297         struct object *o = parse_object(sha1);
298
299         while (o && o->type == OBJ_TAG) {
300                 struct tag *t = (struct tag *) o;
301                 if (!t->tagged)
302                         break; /* broken repository */
303                 o->flags |= COMPLETE;
304                 o = parse_object(t->tagged->sha1);
305         }
306         if (o && o->type == OBJ_COMMIT) {
307                 struct commit *commit = (struct commit *)o;
308                 commit->object.flags |= COMPLETE;
309                 insert_by_date(commit, &complete);
310         }
311         return 0;
312 }
313
314 static void mark_recent_complete_commits(unsigned long cutoff)
315 {
316         while (complete && cutoff <= complete->item->date) {
317                 if (verbose)
318                         fprintf(stderr, "Marking %s as complete\n",
319                                 sha1_to_hex(complete->item->object.sha1));
320                 pop_most_recent_commit(&complete, COMPLETE);
321         }
322 }
323
324 static void filter_refs(struct ref **refs, int nr_match, char **match)
325 {
326         struct ref **return_refs;
327         struct ref *newlist = NULL;
328         struct ref **newtail = &newlist;
329         struct ref *ref, *next;
330         struct ref *fastarray[32];
331
332         if (nr_match && !fetch_all) {
333                 if (ARRAY_SIZE(fastarray) < nr_match)
334                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
335                 else {
336                         return_refs = fastarray;
337                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
338                 }
339         }
340         else
341                 return_refs = NULL;
342
343         for (ref = *refs; ref; ref = next) {
344                 next = ref->next;
345                 if (!memcmp(ref->name, "refs/", 5) &&
346                     check_ref_format(ref->name + 5))
347                         ; /* trash */
348                 else if (fetch_all &&
349                          (!depth || prefixcmp(ref->name, "refs/tags/") )) {
350                         *newtail = ref;
351                         ref->next = NULL;
352                         newtail = &ref->next;
353                         continue;
354                 }
355                 else {
356                         int order = path_match(ref->name, nr_match, match);
357                         if (order) {
358                                 return_refs[order-1] = ref;
359                                 continue; /* we will link it later */
360                         }
361                 }
362                 free(ref);
363         }
364
365         if (!fetch_all) {
366                 int i;
367                 for (i = 0; i < nr_match; i++) {
368                         ref = return_refs[i];
369                         if (ref) {
370                                 *newtail = ref;
371                                 ref->next = NULL;
372                                 newtail = &ref->next;
373                         }
374                 }
375                 if (return_refs != fastarray)
376                         free(return_refs);
377         }
378         *refs = newlist;
379 }
380
381 static int everything_local(struct ref **refs, int nr_match, char **match)
382 {
383         struct ref *ref;
384         int retval;
385         unsigned long cutoff = 0;
386
387         track_object_refs = 0;
388         save_commit_buffer = 0;
389
390         for (ref = *refs; ref; ref = ref->next) {
391                 struct object *o;
392
393                 o = parse_object(ref->old_sha1);
394                 if (!o)
395                         continue;
396
397                 /* We already have it -- which may mean that we were
398                  * in sync with the other side at some time after
399                  * that (it is OK if we guess wrong here).
400                  */
401                 if (o->type == OBJ_COMMIT) {
402                         struct commit *commit = (struct commit *)o;
403                         if (!cutoff || cutoff < commit->date)
404                                 cutoff = commit->date;
405                 }
406         }
407
408         if (!depth) {
409                 for_each_ref(mark_complete, NULL);
410                 if (cutoff)
411                         mark_recent_complete_commits(cutoff);
412         }
413
414         /*
415          * Mark all complete remote refs as common refs.
416          * Don't mark them common yet; the server has to be told so first.
417          */
418         for (ref = *refs; ref; ref = ref->next) {
419                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
420                                              NULL, 0);
421
422                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
423                         continue;
424
425                 if (!(o->flags & SEEN)) {
426                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
427
428                         mark_common((struct commit *)o, 1, 1);
429                 }
430         }
431
432         filter_refs(refs, nr_match, match);
433
434         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
435                 const unsigned char *remote = ref->old_sha1;
436                 unsigned char local[20];
437                 struct object *o;
438
439                 o = lookup_object(remote);
440                 if (!o || !(o->flags & COMPLETE)) {
441                         retval = 0;
442                         if (!verbose)
443                                 continue;
444                         fprintf(stderr,
445                                 "want %s (%s)\n", sha1_to_hex(remote),
446                                 ref->name);
447                         continue;
448                 }
449
450                 hashcpy(ref->new_sha1, local);
451                 if (!verbose)
452                         continue;
453                 fprintf(stderr,
454                         "already have %s (%s)\n", sha1_to_hex(remote),
455                         ref->name);
456         }
457         return retval;
458 }
459
460 static pid_t setup_sideband(int fd[2], int xd[2])
461 {
462         pid_t side_pid;
463
464         if (!use_sideband) {
465                 fd[0] = xd[0];
466                 fd[1] = xd[1];
467                 return 0;
468         }
469         /* xd[] is talking with upload-pack; subprocess reads from
470          * xd[0], spits out band#2 to stderr, and feeds us band#1
471          * through our fd[0].
472          */
473         if (pipe(fd) < 0)
474                 die("fetch-pack: unable to set up pipe");
475         side_pid = fork();
476         if (side_pid < 0)
477                 die("fetch-pack: unable to fork off sideband demultiplexer");
478         if (!side_pid) {
479                 /* subprocess */
480                 close(fd[0]);
481                 if (xd[0] != xd[1])
482                         close(xd[1]);
483                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
484                         exit(1);
485                 exit(0);
486         }
487         close(xd[0]);
488         close(fd[1]);
489         fd[1] = xd[1];
490         return side_pid;
491 }
492
493 static int get_pack(int xd[2])
494 {
495         int status;
496         pid_t pid, side_pid;
497         int fd[2];
498         const char *argv[20];
499         char keep_arg[256];
500         char hdr_arg[256];
501         const char **av;
502         int do_keep = keep_pack;
503
504         side_pid = setup_sideband(fd, xd);
505
506         av = argv;
507         *hdr_arg = 0;
508         if (unpack_limit) {
509                 struct pack_header header;
510
511                 if (read_pack_header(fd[0], &header))
512                         die("protocol error: bad pack header");
513                 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
514                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
515                 if (ntohl(header.hdr_entries) < unpack_limit)
516                         do_keep = 0;
517                 else
518                         do_keep = 1;
519         }
520
521         if (do_keep) {
522                 *av++ = "index-pack";
523                 *av++ = "--stdin";
524                 if (!quiet)
525                         *av++ = "-v";
526                 if (use_thin_pack)
527                         *av++ = "--fix-thin";
528                 if (keep_pack > 1 || unpack_limit) {
529                         int s = sprintf(keep_arg,
530                                         "--keep=fetch-pack %d on ", getpid());
531                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
532                                 strcpy(keep_arg + s, "localhost");
533                         *av++ = keep_arg;
534                 }
535         }
536         else {
537                 *av++ = "unpack-objects";
538                 if (quiet)
539                         *av++ = "-q";
540         }
541         if (*hdr_arg)
542                 *av++ = hdr_arg;
543         *av++ = NULL;
544
545         pid = fork();
546         if (pid < 0)
547                 die("fetch-pack: unable to fork off %s", argv[0]);
548         if (!pid) {
549                 dup2(fd[0], 0);
550                 close(fd[0]);
551                 close(fd[1]);
552                 execv_git_cmd(argv);
553                 die("%s exec failed", argv[0]);
554         }
555         close(fd[0]);
556         close(fd[1]);
557         while (waitpid(pid, &status, 0) < 0) {
558                 if (errno != EINTR)
559                         die("waiting for %s: %s", argv[0], strerror(errno));
560         }
561         if (WIFEXITED(status)) {
562                 int code = WEXITSTATUS(status);
563                 if (code)
564                         die("%s died with error code %d", argv[0], code);
565                 return 0;
566         }
567         if (WIFSIGNALED(status)) {
568                 int sig = WTERMSIG(status);
569                 die("%s died of signal %d", argv[0], sig);
570         }
571         die("%s died of unnatural causes %d", argv[0], status);
572 }
573
574 static int fetch_pack(int fd[2], int nr_match, char **match)
575 {
576         struct ref *ref;
577         unsigned char sha1[20];
578
579         get_remote_heads(fd[0], &ref, 0, NULL, 0);
580         if (is_repository_shallow() && !server_supports("shallow"))
581                 die("Server does not support shallow clients");
582         if (server_supports("multi_ack")) {
583                 if (verbose)
584                         fprintf(stderr, "Server supports multi_ack\n");
585                 multi_ack = 1;
586         }
587         if (server_supports("side-band-64k")) {
588                 if (verbose)
589                         fprintf(stderr, "Server supports side-band-64k\n");
590                 use_sideband = 2;
591         }
592         else if (server_supports("side-band")) {
593                 if (verbose)
594                         fprintf(stderr, "Server supports side-band\n");
595                 use_sideband = 1;
596         }
597         if (!ref) {
598                 packet_flush(fd[1]);
599                 die("no matching remote head");
600         }
601         if (everything_local(&ref, nr_match, match)) {
602                 packet_flush(fd[1]);
603                 goto all_done;
604         }
605         if (find_common(fd, sha1, ref) < 0)
606                 if (keep_pack != 1)
607                         /* When cloning, it is not unusual to have
608                          * no common commit.
609                          */
610                         fprintf(stderr, "warning: no common commits\n");
611
612         if (get_pack(fd))
613                 die("git-fetch-pack: fetch failed.");
614
615  all_done:
616         while (ref) {
617                 printf("%s %s\n",
618                        sha1_to_hex(ref->old_sha1), ref->name);
619                 ref = ref->next;
620         }
621         return 0;
622 }
623
624 static int remove_duplicates(int nr_heads, char **heads)
625 {
626         int src, dst;
627
628         for (src = dst = 0; src < nr_heads; src++) {
629                 /* If heads[src] is different from any of
630                  * heads[0..dst], push it in.
631                  */
632                 int i;
633                 for (i = 0; i < dst; i++) {
634                         if (!strcmp(heads[i], heads[src]))
635                                 break;
636                 }
637                 if (i < dst)
638                         continue;
639                 if (src != dst)
640                         heads[dst] = heads[src];
641                 dst++;
642         }
643         heads[dst] = 0;
644         return dst;
645 }
646
647 static int fetch_pack_config(const char *var, const char *value)
648 {
649         if (strcmp(var, "fetch.unpacklimit") == 0) {
650                 fetch_unpack_limit = git_config_int(var, value);
651                 return 0;
652         }
653
654         if (strcmp(var, "transfer.unpacklimit") == 0) {
655                 transfer_unpack_limit = git_config_int(var, value);
656                 return 0;
657         }
658
659         return git_default_config(var, value);
660 }
661
662 static struct lock_file lock;
663
664 int main(int argc, char **argv)
665 {
666         int i, ret, nr_heads;
667         char *dest = NULL, **heads;
668         int fd[2];
669         pid_t pid;
670         struct stat st;
671
672         setup_git_directory();
673         git_config(fetch_pack_config);
674
675         if (0 <= transfer_unpack_limit)
676                 unpack_limit = transfer_unpack_limit;
677         else if (0 <= fetch_unpack_limit)
678                 unpack_limit = fetch_unpack_limit;
679
680         nr_heads = 0;
681         heads = NULL;
682         for (i = 1; i < argc; i++) {
683                 char *arg = argv[i];
684
685                 if (*arg == '-') {
686                         if (!prefixcmp(arg, "--upload-pack=")) {
687                                 uploadpack = arg + 14;
688                                 continue;
689                         }
690                         if (!prefixcmp(arg, "--exec=")) {
691                                 uploadpack = arg + 7;
692                                 continue;
693                         }
694                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
695                                 quiet = 1;
696                                 continue;
697                         }
698                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
699                                 keep_pack++;
700                                 unpack_limit = 0;
701                                 continue;
702                         }
703                         if (!strcmp("--thin", arg)) {
704                                 use_thin_pack = 1;
705                                 continue;
706                         }
707                         if (!strcmp("--all", arg)) {
708                                 fetch_all = 1;
709                                 continue;
710                         }
711                         if (!strcmp("-v", arg)) {
712                                 verbose = 1;
713                                 continue;
714                         }
715                         if (!prefixcmp(arg, "--depth=")) {
716                                 depth = strtol(arg + 8, NULL, 0);
717                                 if (stat(git_path("shallow"), &st))
718                                         st.st_mtime = 0;
719                                 continue;
720                         }
721                         usage(fetch_pack_usage);
722                 }
723                 dest = arg;
724                 heads = argv + i + 1;
725                 nr_heads = argc - i - 1;
726                 break;
727         }
728         if (!dest)
729                 usage(fetch_pack_usage);
730         pid = git_connect(fd, dest, uploadpack);
731         if (pid < 0)
732                 return 1;
733         if (heads && nr_heads)
734                 nr_heads = remove_duplicates(nr_heads, heads);
735         ret = fetch_pack(fd, nr_heads, heads);
736         close(fd[0]);
737         close(fd[1]);
738         ret |= finish_connect(pid);
739
740         if (!ret && nr_heads) {
741                 /* If the heads to pull were given, we should have
742                  * consumed all of them by matching the remote.
743                  * Otherwise, 'git-fetch remote no-such-ref' would
744                  * silently succeed without issuing an error.
745                  */
746                 for (i = 0; i < nr_heads; i++)
747                         if (heads[i] && heads[i][0]) {
748                                 error("no such remote ref %s", heads[i]);
749                                 ret = 1;
750                         }
751         }
752
753         if (!ret && depth > 0) {
754                 struct cache_time mtime;
755                 char *shallow = git_path("shallow");
756                 int fd;
757
758                 mtime.sec = st.st_mtime;
759 #ifdef USE_NSEC
760                 mtime.usec = st.st_mtim.usec;
761 #endif
762                 if (stat(shallow, &st)) {
763                         if (mtime.sec)
764                                 die("shallow file was removed during fetch");
765                 } else if (st.st_mtime != mtime.sec
766 #ifdef USE_NSEC
767                                 || st.st_mtim.usec != mtime.usec
768 #endif
769                           )
770                         die("shallow file was changed during fetch");
771
772                 fd = hold_lock_file_for_update(&lock, shallow, 1);
773                 if (!write_shallow_commits(fd, 0)) {
774                         unlink(shallow);
775                         rollback_lock_file(&lock);
776                 } else {
777                         close(fd);
778                         commit_lock_file(&lock);
779                 }
780         }
781
782         return !!ret;
783 }