Create pack_report() as a debugging aid.
[git] / fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "sideband.h"
8
9 static int keep_pack;
10 static int quiet;
11 static int verbose;
12 static int fetch_all;
13 static int depth;
14 static const char fetch_pack_usage[] =
15 "git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [--depth=<n>] [host:]directory <refs>...";
16 static const char *exec = "git-upload-pack";
17
18 #define COMPLETE        (1U << 0)
19 #define COMMON          (1U << 1)
20 #define COMMON_REF      (1U << 2)
21 #define SEEN            (1U << 3)
22 #define POPPED          (1U << 4)
23
24 /*
25  * After sending this many "have"s if we do not get any new ACK , we
26  * give up traversing our history.
27  */
28 #define MAX_IN_VAIN 256
29
30 static struct commit_list *rev_list;
31 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
32
33 static void rev_list_push(struct commit *commit, int mark)
34 {
35         if (!(commit->object.flags & mark)) {
36                 commit->object.flags |= mark;
37
38                 if (!(commit->object.parsed))
39                         parse_commit(commit);
40
41                 insert_by_date(commit, &rev_list);
42
43                 if (!(commit->object.flags & COMMON))
44                         non_common_revs++;
45         }
46 }
47
48 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
49 {
50         struct object *o = deref_tag(parse_object(sha1), path, 0);
51
52         if (o && o->type == OBJ_COMMIT)
53                 rev_list_push((struct commit *)o, SEEN);
54
55         return 0;
56 }
57
58 /*
59    This function marks a rev and its ancestors as common.
60    In some cases, it is desirable to mark only the ancestors (for example
61    when only the server does not yet know that they are common).
62 */
63
64 static void mark_common(struct commit *commit,
65                 int ancestors_only, int dont_parse)
66 {
67         if (commit != NULL && !(commit->object.flags & COMMON)) {
68                 struct object *o = (struct object *)commit;
69
70                 if (!ancestors_only)
71                         o->flags |= COMMON;
72
73                 if (!(o->flags & SEEN))
74                         rev_list_push(commit, SEEN);
75                 else {
76                         struct commit_list *parents;
77
78                         if (!ancestors_only && !(o->flags & POPPED))
79                                 non_common_revs--;
80                         if (!o->parsed && !dont_parse)
81                                 parse_commit(commit);
82
83                         for (parents = commit->parents;
84                                         parents;
85                                         parents = parents->next)
86                                 mark_common(parents->item, 0, dont_parse);
87                 }
88         }
89 }
90
91 /*
92   Get the next rev to send, ignoring the common.
93 */
94
95 static const unsigned char* get_rev(void)
96 {
97         struct commit *commit = NULL;
98
99         while (commit == NULL) {
100                 unsigned int mark;
101                 struct commit_list* parents;
102
103                 if (rev_list == NULL || non_common_revs == 0)
104                         return NULL;
105
106                 commit = rev_list->item;
107                 if (!(commit->object.parsed))
108                         parse_commit(commit);
109                 commit->object.flags |= POPPED;
110                 if (!(commit->object.flags & COMMON))
111                         non_common_revs--;
112         
113                 parents = commit->parents;
114
115                 if (commit->object.flags & COMMON) {
116                         /* do not send "have", and ignore ancestors */
117                         commit = NULL;
118                         mark = COMMON | SEEN;
119                 } else if (commit->object.flags & COMMON_REF)
120                         /* send "have", and ignore ancestors */
121                         mark = COMMON | SEEN;
122                 else
123                         /* send "have", also for its ancestors */
124                         mark = SEEN;
125
126                 while (parents) {
127                         if (!(parents->item->object.flags & SEEN))
128                                 rev_list_push(parents->item, mark);
129                         if (mark & COMMON)
130                                 mark_common(parents->item, 1, 0);
131                         parents = parents->next;
132                 }
133
134                 rev_list = rev_list->next;
135         }
136
137         return commit->object.sha1;
138 }
139
140 static int find_common(int fd[2], unsigned char *result_sha1,
141                        struct ref *refs)
142 {
143         int fetching;
144         int count = 0, flushes = 0, retval;
145         const unsigned char *sha1;
146         unsigned in_vain = 0;
147         int got_continue = 0;
148
149         for_each_ref(rev_list_insert_ref, NULL);
150
151         fetching = 0;
152         for ( ; refs ; refs = refs->next) {
153                 unsigned char *remote = refs->old_sha1;
154                 struct object *o;
155
156                 /*
157                  * If that object is complete (i.e. it is an ancestor of a
158                  * local ref), we tell them we have it but do not have to
159                  * tell them about its ancestors, which they already know
160                  * about.
161                  *
162                  * We use lookup_object here because we are only
163                  * interested in the case we *know* the object is
164                  * reachable and we have already scanned it.
165                  */
166                 if (((o = lookup_object(remote)) != NULL) &&
167                                 (o->flags & COMPLETE)) {
168                         continue;
169                 }
170
171                 if (!fetching)
172                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
173                                      sha1_to_hex(remote),
174                                      (multi_ack ? " multi_ack" : ""),
175                                      (use_sideband == 2 ? " side-band-64k" : ""),
176                                      (use_sideband == 1 ? " side-band" : ""),
177                                      (use_thin_pack ? " thin-pack" : ""),
178                                      " ofs-delta");
179                 else
180                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
181                 fetching++;
182         }
183         if (is_repository_shallow())
184                 write_shallow_commits(fd[1], 1);
185         if (depth > 0)
186                 packet_write(fd[1], "deepen %d", depth);
187         packet_flush(fd[1]);
188         if (!fetching)
189                 return 1;
190
191         if (depth > 0) {
192                 char line[1024];
193                 unsigned char sha1[20];
194                 int len;
195
196                 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
197                         if (!strncmp("shallow ", line, 8)) {
198                                 if (get_sha1_hex(line + 8, sha1))
199                                         die("invalid shallow line: %s", line);
200                                 register_shallow(sha1);
201                                 continue;
202                         }
203                         if (!strncmp("unshallow ", line, 10)) {
204                                 if (get_sha1_hex(line + 10, sha1))
205                                         die("invalid unshallow line: %s", line);
206                                 if (!lookup_object(sha1))
207                                         die("object not found: %s", line);
208                                 /* make sure that it is parsed as shallow */
209                                 parse_object(sha1);
210                                 if (unregister_shallow(sha1))
211                                         die("no shallow found: %s", line);
212                                 continue;
213                         }
214                         die("expected shallow/unshallow, got %s", line);
215                 }
216         }
217
218         flushes = 0;
219         retval = -1;
220         while ((sha1 = get_rev())) {
221                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
222                 if (verbose)
223                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
224                 in_vain++;
225                 if (!(31 & ++count)) {
226                         int ack;
227
228                         packet_flush(fd[1]);
229                         flushes++;
230
231                         /*
232                          * We keep one window "ahead" of the other side, and
233                          * will wait for an ACK only on the next one
234                          */
235                         if (count == 32)
236                                 continue;
237
238                         do {
239                                 ack = get_ack(fd[0], result_sha1);
240                                 if (verbose && ack)
241                                         fprintf(stderr, "got ack %d %s\n", ack,
242                                                         sha1_to_hex(result_sha1));
243                                 if (ack == 1) {
244                                         flushes = 0;
245                                         multi_ack = 0;
246                                         retval = 0;
247                                         goto done;
248                                 } else if (ack == 2) {
249                                         struct commit *commit =
250                                                 lookup_commit(result_sha1);
251                                         mark_common(commit, 0, 1);
252                                         retval = 0;
253                                         in_vain = 0;
254                                         got_continue = 1;
255                                 }
256                         } while (ack);
257                         flushes--;
258                         if (got_continue && MAX_IN_VAIN < in_vain) {
259                                 if (verbose)
260                                         fprintf(stderr, "giving up\n");
261                                 break; /* give up */
262                         }
263                 }
264         }
265 done:
266         packet_write(fd[1], "done\n");
267         if (verbose)
268                 fprintf(stderr, "done\n");
269         if (retval != 0) {
270                 multi_ack = 0;
271                 flushes++;
272         }
273         while (flushes || multi_ack) {
274                 int ack = get_ack(fd[0], result_sha1);
275                 if (ack) {
276                         if (verbose)
277                                 fprintf(stderr, "got ack (%d) %s\n", ack,
278                                         sha1_to_hex(result_sha1));
279                         if (ack == 1)
280                                 return 0;
281                         multi_ack = 1;
282                         continue;
283                 }
284                 flushes--;
285         }
286         return retval;
287 }
288
289 static struct commit_list *complete;
290
291 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
292 {
293         struct object *o = parse_object(sha1);
294
295         while (o && o->type == OBJ_TAG) {
296                 struct tag *t = (struct tag *) o;
297                 if (!t->tagged)
298                         break; /* broken repository */
299                 o->flags |= COMPLETE;
300                 o = parse_object(t->tagged->sha1);
301         }
302         if (o && o->type == OBJ_COMMIT) {
303                 struct commit *commit = (struct commit *)o;
304                 commit->object.flags |= COMPLETE;
305                 insert_by_date(commit, &complete);
306         }
307         return 0;
308 }
309
310 static void mark_recent_complete_commits(unsigned long cutoff)
311 {
312         while (complete && cutoff <= complete->item->date) {
313                 if (verbose)
314                         fprintf(stderr, "Marking %s as complete\n",
315                                 sha1_to_hex(complete->item->object.sha1));
316                 pop_most_recent_commit(&complete, COMPLETE);
317         }
318 }
319
320 static void filter_refs(struct ref **refs, int nr_match, char **match)
321 {
322         struct ref **return_refs;
323         struct ref *newlist = NULL;
324         struct ref **newtail = &newlist;
325         struct ref *ref, *next;
326         struct ref *fastarray[32];
327
328         if (nr_match && !fetch_all) {
329                 if (ARRAY_SIZE(fastarray) < nr_match)
330                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
331                 else {
332                         return_refs = fastarray;
333                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
334                 }
335         }
336         else
337                 return_refs = NULL;
338
339         for (ref = *refs; ref; ref = next) {
340                 next = ref->next;
341                 if (!memcmp(ref->name, "refs/", 5) &&
342                     check_ref_format(ref->name + 5))
343                         ; /* trash */
344                 else if (fetch_all &&
345                          (!depth || strncmp(ref->name, "refs/tags/", 10) )) {
346                         *newtail = ref;
347                         ref->next = NULL;
348                         newtail = &ref->next;
349                         continue;
350                 }
351                 else {
352                         int order = path_match(ref->name, nr_match, match);
353                         if (order) {
354                                 return_refs[order-1] = ref;
355                                 continue; /* we will link it later */
356                         }
357                 }
358                 free(ref);
359         }
360
361         if (!fetch_all) {
362                 int i;
363                 for (i = 0; i < nr_match; i++) {
364                         ref = return_refs[i];
365                         if (ref) {
366                                 *newtail = ref;
367                                 ref->next = NULL;
368                                 newtail = &ref->next;
369                         }
370                 }
371                 if (return_refs != fastarray)
372                         free(return_refs);
373         }
374         *refs = newlist;
375 }
376
377 static int everything_local(struct ref **refs, int nr_match, char **match)
378 {
379         struct ref *ref;
380         int retval;
381         unsigned long cutoff = 0;
382
383         track_object_refs = 0;
384         save_commit_buffer = 0;
385
386         for (ref = *refs; ref; ref = ref->next) {
387                 struct object *o;
388
389                 o = parse_object(ref->old_sha1);
390                 if (!o)
391                         continue;
392
393                 /* We already have it -- which may mean that we were
394                  * in sync with the other side at some time after
395                  * that (it is OK if we guess wrong here).
396                  */
397                 if (o->type == OBJ_COMMIT) {
398                         struct commit *commit = (struct commit *)o;
399                         if (!cutoff || cutoff < commit->date)
400                                 cutoff = commit->date;
401                 }
402         }
403
404         if (!depth) {
405                 for_each_ref(mark_complete, NULL);
406                 if (cutoff)
407                         mark_recent_complete_commits(cutoff);
408         }
409
410         /*
411          * Mark all complete remote refs as common refs.
412          * Don't mark them common yet; the server has to be told so first.
413          */
414         for (ref = *refs; ref; ref = ref->next) {
415                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
416                                              NULL, 0);
417
418                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
419                         continue;
420
421                 if (!(o->flags & SEEN)) {
422                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
423
424                         mark_common((struct commit *)o, 1, 1);
425                 }
426         }
427
428         filter_refs(refs, nr_match, match);
429
430         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
431                 const unsigned char *remote = ref->old_sha1;
432                 unsigned char local[20];
433                 struct object *o;
434
435                 o = lookup_object(remote);
436                 if (!o || !(o->flags & COMPLETE)) {
437                         retval = 0;
438                         if (!verbose)
439                                 continue;
440                         fprintf(stderr,
441                                 "want %s (%s)\n", sha1_to_hex(remote),
442                                 ref->name);
443                         continue;
444                 }
445
446                 hashcpy(ref->new_sha1, local);
447                 if (!verbose)
448                         continue;
449                 fprintf(stderr,
450                         "already have %s (%s)\n", sha1_to_hex(remote),
451                         ref->name);
452         }
453         return retval;
454 }
455
456 static pid_t setup_sideband(int fd[2], int xd[2])
457 {
458         pid_t side_pid;
459
460         if (!use_sideband) {
461                 fd[0] = xd[0];
462                 fd[1] = xd[1];
463                 return 0;
464         }
465         /* xd[] is talking with upload-pack; subprocess reads from
466          * xd[0], spits out band#2 to stderr, and feeds us band#1
467          * through our fd[0].
468          */
469         if (pipe(fd) < 0)
470                 die("fetch-pack: unable to set up pipe");
471         side_pid = fork();
472         if (side_pid < 0)
473                 die("fetch-pack: unable to fork off sideband demultiplexer");
474         if (!side_pid) {
475                 /* subprocess */
476                 close(fd[0]);
477                 if (xd[0] != xd[1])
478                         close(xd[1]);
479                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
480                         exit(1);
481                 exit(0);
482         }
483         close(xd[0]);
484         close(fd[1]);
485         fd[1] = xd[1];
486         return side_pid;
487 }
488
489 static int get_pack(int xd[2], const char **argv)
490 {
491         int status;
492         pid_t pid, side_pid;
493         int fd[2];
494
495         side_pid = setup_sideband(fd, xd);
496         pid = fork();
497         if (pid < 0)
498                 die("fetch-pack: unable to fork off %s", argv[0]);
499         if (!pid) {
500                 dup2(fd[0], 0);
501                 close(fd[0]);
502                 close(fd[1]);
503                 execv_git_cmd(argv);
504                 die("%s exec failed", argv[0]);
505         }
506         close(fd[0]);
507         close(fd[1]);
508         while (waitpid(pid, &status, 0) < 0) {
509                 if (errno != EINTR)
510                         die("waiting for %s: %s", argv[0], strerror(errno));
511         }
512         if (WIFEXITED(status)) {
513                 int code = WEXITSTATUS(status);
514                 if (code)
515                         die("%s died with error code %d", argv[0], code);
516                 return 0;
517         }
518         if (WIFSIGNALED(status)) {
519                 int sig = WTERMSIG(status);
520                 die("%s died of signal %d", argv[0], sig);
521         }
522         die("%s died of unnatural causes %d", argv[0], status);
523 }
524
525 static int explode_rx_pack(int xd[2])
526 {
527         const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
528         return get_pack(xd, argv);
529 }
530
531 static int keep_rx_pack(int xd[2])
532 {
533         const char *argv[6];
534         char keep_arg[256];
535         int n = 0;
536
537         argv[n++] = "index-pack";
538         argv[n++] = "--stdin";
539         if (!quiet)
540                 argv[n++] = "-v";
541         if (use_thin_pack)
542                 argv[n++] = "--fix-thin";
543         if (keep_pack > 1) {
544                 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
545                 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
546                         strcpy(keep_arg + s, "localhost");
547                 argv[n++] = keep_arg;
548         }
549         argv[n] = NULL;
550         return get_pack(xd, argv);
551 }
552
553 static int fetch_pack(int fd[2], int nr_match, char **match)
554 {
555         struct ref *ref;
556         unsigned char sha1[20];
557         int status;
558
559         get_remote_heads(fd[0], &ref, 0, NULL, 0);
560         if (is_repository_shallow() && !server_supports("shallow"))
561                 die("Server does not support shallow clients");
562         if (server_supports("multi_ack")) {
563                 if (verbose)
564                         fprintf(stderr, "Server supports multi_ack\n");
565                 multi_ack = 1;
566         }
567         if (server_supports("side-band-64k")) {
568                 if (verbose)
569                         fprintf(stderr, "Server supports side-band-64k\n");
570                 use_sideband = 2;
571         }
572         else if (server_supports("side-band")) {
573                 if (verbose)
574                         fprintf(stderr, "Server supports side-band\n");
575                 use_sideband = 1;
576         }
577         if (!ref) {
578                 packet_flush(fd[1]);
579                 die("no matching remote head");
580         }
581         if (everything_local(&ref, nr_match, match)) {
582                 packet_flush(fd[1]);
583                 goto all_done;
584         }
585         if (find_common(fd, sha1, ref) < 0)
586                 if (keep_pack != 1)
587                         /* When cloning, it is not unusual to have
588                          * no common commit.
589                          */
590                         fprintf(stderr, "warning: no common commits\n");
591
592         status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
593         if (status)
594                 die("git-fetch-pack: fetch failed.");
595
596  all_done:
597         while (ref) {
598                 printf("%s %s\n",
599                        sha1_to_hex(ref->old_sha1), ref->name);
600                 ref = ref->next;
601         }
602         return 0;
603 }
604
605 static int remove_duplicates(int nr_heads, char **heads)
606 {
607         int src, dst;
608
609         for (src = dst = 0; src < nr_heads; src++) {
610                 /* If heads[src] is different from any of
611                  * heads[0..dst], push it in.
612                  */
613                 int i;
614                 for (i = 0; i < dst; i++) {
615                         if (!strcmp(heads[i], heads[src]))
616                                 break;
617                 }
618                 if (i < dst)
619                         continue;
620                 if (src != dst)
621                         heads[dst] = heads[src];
622                 dst++;
623         }
624         heads[dst] = 0;
625         return dst;
626 }
627
628 int main(int argc, char **argv)
629 {
630         int i, ret, nr_heads;
631         char *dest = NULL, **heads;
632         int fd[2];
633         pid_t pid;
634         struct stat st;
635         struct lock_file lock;
636
637         setup_git_directory();
638
639         nr_heads = 0;
640         heads = NULL;
641         for (i = 1; i < argc; i++) {
642                 char *arg = argv[i];
643
644                 if (*arg == '-') {
645                         if (!strncmp("--exec=", arg, 7)) {
646                                 exec = arg + 7;
647                                 continue;
648                         }
649                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
650                                 quiet = 1;
651                                 continue;
652                         }
653                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
654                                 keep_pack++;
655                                 continue;
656                         }
657                         if (!strcmp("--thin", arg)) {
658                                 use_thin_pack = 1;
659                                 continue;
660                         }
661                         if (!strcmp("--all", arg)) {
662                                 fetch_all = 1;
663                                 continue;
664                         }
665                         if (!strcmp("-v", arg)) {
666                                 verbose = 1;
667                                 continue;
668                         }
669                         if (!strncmp("--depth=", arg, 8)) {
670                                 depth = strtol(arg + 8, NULL, 0);
671                                 if (stat(git_path("shallow"), &st))
672                                         st.st_mtime = 0;
673                                 continue;
674                         }
675                         usage(fetch_pack_usage);
676                 }
677                 dest = arg;
678                 heads = argv + i + 1;
679                 nr_heads = argc - i - 1;
680                 break;
681         }
682         if (!dest)
683                 usage(fetch_pack_usage);
684         pid = git_connect(fd, dest, exec);
685         if (pid < 0)
686                 return 1;
687         if (heads && nr_heads)
688                 nr_heads = remove_duplicates(nr_heads, heads);
689         ret = fetch_pack(fd, nr_heads, heads);
690         close(fd[0]);
691         close(fd[1]);
692         ret |= finish_connect(pid);
693
694         if (!ret && nr_heads) {
695                 /* If the heads to pull were given, we should have
696                  * consumed all of them by matching the remote.
697                  * Otherwise, 'git-fetch remote no-such-ref' would
698                  * silently succeed without issuing an error.
699                  */
700                 for (i = 0; i < nr_heads; i++)
701                         if (heads[i] && heads[i][0]) {
702                                 error("no such remote ref %s", heads[i]);
703                                 ret = 1;
704                         }
705         }
706
707         if (!ret && depth > 0) {
708                 struct cache_time mtime;
709                 char *shallow = git_path("shallow");
710                 int fd;
711
712                 mtime.sec = st.st_mtime;
713 #ifdef USE_NSEC
714                 mtime.usec = st.st_mtim.usec;
715 #endif
716                 if (stat(shallow, &st)) {
717                         if (mtime.sec)
718                                 die("shallow file was removed during fetch");
719                 } else if (st.st_mtime != mtime.sec
720 #ifdef USE_NSEC
721                                 || st.st_mtim.usec != mtime.usec
722 #endif
723                           )
724                         die("shallow file was changed during fetch");
725
726                 fd = hold_lock_file_for_update(&lock, shallow, 1);
727                 if (!write_shallow_commits(fd, 0)) {
728                         unlink(shallow);
729                         rollback_lock_file(&lock);
730                 } else {
731                         close(fd);
732                         commit_lock_file(&lock);
733                 }
734         }
735
736         return !!ret;
737 }