Merge branch 'maint'
[git] / fetch-pack.c
1 #include "cache.h"
2 #include "refs.h"
3 #include "pkt-line.h"
4 #include "commit.h"
5 #include "tag.h"
6 #include "exec_cmd.h"
7 #include "pack.h"
8 #include "sideband.h"
9 #include "fetch-pack.h"
10 #include "remote.h"
11 #include "run-command.h"
12 #include "connect.h"
13 #include "transport.h"
14 #include "version.h"
15 #include "prio-queue.h"
16 #include "sha1-array.h"
17
18 static int transfer_unpack_limit = -1;
19 static int fetch_unpack_limit = -1;
20 static int unpack_limit = 100;
21 static int prefer_ofs_delta = 1;
22 static int no_done;
23 static int fetch_fsck_objects = -1;
24 static int transfer_fsck_objects = -1;
25 static int agent_supported;
26 static struct lock_file shallow_lock;
27 static const char *alternate_shallow_file;
28
29 /* Remember to update object flag allocation in object.h */
30 #define COMPLETE        (1U << 0)
31 #define COMMON          (1U << 1)
32 #define COMMON_REF      (1U << 2)
33 #define SEEN            (1U << 3)
34 #define POPPED          (1U << 4)
35
36 static int marked;
37
38 /*
39  * After sending this many "have"s if we do not get any new ACK , we
40  * give up traversing our history.
41  */
42 #define MAX_IN_VAIN 256
43
44 static struct prio_queue rev_list = { compare_commits_by_commit_date };
45 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
46
47 static void rev_list_push(struct commit *commit, int mark)
48 {
49         if (!(commit->object.flags & mark)) {
50                 commit->object.flags |= mark;
51
52                 if (parse_commit(commit))
53                         return;
54
55                 prio_queue_put(&rev_list, commit);
56
57                 if (!(commit->object.flags & COMMON))
58                         non_common_revs++;
59         }
60 }
61
62 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
63 {
64         struct object *o = deref_tag(parse_object(sha1), refname, 0);
65
66         if (o && o->type == OBJ_COMMIT)
67                 rev_list_push((struct commit *)o, SEEN);
68
69         return 0;
70 }
71
72 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
73 {
74         struct object *o = deref_tag(parse_object(sha1), refname, 0);
75
76         if (o && o->type == OBJ_COMMIT)
77                 clear_commit_marks((struct commit *)o,
78                                    COMMON | COMMON_REF | SEEN | POPPED);
79         return 0;
80 }
81
82 /*
83    This function marks a rev and its ancestors as common.
84    In some cases, it is desirable to mark only the ancestors (for example
85    when only the server does not yet know that they are common).
86 */
87
88 static void mark_common(struct commit *commit,
89                 int ancestors_only, int dont_parse)
90 {
91         if (commit != NULL && !(commit->object.flags & COMMON)) {
92                 struct object *o = (struct object *)commit;
93
94                 if (!ancestors_only)
95                         o->flags |= COMMON;
96
97                 if (!(o->flags & SEEN))
98                         rev_list_push(commit, SEEN);
99                 else {
100                         struct commit_list *parents;
101
102                         if (!ancestors_only && !(o->flags & POPPED))
103                                 non_common_revs--;
104                         if (!o->parsed && !dont_parse)
105                                 if (parse_commit(commit))
106                                         return;
107
108                         for (parents = commit->parents;
109                                         parents;
110                                         parents = parents->next)
111                                 mark_common(parents->item, 0, dont_parse);
112                 }
113         }
114 }
115
116 /*
117   Get the next rev to send, ignoring the common.
118 */
119
120 static const unsigned char *get_rev(void)
121 {
122         struct commit *commit = NULL;
123
124         while (commit == NULL) {
125                 unsigned int mark;
126                 struct commit_list *parents;
127
128                 if (rev_list.nr == 0 || non_common_revs == 0)
129                         return NULL;
130
131                 commit = prio_queue_get(&rev_list);
132                 parse_commit(commit);
133                 parents = commit->parents;
134
135                 commit->object.flags |= POPPED;
136                 if (!(commit->object.flags & COMMON))
137                         non_common_revs--;
138
139                 if (commit->object.flags & COMMON) {
140                         /* do not send "have", and ignore ancestors */
141                         commit = NULL;
142                         mark = COMMON | SEEN;
143                 } else if (commit->object.flags & COMMON_REF)
144                         /* send "have", and ignore ancestors */
145                         mark = COMMON | SEEN;
146                 else
147                         /* send "have", also for its ancestors */
148                         mark = SEEN;
149
150                 while (parents) {
151                         if (!(parents->item->object.flags & SEEN))
152                                 rev_list_push(parents->item, mark);
153                         if (mark & COMMON)
154                                 mark_common(parents->item, 1, 0);
155                         parents = parents->next;
156                 }
157         }
158
159         return commit->object.sha1;
160 }
161
162 enum ack_type {
163         NAK = 0,
164         ACK,
165         ACK_continue,
166         ACK_common,
167         ACK_ready
168 };
169
170 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
171 {
172         if (args->stateless_rpc && args->depth > 0) {
173                 /* If we sent a depth we will get back "duplicate"
174                  * shallow and unshallow commands every time there
175                  * is a block of have lines exchanged.
176                  */
177                 char *line;
178                 while ((line = packet_read_line(fd, NULL))) {
179                         if (starts_with(line, "shallow "))
180                                 continue;
181                         if (starts_with(line, "unshallow "))
182                                 continue;
183                         die("git fetch-pack: expected shallow list");
184                 }
185         }
186 }
187
188 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
189 {
190         int len;
191         char *line = packet_read_line(fd, &len);
192         const char *arg;
193
194         if (!len)
195                 die("git fetch-pack: expected ACK/NAK, got EOF");
196         if (!strcmp(line, "NAK"))
197                 return NAK;
198         if (skip_prefix(line, "ACK ", &arg)) {
199                 if (!get_sha1_hex(arg, result_sha1)) {
200                         arg += 40;
201                         len -= arg - line;
202                         if (len < 1)
203                                 return ACK;
204                         if (strstr(arg, "continue"))
205                                 return ACK_continue;
206                         if (strstr(arg, "common"))
207                                 return ACK_common;
208                         if (strstr(arg, "ready"))
209                                 return ACK_ready;
210                         return ACK;
211                 }
212         }
213         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
214 }
215
216 static void send_request(struct fetch_pack_args *args,
217                          int fd, struct strbuf *buf)
218 {
219         if (args->stateless_rpc) {
220                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
221                 packet_flush(fd);
222         } else
223                 write_or_die(fd, buf->buf, buf->len);
224 }
225
226 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
227 {
228         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
229 }
230
231 #define INITIAL_FLUSH 16
232 #define PIPESAFE_FLUSH 32
233 #define LARGE_FLUSH 1024
234
235 static int next_flush(struct fetch_pack_args *args, int count)
236 {
237         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
238
239         if (count < flush_limit)
240                 count <<= 1;
241         else
242                 count += flush_limit;
243         return count;
244 }
245
246 static int find_common(struct fetch_pack_args *args,
247                        int fd[2], unsigned char *result_sha1,
248                        struct ref *refs)
249 {
250         int fetching;
251         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
252         const unsigned char *sha1;
253         unsigned in_vain = 0;
254         int got_continue = 0;
255         int got_ready = 0;
256         struct strbuf req_buf = STRBUF_INIT;
257         size_t state_len = 0;
258
259         if (args->stateless_rpc && multi_ack == 1)
260                 die("--stateless-rpc requires multi_ack_detailed");
261         if (marked)
262                 for_each_ref(clear_marks, NULL);
263         marked = 1;
264
265         for_each_ref(rev_list_insert_ref, NULL);
266         for_each_alternate_ref(insert_one_alternate_ref, NULL);
267
268         fetching = 0;
269         for ( ; refs ; refs = refs->next) {
270                 unsigned char *remote = refs->old_sha1;
271                 const char *remote_hex;
272                 struct object *o;
273
274                 /*
275                  * If that object is complete (i.e. it is an ancestor of a
276                  * local ref), we tell them we have it but do not have to
277                  * tell them about its ancestors, which they already know
278                  * about.
279                  *
280                  * We use lookup_object here because we are only
281                  * interested in the case we *know* the object is
282                  * reachable and we have already scanned it.
283                  */
284                 if (((o = lookup_object(remote)) != NULL) &&
285                                 (o->flags & COMPLETE)) {
286                         continue;
287                 }
288
289                 remote_hex = sha1_to_hex(remote);
290                 if (!fetching) {
291                         struct strbuf c = STRBUF_INIT;
292                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
293                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
294                         if (no_done)            strbuf_addstr(&c, " no-done");
295                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
296                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
297                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
298                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
299                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
300                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
301                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
302                                                             git_user_agent_sanitized());
303                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
304                         strbuf_release(&c);
305                 } else
306                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
307                 fetching++;
308         }
309
310         if (!fetching) {
311                 strbuf_release(&req_buf);
312                 packet_flush(fd[1]);
313                 return 1;
314         }
315
316         if (is_repository_shallow())
317                 write_shallow_commits(&req_buf, 1, NULL);
318         if (args->depth > 0)
319                 packet_buf_write(&req_buf, "deepen %d", args->depth);
320         packet_buf_flush(&req_buf);
321         state_len = req_buf.len;
322
323         if (args->depth > 0) {
324                 char *line;
325                 const char *arg;
326                 unsigned char sha1[20];
327
328                 send_request(args, fd[1], &req_buf);
329                 while ((line = packet_read_line(fd[0], NULL))) {
330                         if (skip_prefix(line, "shallow ", &arg)) {
331                                 if (get_sha1_hex(arg, sha1))
332                                         die("invalid shallow line: %s", line);
333                                 register_shallow(sha1);
334                                 continue;
335                         }
336                         if (skip_prefix(line, "unshallow ", &arg)) {
337                                 if (get_sha1_hex(arg, sha1))
338                                         die("invalid unshallow line: %s", line);
339                                 if (!lookup_object(sha1))
340                                         die("object not found: %s", line);
341                                 /* make sure that it is parsed as shallow */
342                                 if (!parse_object(sha1))
343                                         die("error in object: %s", line);
344                                 if (unregister_shallow(sha1))
345                                         die("no shallow found: %s", line);
346                                 continue;
347                         }
348                         die("expected shallow/unshallow, got %s", line);
349                 }
350         } else if (!args->stateless_rpc)
351                 send_request(args, fd[1], &req_buf);
352
353         if (!args->stateless_rpc) {
354                 /* If we aren't using the stateless-rpc interface
355                  * we don't need to retain the headers.
356                  */
357                 strbuf_setlen(&req_buf, 0);
358                 state_len = 0;
359         }
360
361         flushes = 0;
362         retval = -1;
363         while ((sha1 = get_rev())) {
364                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
365                 if (args->verbose)
366                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
367                 in_vain++;
368                 if (flush_at <= ++count) {
369                         int ack;
370
371                         packet_buf_flush(&req_buf);
372                         send_request(args, fd[1], &req_buf);
373                         strbuf_setlen(&req_buf, state_len);
374                         flushes++;
375                         flush_at = next_flush(args, count);
376
377                         /*
378                          * We keep one window "ahead" of the other side, and
379                          * will wait for an ACK only on the next one
380                          */
381                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
382                                 continue;
383
384                         consume_shallow_list(args, fd[0]);
385                         do {
386                                 ack = get_ack(fd[0], result_sha1);
387                                 if (args->verbose && ack)
388                                         fprintf(stderr, "got ack %d %s\n", ack,
389                                                         sha1_to_hex(result_sha1));
390                                 switch (ack) {
391                                 case ACK:
392                                         flushes = 0;
393                                         multi_ack = 0;
394                                         retval = 0;
395                                         goto done;
396                                 case ACK_common:
397                                 case ACK_ready:
398                                 case ACK_continue: {
399                                         struct commit *commit =
400                                                 lookup_commit(result_sha1);
401                                         if (!commit)
402                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
403                                         if (args->stateless_rpc
404                                          && ack == ACK_common
405                                          && !(commit->object.flags & COMMON)) {
406                                                 /* We need to replay the have for this object
407                                                  * on the next RPC request so the peer knows
408                                                  * it is in common with us.
409                                                  */
410                                                 const char *hex = sha1_to_hex(result_sha1);
411                                                 packet_buf_write(&req_buf, "have %s\n", hex);
412                                                 state_len = req_buf.len;
413                                         }
414                                         mark_common(commit, 0, 1);
415                                         retval = 0;
416                                         in_vain = 0;
417                                         got_continue = 1;
418                                         if (ack == ACK_ready) {
419                                                 clear_prio_queue(&rev_list);
420                                                 got_ready = 1;
421                                         }
422                                         break;
423                                         }
424                                 }
425                         } while (ack);
426                         flushes--;
427                         if (got_continue && MAX_IN_VAIN < in_vain) {
428                                 if (args->verbose)
429                                         fprintf(stderr, "giving up\n");
430                                 break; /* give up */
431                         }
432                 }
433         }
434 done:
435         if (!got_ready || !no_done) {
436                 packet_buf_write(&req_buf, "done\n");
437                 send_request(args, fd[1], &req_buf);
438         }
439         if (args->verbose)
440                 fprintf(stderr, "done\n");
441         if (retval != 0) {
442                 multi_ack = 0;
443                 flushes++;
444         }
445         strbuf_release(&req_buf);
446
447         if (!got_ready || !no_done)
448                 consume_shallow_list(args, fd[0]);
449         while (flushes || multi_ack) {
450                 int ack = get_ack(fd[0], result_sha1);
451                 if (ack) {
452                         if (args->verbose)
453                                 fprintf(stderr, "got ack (%d) %s\n", ack,
454                                         sha1_to_hex(result_sha1));
455                         if (ack == ACK)
456                                 return 0;
457                         multi_ack = 1;
458                         continue;
459                 }
460                 flushes--;
461         }
462         /* it is no error to fetch into a completely empty repo */
463         return count ? retval : 0;
464 }
465
466 static struct commit_list *complete;
467
468 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
469 {
470         struct object *o = parse_object(sha1);
471
472         while (o && o->type == OBJ_TAG) {
473                 struct tag *t = (struct tag *) o;
474                 if (!t->tagged)
475                         break; /* broken repository */
476                 o->flags |= COMPLETE;
477                 o = parse_object(t->tagged->sha1);
478         }
479         if (o && o->type == OBJ_COMMIT) {
480                 struct commit *commit = (struct commit *)o;
481                 if (!(commit->object.flags & COMPLETE)) {
482                         commit->object.flags |= COMPLETE;
483                         commit_list_insert(commit, &complete);
484                 }
485         }
486         return 0;
487 }
488
489 static void mark_recent_complete_commits(struct fetch_pack_args *args,
490                                          unsigned long cutoff)
491 {
492         while (complete && cutoff <= complete->item->date) {
493                 if (args->verbose)
494                         fprintf(stderr, "Marking %s as complete\n",
495                                 sha1_to_hex(complete->item->object.sha1));
496                 pop_most_recent_commit(&complete, COMPLETE);
497         }
498 }
499
500 static void filter_refs(struct fetch_pack_args *args,
501                         struct ref **refs,
502                         struct ref **sought, int nr_sought)
503 {
504         struct ref *newlist = NULL;
505         struct ref **newtail = &newlist;
506         struct ref *ref, *next;
507         int i;
508
509         i = 0;
510         for (ref = *refs; ref; ref = next) {
511                 int keep = 0;
512                 next = ref->next;
513
514                 if (starts_with(ref->name, "refs/") &&
515                     check_refname_format(ref->name, 0))
516                         ; /* trash */
517                 else {
518                         while (i < nr_sought) {
519                                 int cmp = strcmp(ref->name, sought[i]->name);
520                                 if (cmp < 0)
521                                         break; /* definitely do not have it */
522                                 else if (cmp == 0) {
523                                         keep = 1; /* definitely have it */
524                                         sought[i]->matched = 1;
525                                 }
526                                 i++;
527                         }
528                 }
529
530                 if (!keep && args->fetch_all &&
531                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
532                         keep = 1;
533
534                 if (keep) {
535                         *newtail = ref;
536                         ref->next = NULL;
537                         newtail = &ref->next;
538                 } else {
539                         free(ref);
540                 }
541         }
542
543         /* Append unmatched requests to the list */
544         if (allow_tip_sha1_in_want) {
545                 for (i = 0; i < nr_sought; i++) {
546                         ref = sought[i];
547                         if (ref->matched)
548                                 continue;
549                         if (get_sha1_hex(ref->name, ref->old_sha1))
550                                 continue;
551
552                         ref->matched = 1;
553                         *newtail = ref;
554                         ref->next = NULL;
555                         newtail = &ref->next;
556                 }
557         }
558         *refs = newlist;
559 }
560
561 static void mark_alternate_complete(const struct ref *ref, void *unused)
562 {
563         mark_complete(NULL, ref->old_sha1, 0, NULL);
564 }
565
566 static int everything_local(struct fetch_pack_args *args,
567                             struct ref **refs,
568                             struct ref **sought, int nr_sought)
569 {
570         struct ref *ref;
571         int retval;
572         unsigned long cutoff = 0;
573
574         save_commit_buffer = 0;
575
576         for (ref = *refs; ref; ref = ref->next) {
577                 struct object *o;
578
579                 if (!has_sha1_file(ref->old_sha1))
580                         continue;
581
582                 o = parse_object(ref->old_sha1);
583                 if (!o)
584                         continue;
585
586                 /* We already have it -- which may mean that we were
587                  * in sync with the other side at some time after
588                  * that (it is OK if we guess wrong here).
589                  */
590                 if (o->type == OBJ_COMMIT) {
591                         struct commit *commit = (struct commit *)o;
592                         if (!cutoff || cutoff < commit->date)
593                                 cutoff = commit->date;
594                 }
595         }
596
597         if (!args->depth) {
598                 for_each_ref(mark_complete, NULL);
599                 for_each_alternate_ref(mark_alternate_complete, NULL);
600                 commit_list_sort_by_date(&complete);
601                 if (cutoff)
602                         mark_recent_complete_commits(args, cutoff);
603         }
604
605         /*
606          * Mark all complete remote refs as common refs.
607          * Don't mark them common yet; the server has to be told so first.
608          */
609         for (ref = *refs; ref; ref = ref->next) {
610                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
611                                              NULL, 0);
612
613                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
614                         continue;
615
616                 if (!(o->flags & SEEN)) {
617                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
618
619                         mark_common((struct commit *)o, 1, 1);
620                 }
621         }
622
623         filter_refs(args, refs, sought, nr_sought);
624
625         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
626                 const unsigned char *remote = ref->old_sha1;
627                 unsigned char local[20];
628                 struct object *o;
629
630                 o = lookup_object(remote);
631                 if (!o || !(o->flags & COMPLETE)) {
632                         retval = 0;
633                         if (!args->verbose)
634                                 continue;
635                         fprintf(stderr,
636                                 "want %s (%s)\n", sha1_to_hex(remote),
637                                 ref->name);
638                         continue;
639                 }
640
641                 hashcpy(ref->new_sha1, local);
642                 if (!args->verbose)
643                         continue;
644                 fprintf(stderr,
645                         "already have %s (%s)\n", sha1_to_hex(remote),
646                         ref->name);
647         }
648         return retval;
649 }
650
651 static int sideband_demux(int in, int out, void *data)
652 {
653         int *xd = data;
654
655         int ret = recv_sideband("fetch-pack", xd[0], out);
656         close(out);
657         return ret;
658 }
659
660 static int get_pack(struct fetch_pack_args *args,
661                     int xd[2], char **pack_lockfile)
662 {
663         struct async demux;
664         const char *argv[22];
665         char keep_arg[256];
666         char hdr_arg[256];
667         const char **av, *cmd_name;
668         int do_keep = args->keep_pack;
669         struct child_process cmd = CHILD_PROCESS_INIT;
670         int ret;
671
672         memset(&demux, 0, sizeof(demux));
673         if (use_sideband) {
674                 /* xd[] is talking with upload-pack; subprocess reads from
675                  * xd[0], spits out band#2 to stderr, and feeds us band#1
676                  * through demux->out.
677                  */
678                 demux.proc = sideband_demux;
679                 demux.data = xd;
680                 demux.out = -1;
681                 if (start_async(&demux))
682                         die("fetch-pack: unable to fork off sideband"
683                             " demultiplexer");
684         }
685         else
686                 demux.out = xd[0];
687
688         cmd.argv = argv;
689         av = argv;
690         *hdr_arg = 0;
691         if (!args->keep_pack && unpack_limit) {
692                 struct pack_header header;
693
694                 if (read_pack_header(demux.out, &header))
695                         die("protocol error: bad pack header");
696                 snprintf(hdr_arg, sizeof(hdr_arg),
697                          "--pack_header=%"PRIu32",%"PRIu32,
698                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
699                 if (ntohl(header.hdr_entries) < unpack_limit)
700                         do_keep = 0;
701                 else
702                         do_keep = 1;
703         }
704
705         if (alternate_shallow_file) {
706                 *av++ = "--shallow-file";
707                 *av++ = alternate_shallow_file;
708         }
709
710         if (do_keep) {
711                 if (pack_lockfile)
712                         cmd.out = -1;
713                 *av++ = cmd_name = "index-pack";
714                 *av++ = "--stdin";
715                 if (!args->quiet && !args->no_progress)
716                         *av++ = "-v";
717                 if (args->use_thin_pack)
718                         *av++ = "--fix-thin";
719                 if (args->lock_pack || unpack_limit) {
720                         int s = sprintf(keep_arg,
721                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
722                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
723                                 strcpy(keep_arg + s, "localhost");
724                         *av++ = keep_arg;
725                 }
726                 if (args->check_self_contained_and_connected)
727                         *av++ = "--check-self-contained-and-connected";
728         }
729         else {
730                 *av++ = cmd_name = "unpack-objects";
731                 if (args->quiet || args->no_progress)
732                         *av++ = "-q";
733                 args->check_self_contained_and_connected = 0;
734         }
735         if (*hdr_arg)
736                 *av++ = hdr_arg;
737         if (fetch_fsck_objects >= 0
738             ? fetch_fsck_objects
739             : transfer_fsck_objects >= 0
740             ? transfer_fsck_objects
741             : 0)
742                 *av++ = "--strict";
743         *av++ = NULL;
744
745         cmd.in = demux.out;
746         cmd.git_cmd = 1;
747         if (start_command(&cmd))
748                 die("fetch-pack: unable to fork off %s", cmd_name);
749         if (do_keep && pack_lockfile) {
750                 *pack_lockfile = index_pack_lockfile(cmd.out);
751                 close(cmd.out);
752         }
753
754         if (!use_sideband)
755                 /* Closed by start_command() */
756                 xd[0] = -1;
757
758         ret = finish_command(&cmd);
759         if (!ret || (args->check_self_contained_and_connected && ret == 1))
760                 args->self_contained_and_connected =
761                         args->check_self_contained_and_connected &&
762                         ret == 0;
763         else
764                 die("%s failed", cmd_name);
765         if (use_sideband && finish_async(&demux))
766                 die("error in sideband demultiplexer");
767         return 0;
768 }
769
770 static int cmp_ref_by_name(const void *a_, const void *b_)
771 {
772         const struct ref *a = *((const struct ref **)a_);
773         const struct ref *b = *((const struct ref **)b_);
774         return strcmp(a->name, b->name);
775 }
776
777 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
778                                  int fd[2],
779                                  const struct ref *orig_ref,
780                                  struct ref **sought, int nr_sought,
781                                  struct shallow_info *si,
782                                  char **pack_lockfile)
783 {
784         struct ref *ref = copy_ref_list(orig_ref);
785         unsigned char sha1[20];
786         const char *agent_feature;
787         int agent_len;
788
789         sort_ref_list(&ref, ref_compare_name);
790         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
791
792         if (is_repository_shallow() && !server_supports("shallow"))
793                 die("Server does not support shallow clients");
794         if (server_supports("multi_ack_detailed")) {
795                 if (args->verbose)
796                         fprintf(stderr, "Server supports multi_ack_detailed\n");
797                 multi_ack = 2;
798                 if (server_supports("no-done")) {
799                         if (args->verbose)
800                                 fprintf(stderr, "Server supports no-done\n");
801                         if (args->stateless_rpc)
802                                 no_done = 1;
803                 }
804         }
805         else if (server_supports("multi_ack")) {
806                 if (args->verbose)
807                         fprintf(stderr, "Server supports multi_ack\n");
808                 multi_ack = 1;
809         }
810         if (server_supports("side-band-64k")) {
811                 if (args->verbose)
812                         fprintf(stderr, "Server supports side-band-64k\n");
813                 use_sideband = 2;
814         }
815         else if (server_supports("side-band")) {
816                 if (args->verbose)
817                         fprintf(stderr, "Server supports side-band\n");
818                 use_sideband = 1;
819         }
820         if (server_supports("allow-tip-sha1-in-want")) {
821                 if (args->verbose)
822                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
823                 allow_tip_sha1_in_want = 1;
824         }
825         if (!server_supports("thin-pack"))
826                 args->use_thin_pack = 0;
827         if (!server_supports("no-progress"))
828                 args->no_progress = 0;
829         if (!server_supports("include-tag"))
830                 args->include_tag = 0;
831         if (server_supports("ofs-delta")) {
832                 if (args->verbose)
833                         fprintf(stderr, "Server supports ofs-delta\n");
834         } else
835                 prefer_ofs_delta = 0;
836
837         if ((agent_feature = server_feature_value("agent", &agent_len))) {
838                 agent_supported = 1;
839                 if (args->verbose && agent_len)
840                         fprintf(stderr, "Server version is %.*s\n",
841                                 agent_len, agent_feature);
842         }
843
844         if (everything_local(args, &ref, sought, nr_sought)) {
845                 packet_flush(fd[1]);
846                 goto all_done;
847         }
848         if (find_common(args, fd, sha1, ref) < 0)
849                 if (!args->keep_pack)
850                         /* When cloning, it is not unusual to have
851                          * no common commit.
852                          */
853                         warning("no common commits");
854
855         if (args->stateless_rpc)
856                 packet_flush(fd[1]);
857         if (args->depth > 0)
858                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
859                                         NULL);
860         else if (si->nr_ours || si->nr_theirs)
861                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
862         else
863                 alternate_shallow_file = NULL;
864         if (get_pack(args, fd, pack_lockfile))
865                 die("git fetch-pack: fetch failed.");
866
867  all_done:
868         return ref;
869 }
870
871 static void fetch_pack_config(void)
872 {
873         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
874         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
875         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
876         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
877         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
878
879         git_config(git_default_config, NULL);
880 }
881
882 static void fetch_pack_setup(void)
883 {
884         static int did_setup;
885         if (did_setup)
886                 return;
887         fetch_pack_config();
888         if (0 <= transfer_unpack_limit)
889                 unpack_limit = transfer_unpack_limit;
890         else if (0 <= fetch_unpack_limit)
891                 unpack_limit = fetch_unpack_limit;
892         did_setup = 1;
893 }
894
895 static int remove_duplicates_in_refs(struct ref **ref, int nr)
896 {
897         struct string_list names = STRING_LIST_INIT_NODUP;
898         int src, dst;
899
900         for (src = dst = 0; src < nr; src++) {
901                 struct string_list_item *item;
902                 item = string_list_insert(&names, ref[src]->name);
903                 if (item->util)
904                         continue; /* already have it */
905                 item->util = ref[src];
906                 if (src != dst)
907                         ref[dst] = ref[src];
908                 dst++;
909         }
910         for (src = dst; src < nr; src++)
911                 ref[src] = NULL;
912         string_list_clear(&names, 0);
913         return dst;
914 }
915
916 static void update_shallow(struct fetch_pack_args *args,
917                            struct ref **sought, int nr_sought,
918                            struct shallow_info *si)
919 {
920         struct sha1_array ref = SHA1_ARRAY_INIT;
921         int *status;
922         int i;
923
924         if (args->depth > 0 && alternate_shallow_file) {
925                 if (*alternate_shallow_file == '\0') { /* --unshallow */
926                         unlink_or_warn(git_path("shallow"));
927                         rollback_lock_file(&shallow_lock);
928                 } else
929                         commit_lock_file(&shallow_lock);
930                 return;
931         }
932
933         if (!si->shallow || !si->shallow->nr)
934                 return;
935
936         if (args->cloning) {
937                 /*
938                  * remote is shallow, but this is a clone, there are
939                  * no objects in repo to worry about. Accept any
940                  * shallow points that exist in the pack (iow in repo
941                  * after get_pack() and reprepare_packed_git())
942                  */
943                 struct sha1_array extra = SHA1_ARRAY_INIT;
944                 unsigned char (*sha1)[20] = si->shallow->sha1;
945                 for (i = 0; i < si->shallow->nr; i++)
946                         if (has_sha1_file(sha1[i]))
947                                 sha1_array_append(&extra, sha1[i]);
948                 if (extra.nr) {
949                         setup_alternate_shallow(&shallow_lock,
950                                                 &alternate_shallow_file,
951                                                 &extra);
952                         commit_lock_file(&shallow_lock);
953                 }
954                 sha1_array_clear(&extra);
955                 return;
956         }
957
958         if (!si->nr_ours && !si->nr_theirs)
959                 return;
960
961         remove_nonexistent_theirs_shallow(si);
962         if (!si->nr_ours && !si->nr_theirs)
963                 return;
964         for (i = 0; i < nr_sought; i++)
965                 sha1_array_append(&ref, sought[i]->old_sha1);
966         si->ref = &ref;
967
968         if (args->update_shallow) {
969                 /*
970                  * remote is also shallow, .git/shallow may be updated
971                  * so all refs can be accepted. Make sure we only add
972                  * shallow roots that are actually reachable from new
973                  * refs.
974                  */
975                 struct sha1_array extra = SHA1_ARRAY_INIT;
976                 unsigned char (*sha1)[20] = si->shallow->sha1;
977                 assign_shallow_commits_to_refs(si, NULL, NULL);
978                 if (!si->nr_ours && !si->nr_theirs) {
979                         sha1_array_clear(&ref);
980                         return;
981                 }
982                 for (i = 0; i < si->nr_ours; i++)
983                         sha1_array_append(&extra, sha1[si->ours[i]]);
984                 for (i = 0; i < si->nr_theirs; i++)
985                         sha1_array_append(&extra, sha1[si->theirs[i]]);
986                 setup_alternate_shallow(&shallow_lock,
987                                         &alternate_shallow_file,
988                                         &extra);
989                 commit_lock_file(&shallow_lock);
990                 sha1_array_clear(&extra);
991                 sha1_array_clear(&ref);
992                 return;
993         }
994
995         /*
996          * remote is also shallow, check what ref is safe to update
997          * without updating .git/shallow
998          */
999         status = xcalloc(nr_sought, sizeof(*status));
1000         assign_shallow_commits_to_refs(si, NULL, status);
1001         if (si->nr_ours || si->nr_theirs) {
1002                 for (i = 0; i < nr_sought; i++)
1003                         if (status[i])
1004                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1005         }
1006         free(status);
1007         sha1_array_clear(&ref);
1008 }
1009
1010 struct ref *fetch_pack(struct fetch_pack_args *args,
1011                        int fd[], struct child_process *conn,
1012                        const struct ref *ref,
1013                        const char *dest,
1014                        struct ref **sought, int nr_sought,
1015                        struct sha1_array *shallow,
1016                        char **pack_lockfile)
1017 {
1018         struct ref *ref_cpy;
1019         struct shallow_info si;
1020
1021         fetch_pack_setup();
1022         if (nr_sought)
1023                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1024
1025         if (!ref) {
1026                 packet_flush(fd[1]);
1027                 die("no matching remote head");
1028         }
1029         prepare_shallow_info(&si, shallow);
1030         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1031                                 &si, pack_lockfile);
1032         reprepare_packed_git();
1033         update_shallow(args, sought, nr_sought, &si);
1034         clear_shallow_info(&si);
1035         return ref_cpy;
1036 }