Merge branch 'jk/fsck-gitmodules-gently'
[git] / fetch-pack.c
1 #include "cache.h"
2 #include "repository.h"
3 #include "config.h"
4 #include "lockfile.h"
5 #include "refs.h"
6 #include "pkt-line.h"
7 #include "commit.h"
8 #include "tag.h"
9 #include "exec-cmd.h"
10 #include "pack.h"
11 #include "sideband.h"
12 #include "fetch-pack.h"
13 #include "remote.h"
14 #include "run-command.h"
15 #include "connect.h"
16 #include "transport.h"
17 #include "version.h"
18 #include "prio-queue.h"
19 #include "sha1-array.h"
20 #include "oidset.h"
21 #include "packfile.h"
22 #include "object-store.h"
23 #include "connected.h"
24
25 static int transfer_unpack_limit = -1;
26 static int fetch_unpack_limit = -1;
27 static int unpack_limit = 100;
28 static int prefer_ofs_delta = 1;
29 static int no_done;
30 static int deepen_since_ok;
31 static int deepen_not_ok;
32 static int fetch_fsck_objects = -1;
33 static int transfer_fsck_objects = -1;
34 static int agent_supported;
35 static int server_supports_filtering;
36 static struct lock_file shallow_lock;
37 static const char *alternate_shallow_file;
38
39 /* Remember to update object flag allocation in object.h */
40 #define COMPLETE        (1U << 0)
41 #define COMMON          (1U << 1)
42 #define COMMON_REF      (1U << 2)
43 #define SEEN            (1U << 3)
44 #define POPPED          (1U << 4)
45 #define ALTERNATE       (1U << 5)
46
47 static int marked;
48
49 /*
50  * After sending this many "have"s if we do not get any new ACK , we
51  * give up traversing our history.
52  */
53 #define MAX_IN_VAIN 256
54
55 static struct prio_queue rev_list = { compare_commits_by_commit_date };
56 static int non_common_revs, multi_ack, use_sideband;
57 /* Allow specifying sha1 if it is a ref tip. */
58 #define ALLOW_TIP_SHA1  01
59 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
60 #define ALLOW_REACHABLE_SHA1    02
61 static unsigned int allow_unadvertised_object_request;
62
63 __attribute__((format (printf, 2, 3)))
64 static inline void print_verbose(const struct fetch_pack_args *args,
65                                  const char *fmt, ...)
66 {
67         va_list params;
68
69         if (!args->verbose)
70                 return;
71
72         va_start(params, fmt);
73         vfprintf(stderr, fmt, params);
74         va_end(params);
75         fputc('\n', stderr);
76 }
77
78 struct alternate_object_cache {
79         struct object **items;
80         size_t nr, alloc;
81 };
82
83 static void cache_one_alternate(const char *refname,
84                                 const struct object_id *oid,
85                                 void *vcache)
86 {
87         struct alternate_object_cache *cache = vcache;
88         struct object *obj = parse_object(oid);
89
90         if (!obj || (obj->flags & ALTERNATE))
91                 return;
92
93         obj->flags |= ALTERNATE;
94         ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
95         cache->items[cache->nr++] = obj;
96 }
97
98 static void for_each_cached_alternate(void (*cb)(struct object *))
99 {
100         static int initialized;
101         static struct alternate_object_cache cache;
102         size_t i;
103
104         if (!initialized) {
105                 for_each_alternate_ref(cache_one_alternate, &cache);
106                 initialized = 1;
107         }
108
109         for (i = 0; i < cache.nr; i++)
110                 cb(cache.items[i]);
111 }
112
113 static void rev_list_push(struct commit *commit, int mark)
114 {
115         if (!(commit->object.flags & mark)) {
116                 commit->object.flags |= mark;
117
118                 if (parse_commit(commit))
119                         return;
120
121                 prio_queue_put(&rev_list, commit);
122
123                 if (!(commit->object.flags & COMMON))
124                         non_common_revs++;
125         }
126 }
127
128 static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
129 {
130         struct object *o = deref_tag(parse_object(oid), refname, 0);
131
132         if (o && o->type == OBJ_COMMIT)
133                 rev_list_push((struct commit *)o, SEEN);
134
135         return 0;
136 }
137
138 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
139                                    int flag, void *cb_data)
140 {
141         return rev_list_insert_ref(refname, oid);
142 }
143
144 static int clear_marks(const char *refname, const struct object_id *oid,
145                        int flag, void *cb_data)
146 {
147         struct object *o = deref_tag(parse_object(oid), refname, 0);
148
149         if (o && o->type == OBJ_COMMIT)
150                 clear_commit_marks((struct commit *)o,
151                                    COMMON | COMMON_REF | SEEN | POPPED);
152         return 0;
153 }
154
155 /*
156    This function marks a rev and its ancestors as common.
157    In some cases, it is desirable to mark only the ancestors (for example
158    when only the server does not yet know that they are common).
159 */
160
161 static void mark_common(struct commit *commit,
162                 int ancestors_only, int dont_parse)
163 {
164         if (commit != NULL && !(commit->object.flags & COMMON)) {
165                 struct object *o = (struct object *)commit;
166
167                 if (!ancestors_only)
168                         o->flags |= COMMON;
169
170                 if (!(o->flags & SEEN))
171                         rev_list_push(commit, SEEN);
172                 else {
173                         struct commit_list *parents;
174
175                         if (!ancestors_only && !(o->flags & POPPED))
176                                 non_common_revs--;
177                         if (!o->parsed && !dont_parse)
178                                 if (parse_commit(commit))
179                                         return;
180
181                         for (parents = commit->parents;
182                                         parents;
183                                         parents = parents->next)
184                                 mark_common(parents->item, 0, dont_parse);
185                 }
186         }
187 }
188
189 /*
190   Get the next rev to send, ignoring the common.
191 */
192
193 static const struct object_id *get_rev(void)
194 {
195         struct commit *commit = NULL;
196
197         while (commit == NULL) {
198                 unsigned int mark;
199                 struct commit_list *parents;
200
201                 if (rev_list.nr == 0 || non_common_revs == 0)
202                         return NULL;
203
204                 commit = prio_queue_get(&rev_list);
205                 parse_commit(commit);
206                 parents = commit->parents;
207
208                 commit->object.flags |= POPPED;
209                 if (!(commit->object.flags & COMMON))
210                         non_common_revs--;
211
212                 if (commit->object.flags & COMMON) {
213                         /* do not send "have", and ignore ancestors */
214                         commit = NULL;
215                         mark = COMMON | SEEN;
216                 } else if (commit->object.flags & COMMON_REF)
217                         /* send "have", and ignore ancestors */
218                         mark = COMMON | SEEN;
219                 else
220                         /* send "have", also for its ancestors */
221                         mark = SEEN;
222
223                 while (parents) {
224                         if (!(parents->item->object.flags & SEEN))
225                                 rev_list_push(parents->item, mark);
226                         if (mark & COMMON)
227                                 mark_common(parents->item, 1, 0);
228                         parents = parents->next;
229                 }
230         }
231
232         return &commit->object.oid;
233 }
234
235 enum ack_type {
236         NAK = 0,
237         ACK,
238         ACK_continue,
239         ACK_common,
240         ACK_ready
241 };
242
243 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
244 {
245         if (args->stateless_rpc && args->deepen) {
246                 /* If we sent a depth we will get back "duplicate"
247                  * shallow and unshallow commands every time there
248                  * is a block of have lines exchanged.
249                  */
250                 char *line;
251                 while ((line = packet_read_line(fd, NULL))) {
252                         if (starts_with(line, "shallow "))
253                                 continue;
254                         if (starts_with(line, "unshallow "))
255                                 continue;
256                         die(_("git fetch-pack: expected shallow list"));
257                 }
258         }
259 }
260
261 static enum ack_type get_ack(int fd, struct object_id *result_oid)
262 {
263         int len;
264         char *line = packet_read_line(fd, &len);
265         const char *arg;
266
267         if (!line)
268                 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
269         if (!strcmp(line, "NAK"))
270                 return NAK;
271         if (skip_prefix(line, "ACK ", &arg)) {
272                 if (!get_oid_hex(arg, result_oid)) {
273                         arg += 40;
274                         len -= arg - line;
275                         if (len < 1)
276                                 return ACK;
277                         if (strstr(arg, "continue"))
278                                 return ACK_continue;
279                         if (strstr(arg, "common"))
280                                 return ACK_common;
281                         if (strstr(arg, "ready"))
282                                 return ACK_ready;
283                         return ACK;
284                 }
285         }
286         if (skip_prefix(line, "ERR ", &arg))
287                 die(_("remote error: %s"), arg);
288         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
289 }
290
291 static void send_request(struct fetch_pack_args *args,
292                          int fd, struct strbuf *buf)
293 {
294         if (args->stateless_rpc) {
295                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
296                 packet_flush(fd);
297         } else
298                 write_or_die(fd, buf->buf, buf->len);
299 }
300
301 static void insert_one_alternate_object(struct object *obj)
302 {
303         rev_list_insert_ref(NULL, &obj->oid);
304 }
305
306 #define INITIAL_FLUSH 16
307 #define PIPESAFE_FLUSH 32
308 #define LARGE_FLUSH 16384
309
310 static int next_flush(int stateless_rpc, int count)
311 {
312         if (stateless_rpc) {
313                 if (count < LARGE_FLUSH)
314                         count <<= 1;
315                 else
316                         count = count * 11 / 10;
317         } else {
318                 if (count < PIPESAFE_FLUSH)
319                         count <<= 1;
320                 else
321                         count += PIPESAFE_FLUSH;
322         }
323         return count;
324 }
325
326 static int find_common(struct fetch_pack_args *args,
327                        int fd[2], struct object_id *result_oid,
328                        struct ref *refs)
329 {
330         int fetching;
331         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
332         const struct object_id *oid;
333         unsigned in_vain = 0;
334         int got_continue = 0;
335         int got_ready = 0;
336         struct strbuf req_buf = STRBUF_INIT;
337         size_t state_len = 0;
338
339         if (args->stateless_rpc && multi_ack == 1)
340                 die(_("--stateless-rpc requires multi_ack_detailed"));
341         if (marked)
342                 for_each_ref(clear_marks, NULL);
343         marked = 1;
344
345         for_each_ref(rev_list_insert_ref_oid, NULL);
346         for_each_cached_alternate(insert_one_alternate_object);
347
348         fetching = 0;
349         for ( ; refs ; refs = refs->next) {
350                 struct object_id *remote = &refs->old_oid;
351                 const char *remote_hex;
352                 struct object *o;
353
354                 /*
355                  * If that object is complete (i.e. it is an ancestor of a
356                  * local ref), we tell them we have it but do not have to
357                  * tell them about its ancestors, which they already know
358                  * about.
359                  *
360                  * We use lookup_object here because we are only
361                  * interested in the case we *know* the object is
362                  * reachable and we have already scanned it.
363                  */
364                 if (((o = lookup_object(remote->hash)) != NULL) &&
365                                 (o->flags & COMPLETE)) {
366                         continue;
367                 }
368
369                 remote_hex = oid_to_hex(remote);
370                 if (!fetching) {
371                         struct strbuf c = STRBUF_INIT;
372                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
373                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
374                         if (no_done)            strbuf_addstr(&c, " no-done");
375                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
376                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
377                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
378                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
379                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
380                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
381                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
382                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
383                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
384                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
385                                                             git_user_agent_sanitized());
386                         if (args->filter_options.choice)
387                                 strbuf_addstr(&c, " filter");
388                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
389                         strbuf_release(&c);
390                 } else
391                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
392                 fetching++;
393         }
394
395         if (!fetching) {
396                 strbuf_release(&req_buf);
397                 packet_flush(fd[1]);
398                 return 1;
399         }
400
401         if (is_repository_shallow(the_repository))
402                 write_shallow_commits(&req_buf, 1, NULL);
403         if (args->depth > 0)
404                 packet_buf_write(&req_buf, "deepen %d", args->depth);
405         if (args->deepen_since) {
406                 timestamp_t max_age = approxidate(args->deepen_since);
407                 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
408         }
409         if (args->deepen_not) {
410                 int i;
411                 for (i = 0; i < args->deepen_not->nr; i++) {
412                         struct string_list_item *s = args->deepen_not->items + i;
413                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
414                 }
415         }
416         if (server_supports_filtering && args->filter_options.choice)
417                 packet_buf_write(&req_buf, "filter %s",
418                                  args->filter_options.filter_spec);
419         packet_buf_flush(&req_buf);
420         state_len = req_buf.len;
421
422         if (args->deepen) {
423                 char *line;
424                 const char *arg;
425                 struct object_id oid;
426
427                 send_request(args, fd[1], &req_buf);
428                 while ((line = packet_read_line(fd[0], NULL))) {
429                         if (skip_prefix(line, "shallow ", &arg)) {
430                                 if (get_oid_hex(arg, &oid))
431                                         die(_("invalid shallow line: %s"), line);
432                                 register_shallow(the_repository, &oid);
433                                 continue;
434                         }
435                         if (skip_prefix(line, "unshallow ", &arg)) {
436                                 if (get_oid_hex(arg, &oid))
437                                         die(_("invalid unshallow line: %s"), line);
438                                 if (!lookup_object(oid.hash))
439                                         die(_("object not found: %s"), line);
440                                 /* make sure that it is parsed as shallow */
441                                 if (!parse_object(&oid))
442                                         die(_("error in object: %s"), line);
443                                 if (unregister_shallow(&oid))
444                                         die(_("no shallow found: %s"), line);
445                                 continue;
446                         }
447                         die(_("expected shallow/unshallow, got %s"), line);
448                 }
449         } else if (!args->stateless_rpc)
450                 send_request(args, fd[1], &req_buf);
451
452         if (!args->stateless_rpc) {
453                 /* If we aren't using the stateless-rpc interface
454                  * we don't need to retain the headers.
455                  */
456                 strbuf_setlen(&req_buf, 0);
457                 state_len = 0;
458         }
459
460         flushes = 0;
461         retval = -1;
462         if (args->no_dependents)
463                 goto done;
464         while ((oid = get_rev())) {
465                 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
466                 print_verbose(args, "have %s", oid_to_hex(oid));
467                 in_vain++;
468                 if (flush_at <= ++count) {
469                         int ack;
470
471                         packet_buf_flush(&req_buf);
472                         send_request(args, fd[1], &req_buf);
473                         strbuf_setlen(&req_buf, state_len);
474                         flushes++;
475                         flush_at = next_flush(args->stateless_rpc, count);
476
477                         /*
478                          * We keep one window "ahead" of the other side, and
479                          * will wait for an ACK only on the next one
480                          */
481                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
482                                 continue;
483
484                         consume_shallow_list(args, fd[0]);
485                         do {
486                                 ack = get_ack(fd[0], result_oid);
487                                 if (ack)
488                                         print_verbose(args, _("got %s %d %s"), "ack",
489                                                       ack, oid_to_hex(result_oid));
490                                 switch (ack) {
491                                 case ACK:
492                                         flushes = 0;
493                                         multi_ack = 0;
494                                         retval = 0;
495                                         goto done;
496                                 case ACK_common:
497                                 case ACK_ready:
498                                 case ACK_continue: {
499                                         struct commit *commit =
500                                                 lookup_commit(result_oid);
501                                         if (!commit)
502                                                 die(_("invalid commit %s"), oid_to_hex(result_oid));
503                                         if (args->stateless_rpc
504                                          && ack == ACK_common
505                                          && !(commit->object.flags & COMMON)) {
506                                                 /* We need to replay the have for this object
507                                                  * on the next RPC request so the peer knows
508                                                  * it is in common with us.
509                                                  */
510                                                 const char *hex = oid_to_hex(result_oid);
511                                                 packet_buf_write(&req_buf, "have %s\n", hex);
512                                                 state_len = req_buf.len;
513                                                 /*
514                                                  * Reset in_vain because an ack
515                                                  * for this commit has not been
516                                                  * seen.
517                                                  */
518                                                 in_vain = 0;
519                                         } else if (!args->stateless_rpc
520                                                    || ack != ACK_common)
521                                                 in_vain = 0;
522                                         mark_common(commit, 0, 1);
523                                         retval = 0;
524                                         got_continue = 1;
525                                         if (ack == ACK_ready) {
526                                                 clear_prio_queue(&rev_list);
527                                                 got_ready = 1;
528                                         }
529                                         break;
530                                         }
531                                 }
532                         } while (ack);
533                         flushes--;
534                         if (got_continue && MAX_IN_VAIN < in_vain) {
535                                 print_verbose(args, _("giving up"));
536                                 break; /* give up */
537                         }
538                 }
539         }
540 done:
541         if (!got_ready || !no_done) {
542                 packet_buf_write(&req_buf, "done\n");
543                 send_request(args, fd[1], &req_buf);
544         }
545         print_verbose(args, _("done"));
546         if (retval != 0) {
547                 multi_ack = 0;
548                 flushes++;
549         }
550         strbuf_release(&req_buf);
551
552         if (!got_ready || !no_done)
553                 consume_shallow_list(args, fd[0]);
554         while (flushes || multi_ack) {
555                 int ack = get_ack(fd[0], result_oid);
556                 if (ack) {
557                         print_verbose(args, _("got %s (%d) %s"), "ack",
558                                       ack, oid_to_hex(result_oid));
559                         if (ack == ACK)
560                                 return 0;
561                         multi_ack = 1;
562                         continue;
563                 }
564                 flushes--;
565         }
566         /* it is no error to fetch into a completely empty repo */
567         return count ? retval : 0;
568 }
569
570 static struct commit_list *complete;
571
572 static int mark_complete(const struct object_id *oid)
573 {
574         struct object *o = parse_object(oid);
575
576         while (o && o->type == OBJ_TAG) {
577                 struct tag *t = (struct tag *) o;
578                 if (!t->tagged)
579                         break; /* broken repository */
580                 o->flags |= COMPLETE;
581                 o = parse_object(&t->tagged->oid);
582         }
583         if (o && o->type == OBJ_COMMIT) {
584                 struct commit *commit = (struct commit *)o;
585                 if (!(commit->object.flags & COMPLETE)) {
586                         commit->object.flags |= COMPLETE;
587                         commit_list_insert(commit, &complete);
588                 }
589         }
590         return 0;
591 }
592
593 static int mark_complete_oid(const char *refname, const struct object_id *oid,
594                              int flag, void *cb_data)
595 {
596         return mark_complete(oid);
597 }
598
599 static void mark_recent_complete_commits(struct fetch_pack_args *args,
600                                          timestamp_t cutoff)
601 {
602         while (complete && cutoff <= complete->item->date) {
603                 print_verbose(args, _("Marking %s as complete"),
604                               oid_to_hex(&complete->item->object.oid));
605                 pop_most_recent_commit(&complete, COMPLETE);
606         }
607 }
608
609 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
610 {
611         for (; refs; refs = refs->next)
612                 oidset_insert(oids, &refs->old_oid);
613 }
614
615 static int tip_oids_contain(struct oidset *tip_oids,
616                             struct ref *unmatched, struct ref *newlist,
617                             const struct object_id *id)
618 {
619         /*
620          * Note that this only looks at the ref lists the first time it's
621          * called. This works out in filter_refs() because even though it may
622          * add to "newlist" between calls, the additions will always be for
623          * oids that are already in the set.
624          */
625         if (!tip_oids->map.map.tablesize) {
626                 add_refs_to_oidset(tip_oids, unmatched);
627                 add_refs_to_oidset(tip_oids, newlist);
628         }
629         return oidset_contains(tip_oids, id);
630 }
631
632 static void filter_refs(struct fetch_pack_args *args,
633                         struct ref **refs,
634                         struct ref **sought, int nr_sought)
635 {
636         struct ref *newlist = NULL;
637         struct ref **newtail = &newlist;
638         struct ref *unmatched = NULL;
639         struct ref *ref, *next;
640         struct oidset tip_oids = OIDSET_INIT;
641         int i;
642
643         i = 0;
644         for (ref = *refs; ref; ref = next) {
645                 int keep = 0;
646                 next = ref->next;
647
648                 if (starts_with(ref->name, "refs/") &&
649                     check_refname_format(ref->name, 0))
650                         ; /* trash */
651                 else {
652                         while (i < nr_sought) {
653                                 int cmp = strcmp(ref->name, sought[i]->name);
654                                 if (cmp < 0)
655                                         break; /* definitely do not have it */
656                                 else if (cmp == 0) {
657                                         keep = 1; /* definitely have it */
658                                         sought[i]->match_status = REF_MATCHED;
659                                 }
660                                 i++;
661                         }
662
663                         if (!keep && args->fetch_all &&
664                             (!args->deepen || !starts_with(ref->name, "refs/tags/")))
665                                 keep = 1;
666                 }
667
668                 if (keep) {
669                         *newtail = ref;
670                         ref->next = NULL;
671                         newtail = &ref->next;
672                 } else {
673                         ref->next = unmatched;
674                         unmatched = ref;
675                 }
676         }
677
678         /* Append unmatched requests to the list */
679         for (i = 0; i < nr_sought; i++) {
680                 struct object_id oid;
681                 const char *p;
682
683                 ref = sought[i];
684                 if (ref->match_status != REF_NOT_MATCHED)
685                         continue;
686                 if (parse_oid_hex(ref->name, &oid, &p) ||
687                     *p != '\0' ||
688                     oidcmp(&oid, &ref->old_oid))
689                         continue;
690
691                 if ((allow_unadvertised_object_request &
692                      (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
693                     tip_oids_contain(&tip_oids, unmatched, newlist,
694                                      &ref->old_oid)) {
695                         ref->match_status = REF_MATCHED;
696                         *newtail = copy_ref(ref);
697                         newtail = &(*newtail)->next;
698                 } else {
699                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
700                 }
701         }
702
703         oidset_clear(&tip_oids);
704         for (ref = unmatched; ref; ref = next) {
705                 next = ref->next;
706                 free(ref);
707         }
708
709         *refs = newlist;
710 }
711
712 static void mark_alternate_complete(struct object *obj)
713 {
714         mark_complete(&obj->oid);
715 }
716
717 struct loose_object_iter {
718         struct oidset *loose_object_set;
719         struct ref *refs;
720 };
721
722 /*
723  *  If the number of refs is not larger than the number of loose objects,
724  *  this function stops inserting.
725  */
726 static int add_loose_objects_to_set(const struct object_id *oid,
727                                     const char *path,
728                                     void *data)
729 {
730         struct loose_object_iter *iter = data;
731         oidset_insert(iter->loose_object_set, oid);
732         if (iter->refs == NULL)
733                 return 1;
734
735         iter->refs = iter->refs->next;
736         return 0;
737 }
738
739 static int everything_local(struct fetch_pack_args *args,
740                             struct ref **refs,
741                             struct ref **sought, int nr_sought)
742 {
743         struct ref *ref;
744         int retval;
745         int old_save_commit_buffer = save_commit_buffer;
746         timestamp_t cutoff = 0;
747         struct oidset loose_oid_set = OIDSET_INIT;
748         int use_oidset = 0;
749         struct loose_object_iter iter = {&loose_oid_set, *refs};
750
751         /* Enumerate all loose objects or know refs are not so many. */
752         use_oidset = !for_each_loose_object(add_loose_objects_to_set,
753                                             &iter, 0);
754
755         save_commit_buffer = 0;
756
757         for (ref = *refs; ref; ref = ref->next) {
758                 struct object *o;
759                 unsigned int flags = OBJECT_INFO_QUICK;
760
761                 if (use_oidset &&
762                     !oidset_contains(&loose_oid_set, &ref->old_oid)) {
763                         /*
764                          * I know this does not exist in the loose form,
765                          * so check if it exists in a non-loose form.
766                          */
767                         flags |= OBJECT_INFO_IGNORE_LOOSE;
768                 }
769
770                 if (!has_object_file_with_flags(&ref->old_oid, flags))
771                         continue;
772                 o = parse_object(&ref->old_oid);
773                 if (!o)
774                         continue;
775
776                 /* We already have it -- which may mean that we were
777                  * in sync with the other side at some time after
778                  * that (it is OK if we guess wrong here).
779                  */
780                 if (o->type == OBJ_COMMIT) {
781                         struct commit *commit = (struct commit *)o;
782                         if (!cutoff || cutoff < commit->date)
783                                 cutoff = commit->date;
784                 }
785         }
786
787         oidset_clear(&loose_oid_set);
788
789         if (!args->no_dependents) {
790                 if (!args->deepen) {
791                         for_each_ref(mark_complete_oid, NULL);
792                         for_each_cached_alternate(mark_alternate_complete);
793                         commit_list_sort_by_date(&complete);
794                         if (cutoff)
795                                 mark_recent_complete_commits(args, cutoff);
796                 }
797
798                 /*
799                  * Mark all complete remote refs as common refs.
800                  * Don't mark them common yet; the server has to be told so first.
801                  */
802                 for (ref = *refs; ref; ref = ref->next) {
803                         struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
804                                                      NULL, 0);
805
806                         if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
807                                 continue;
808
809                         if (!(o->flags & SEEN)) {
810                                 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
811
812                                 mark_common((struct commit *)o, 1, 1);
813                         }
814                 }
815         }
816
817         filter_refs(args, refs, sought, nr_sought);
818
819         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
820                 const struct object_id *remote = &ref->old_oid;
821                 struct object *o;
822
823                 o = lookup_object(remote->hash);
824                 if (!o || !(o->flags & COMPLETE)) {
825                         retval = 0;
826                         print_verbose(args, "want %s (%s)", oid_to_hex(remote),
827                                       ref->name);
828                         continue;
829                 }
830                 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
831                               ref->name);
832         }
833
834         save_commit_buffer = old_save_commit_buffer;
835
836         return retval;
837 }
838
839 static int sideband_demux(int in, int out, void *data)
840 {
841         int *xd = data;
842         int ret;
843
844         ret = recv_sideband("fetch-pack", xd[0], out);
845         close(out);
846         return ret;
847 }
848
849 static int get_pack(struct fetch_pack_args *args,
850                     int xd[2], char **pack_lockfile)
851 {
852         struct async demux;
853         int do_keep = args->keep_pack;
854         const char *cmd_name;
855         struct pack_header header;
856         int pass_header = 0;
857         struct child_process cmd = CHILD_PROCESS_INIT;
858         int ret;
859
860         memset(&demux, 0, sizeof(demux));
861         if (use_sideband) {
862                 /* xd[] is talking with upload-pack; subprocess reads from
863                  * xd[0], spits out band#2 to stderr, and feeds us band#1
864                  * through demux->out.
865                  */
866                 demux.proc = sideband_demux;
867                 demux.data = xd;
868                 demux.out = -1;
869                 demux.isolate_sigpipe = 1;
870                 if (start_async(&demux))
871                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
872         }
873         else
874                 demux.out = xd[0];
875
876         if (!args->keep_pack && unpack_limit) {
877
878                 if (read_pack_header(demux.out, &header))
879                         die(_("protocol error: bad pack header"));
880                 pass_header = 1;
881                 if (ntohl(header.hdr_entries) < unpack_limit)
882                         do_keep = 0;
883                 else
884                         do_keep = 1;
885         }
886
887         if (alternate_shallow_file) {
888                 argv_array_push(&cmd.args, "--shallow-file");
889                 argv_array_push(&cmd.args, alternate_shallow_file);
890         }
891
892         if (do_keep || args->from_promisor) {
893                 if (pack_lockfile)
894                         cmd.out = -1;
895                 cmd_name = "index-pack";
896                 argv_array_push(&cmd.args, cmd_name);
897                 argv_array_push(&cmd.args, "--stdin");
898                 if (!args->quiet && !args->no_progress)
899                         argv_array_push(&cmd.args, "-v");
900                 if (args->use_thin_pack)
901                         argv_array_push(&cmd.args, "--fix-thin");
902                 if (do_keep && (args->lock_pack || unpack_limit)) {
903                         char hostname[HOST_NAME_MAX + 1];
904                         if (xgethostname(hostname, sizeof(hostname)))
905                                 xsnprintf(hostname, sizeof(hostname), "localhost");
906                         argv_array_pushf(&cmd.args,
907                                         "--keep=fetch-pack %"PRIuMAX " on %s",
908                                         (uintmax_t)getpid(), hostname);
909                 }
910                 if (args->check_self_contained_and_connected)
911                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
912                 if (args->from_promisor)
913                         argv_array_push(&cmd.args, "--promisor");
914         }
915         else {
916                 cmd_name = "unpack-objects";
917                 argv_array_push(&cmd.args, cmd_name);
918                 if (args->quiet || args->no_progress)
919                         argv_array_push(&cmd.args, "-q");
920                 args->check_self_contained_and_connected = 0;
921         }
922
923         if (pass_header)
924                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
925                                  ntohl(header.hdr_version),
926                                  ntohl(header.hdr_entries));
927         if (fetch_fsck_objects >= 0
928             ? fetch_fsck_objects
929             : transfer_fsck_objects >= 0
930             ? transfer_fsck_objects
931             : 0) {
932                 if (args->from_promisor)
933                         /*
934                          * We cannot use --strict in index-pack because it
935                          * checks both broken objects and links, but we only
936                          * want to check for broken objects.
937                          */
938                         argv_array_push(&cmd.args, "--fsck-objects");
939                 else
940                         argv_array_push(&cmd.args, "--strict");
941         }
942
943         cmd.in = demux.out;
944         cmd.git_cmd = 1;
945         if (start_command(&cmd))
946                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
947         if (do_keep && pack_lockfile) {
948                 *pack_lockfile = index_pack_lockfile(cmd.out);
949                 close(cmd.out);
950         }
951
952         if (!use_sideband)
953                 /* Closed by start_command() */
954                 xd[0] = -1;
955
956         ret = finish_command(&cmd);
957         if (!ret || (args->check_self_contained_and_connected && ret == 1))
958                 args->self_contained_and_connected =
959                         args->check_self_contained_and_connected &&
960                         ret == 0;
961         else
962                 die(_("%s failed"), cmd_name);
963         if (use_sideband && finish_async(&demux))
964                 die(_("error in sideband demultiplexer"));
965         return 0;
966 }
967
968 static int cmp_ref_by_name(const void *a_, const void *b_)
969 {
970         const struct ref *a = *((const struct ref **)a_);
971         const struct ref *b = *((const struct ref **)b_);
972         return strcmp(a->name, b->name);
973 }
974
975 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
976                                  int fd[2],
977                                  const struct ref *orig_ref,
978                                  struct ref **sought, int nr_sought,
979                                  struct shallow_info *si,
980                                  char **pack_lockfile)
981 {
982         struct ref *ref = copy_ref_list(orig_ref);
983         struct object_id oid;
984         const char *agent_feature;
985         int agent_len;
986
987         sort_ref_list(&ref, ref_compare_name);
988         QSORT(sought, nr_sought, cmp_ref_by_name);
989
990         if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
991                 die(_("Server does not support shallow clients"));
992         if (args->depth > 0 || args->deepen_since || args->deepen_not)
993                 args->deepen = 1;
994         if (server_supports("multi_ack_detailed")) {
995                 print_verbose(args, _("Server supports multi_ack_detailed"));
996                 multi_ack = 2;
997                 if (server_supports("no-done")) {
998                         print_verbose(args, _("Server supports no-done"));
999                         if (args->stateless_rpc)
1000                                 no_done = 1;
1001                 }
1002         }
1003         else if (server_supports("multi_ack")) {
1004                 print_verbose(args, _("Server supports multi_ack"));
1005                 multi_ack = 1;
1006         }
1007         if (server_supports("side-band-64k")) {
1008                 print_verbose(args, _("Server supports side-band-64k"));
1009                 use_sideband = 2;
1010         }
1011         else if (server_supports("side-band")) {
1012                 print_verbose(args, _("Server supports side-band"));
1013                 use_sideband = 1;
1014         }
1015         if (server_supports("allow-tip-sha1-in-want")) {
1016                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
1017                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1018         }
1019         if (server_supports("allow-reachable-sha1-in-want")) {
1020                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
1021                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1022         }
1023         if (!server_supports("thin-pack"))
1024                 args->use_thin_pack = 0;
1025         if (!server_supports("no-progress"))
1026                 args->no_progress = 0;
1027         if (!server_supports("include-tag"))
1028                 args->include_tag = 0;
1029         if (server_supports("ofs-delta"))
1030                 print_verbose(args, _("Server supports ofs-delta"));
1031         else
1032                 prefer_ofs_delta = 0;
1033
1034         if (server_supports("filter")) {
1035                 server_supports_filtering = 1;
1036                 print_verbose(args, _("Server supports filter"));
1037         } else if (args->filter_options.choice) {
1038                 warning("filtering not recognized by server, ignoring");
1039         }
1040
1041         if ((agent_feature = server_feature_value("agent", &agent_len))) {
1042                 agent_supported = 1;
1043                 if (agent_len)
1044                         print_verbose(args, _("Server version is %.*s"),
1045                                       agent_len, agent_feature);
1046         }
1047         if (server_supports("deepen-since"))
1048                 deepen_since_ok = 1;
1049         else if (args->deepen_since)
1050                 die(_("Server does not support --shallow-since"));
1051         if (server_supports("deepen-not"))
1052                 deepen_not_ok = 1;
1053         else if (args->deepen_not)
1054                 die(_("Server does not support --shallow-exclude"));
1055         if (!server_supports("deepen-relative") && args->deepen_relative)
1056                 die(_("Server does not support --deepen"));
1057
1058         if (everything_local(args, &ref, sought, nr_sought)) {
1059                 packet_flush(fd[1]);
1060                 goto all_done;
1061         }
1062         if (find_common(args, fd, &oid, ref) < 0)
1063                 if (!args->keep_pack)
1064                         /* When cloning, it is not unusual to have
1065                          * no common commit.
1066                          */
1067                         warning(_("no common commits"));
1068
1069         if (args->stateless_rpc)
1070                 packet_flush(fd[1]);
1071         if (args->deepen)
1072                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1073                                         NULL);
1074         else if (si->nr_ours || si->nr_theirs)
1075                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1076         else
1077                 alternate_shallow_file = NULL;
1078         if (get_pack(args, fd, pack_lockfile))
1079                 die(_("git fetch-pack: fetch failed."));
1080
1081  all_done:
1082         return ref;
1083 }
1084
1085 static void add_shallow_requests(struct strbuf *req_buf,
1086                                  const struct fetch_pack_args *args)
1087 {
1088         if (is_repository_shallow(the_repository))
1089                 write_shallow_commits(req_buf, 1, NULL);
1090         if (args->depth > 0)
1091                 packet_buf_write(req_buf, "deepen %d", args->depth);
1092         if (args->deepen_since) {
1093                 timestamp_t max_age = approxidate(args->deepen_since);
1094                 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1095         }
1096         if (args->deepen_not) {
1097                 int i;
1098                 for (i = 0; i < args->deepen_not->nr; i++) {
1099                         struct string_list_item *s = args->deepen_not->items + i;
1100                         packet_buf_write(req_buf, "deepen-not %s", s->string);
1101                 }
1102         }
1103 }
1104
1105 static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1106 {
1107         int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
1108
1109         for ( ; wants ; wants = wants->next) {
1110                 const struct object_id *remote = &wants->old_oid;
1111                 struct object *o;
1112
1113                 /*
1114                  * If that object is complete (i.e. it is an ancestor of a
1115                  * local ref), we tell them we have it but do not have to
1116                  * tell them about its ancestors, which they already know
1117                  * about.
1118                  *
1119                  * We use lookup_object here because we are only
1120                  * interested in the case we *know* the object is
1121                  * reachable and we have already scanned it.
1122                  */
1123                 if (((o = lookup_object(remote->hash)) != NULL) &&
1124                     (o->flags & COMPLETE)) {
1125                         continue;
1126                 }
1127
1128                 if (!use_ref_in_want || wants->exact_oid)
1129                         packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
1130                 else
1131                         packet_buf_write(req_buf, "want-ref %s\n", wants->name);
1132         }
1133 }
1134
1135 static void add_common(struct strbuf *req_buf, struct oidset *common)
1136 {
1137         struct oidset_iter iter;
1138         const struct object_id *oid;
1139         oidset_iter_init(common, &iter);
1140
1141         while ((oid = oidset_iter_next(&iter))) {
1142                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1143         }
1144 }
1145
1146 static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
1147 {
1148         int ret = 0;
1149         int haves_added = 0;
1150         const struct object_id *oid;
1151
1152         while ((oid = get_rev())) {
1153                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1154                 if (++haves_added >= *haves_to_send)
1155                         break;
1156         }
1157
1158         *in_vain += haves_added;
1159         if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1160                 /* Send Done */
1161                 packet_buf_write(req_buf, "done\n");
1162                 ret = 1;
1163         }
1164
1165         /* Increase haves to send on next round */
1166         *haves_to_send = next_flush(1, *haves_to_send);
1167
1168         return ret;
1169 }
1170
1171 static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
1172                               const struct ref *wants, struct oidset *common,
1173                               int *haves_to_send, int *in_vain)
1174 {
1175         int ret = 0;
1176         struct strbuf req_buf = STRBUF_INIT;
1177
1178         if (server_supports_v2("fetch", 1))
1179                 packet_buf_write(&req_buf, "command=fetch");
1180         if (server_supports_v2("agent", 0))
1181                 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1182         if (args->server_options && args->server_options->nr &&
1183             server_supports_v2("server-option", 1)) {
1184                 int i;
1185                 for (i = 0; i < args->server_options->nr; i++)
1186                         packet_write_fmt(fd_out, "server-option=%s",
1187                                          args->server_options->items[i].string);
1188         }
1189
1190         packet_buf_delim(&req_buf);
1191         if (args->use_thin_pack)
1192                 packet_buf_write(&req_buf, "thin-pack");
1193         if (args->no_progress)
1194                 packet_buf_write(&req_buf, "no-progress");
1195         if (args->include_tag)
1196                 packet_buf_write(&req_buf, "include-tag");
1197         if (prefer_ofs_delta)
1198                 packet_buf_write(&req_buf, "ofs-delta");
1199
1200         /* Add shallow-info and deepen request */
1201         if (server_supports_feature("fetch", "shallow", 0))
1202                 add_shallow_requests(&req_buf, args);
1203         else if (is_repository_shallow(the_repository) || args->deepen)
1204                 die(_("Server does not support shallow requests"));
1205
1206         /* Add filter */
1207         if (server_supports_feature("fetch", "filter", 0) &&
1208             args->filter_options.choice) {
1209                 print_verbose(args, _("Server supports filter"));
1210                 packet_buf_write(&req_buf, "filter %s",
1211                                  args->filter_options.filter_spec);
1212         } else if (args->filter_options.choice) {
1213                 warning("filtering not recognized by server, ignoring");
1214         }
1215
1216         /* add wants */
1217         add_wants(wants, &req_buf);
1218
1219         if (args->no_dependents) {
1220                 packet_buf_write(&req_buf, "done");
1221                 ret = 1;
1222         } else {
1223                 /* Add all of the common commits we've found in previous rounds */
1224                 add_common(&req_buf, common);
1225
1226                 /* Add initial haves */
1227                 ret = add_haves(&req_buf, haves_to_send, in_vain);
1228         }
1229
1230         /* Send request */
1231         packet_buf_flush(&req_buf);
1232         write_or_die(fd_out, req_buf.buf, req_buf.len);
1233
1234         strbuf_release(&req_buf);
1235         return ret;
1236 }
1237
1238 /*
1239  * Processes a section header in a server's response and checks if it matches
1240  * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1241  * not consumed); if 0, the line will be consumed and the function will die if
1242  * the section header doesn't match what was expected.
1243  */
1244 static int process_section_header(struct packet_reader *reader,
1245                                   const char *section, int peek)
1246 {
1247         int ret;
1248
1249         if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1250                 die("error reading section header '%s'", section);
1251
1252         ret = !strcmp(reader->line, section);
1253
1254         if (!peek) {
1255                 if (!ret)
1256                         die("expected '%s', received '%s'",
1257                             section, reader->line);
1258                 packet_reader_read(reader);
1259         }
1260
1261         return ret;
1262 }
1263
1264 static int process_acks(struct packet_reader *reader, struct oidset *common)
1265 {
1266         /* received */
1267         int received_ready = 0;
1268         int received_ack = 0;
1269
1270         process_section_header(reader, "acknowledgments", 0);
1271         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1272                 const char *arg;
1273
1274                 if (!strcmp(reader->line, "NAK"))
1275                         continue;
1276
1277                 if (skip_prefix(reader->line, "ACK ", &arg)) {
1278                         struct object_id oid;
1279                         if (!get_oid_hex(arg, &oid)) {
1280                                 struct commit *commit;
1281                                 oidset_insert(common, &oid);
1282                                 commit = lookup_commit(&oid);
1283                                 mark_common(commit, 0, 1);
1284                         }
1285                         continue;
1286                 }
1287
1288                 if (!strcmp(reader->line, "ready")) {
1289                         clear_prio_queue(&rev_list);
1290                         received_ready = 1;
1291                         continue;
1292                 }
1293
1294                 die("unexpected acknowledgment line: '%s'", reader->line);
1295         }
1296
1297         if (reader->status != PACKET_READ_FLUSH &&
1298             reader->status != PACKET_READ_DELIM)
1299                 die("error processing acks: %d", reader->status);
1300
1301         /* return 0 if no common, 1 if there are common, or 2 if ready */
1302         return received_ready ? 2 : (received_ack ? 1 : 0);
1303 }
1304
1305 static void receive_shallow_info(struct fetch_pack_args *args,
1306                                  struct packet_reader *reader)
1307 {
1308         process_section_header(reader, "shallow-info", 0);
1309         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1310                 const char *arg;
1311                 struct object_id oid;
1312
1313                 if (skip_prefix(reader->line, "shallow ", &arg)) {
1314                         if (get_oid_hex(arg, &oid))
1315                                 die(_("invalid shallow line: %s"), reader->line);
1316                         register_shallow(the_repository, &oid);
1317                         continue;
1318                 }
1319                 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1320                         if (get_oid_hex(arg, &oid))
1321                                 die(_("invalid unshallow line: %s"), reader->line);
1322                         if (!lookup_object(oid.hash))
1323                                 die(_("object not found: %s"), reader->line);
1324                         /* make sure that it is parsed as shallow */
1325                         if (!parse_object(&oid))
1326                                 die(_("error in object: %s"), reader->line);
1327                         if (unregister_shallow(&oid))
1328                                 die(_("no shallow found: %s"), reader->line);
1329                         continue;
1330                 }
1331                 die(_("expected shallow/unshallow, got %s"), reader->line);
1332         }
1333
1334         if (reader->status != PACKET_READ_FLUSH &&
1335             reader->status != PACKET_READ_DELIM)
1336                 die("error processing shallow info: %d", reader->status);
1337
1338         setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1339         args->deepen = 1;
1340 }
1341
1342 static void receive_wanted_refs(struct packet_reader *reader, struct ref *refs)
1343 {
1344         process_section_header(reader, "wanted-refs", 0);
1345         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1346                 struct object_id oid;
1347                 const char *end;
1348                 struct ref *r = NULL;
1349
1350                 if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
1351                         die("expected wanted-ref, got '%s'", reader->line);
1352
1353                 for (r = refs; r; r = r->next) {
1354                         if (!strcmp(end, r->name)) {
1355                                 oidcpy(&r->old_oid, &oid);
1356                                 break;
1357                         }
1358                 }
1359
1360                 if (!r)
1361                         die("unexpected wanted-ref: '%s'", reader->line);
1362         }
1363
1364         if (reader->status != PACKET_READ_DELIM)
1365                 die("error processing wanted refs: %d", reader->status);
1366 }
1367
1368 enum fetch_state {
1369         FETCH_CHECK_LOCAL = 0,
1370         FETCH_SEND_REQUEST,
1371         FETCH_PROCESS_ACKS,
1372         FETCH_GET_PACK,
1373         FETCH_DONE,
1374 };
1375
1376 static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1377                                     int fd[2],
1378                                     const struct ref *orig_ref,
1379                                     struct ref **sought, int nr_sought,
1380                                     char **pack_lockfile)
1381 {
1382         struct ref *ref = copy_ref_list(orig_ref);
1383         enum fetch_state state = FETCH_CHECK_LOCAL;
1384         struct oidset common = OIDSET_INIT;
1385         struct packet_reader reader;
1386         int in_vain = 0;
1387         int haves_to_send = INITIAL_FLUSH;
1388         packet_reader_init(&reader, fd[0], NULL, 0,
1389                            PACKET_READ_CHOMP_NEWLINE);
1390
1391         while (state != FETCH_DONE) {
1392                 switch (state) {
1393                 case FETCH_CHECK_LOCAL:
1394                         sort_ref_list(&ref, ref_compare_name);
1395                         QSORT(sought, nr_sought, cmp_ref_by_name);
1396
1397                         /* v2 supports these by default */
1398                         allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1399                         use_sideband = 2;
1400                         if (args->depth > 0 || args->deepen_since || args->deepen_not)
1401                                 args->deepen = 1;
1402
1403                         if (marked)
1404                                 for_each_ref(clear_marks, NULL);
1405                         marked = 1;
1406
1407                         for_each_ref(rev_list_insert_ref_oid, NULL);
1408                         for_each_cached_alternate(insert_one_alternate_object);
1409
1410                         /* Filter 'ref' by 'sought' and those that aren't local */
1411                         if (everything_local(args, &ref, sought, nr_sought))
1412                                 state = FETCH_DONE;
1413                         else
1414                                 state = FETCH_SEND_REQUEST;
1415                         break;
1416                 case FETCH_SEND_REQUEST:
1417                         if (send_fetch_request(fd[1], args, ref, &common,
1418                                                &haves_to_send, &in_vain))
1419                                 state = FETCH_GET_PACK;
1420                         else
1421                                 state = FETCH_PROCESS_ACKS;
1422                         break;
1423                 case FETCH_PROCESS_ACKS:
1424                         /* Process ACKs/NAKs */
1425                         switch (process_acks(&reader, &common)) {
1426                         case 2:
1427                                 state = FETCH_GET_PACK;
1428                                 break;
1429                         case 1:
1430                                 in_vain = 0;
1431                                 /* fallthrough */
1432                         default:
1433                                 state = FETCH_SEND_REQUEST;
1434                                 break;
1435                         }
1436                         break;
1437                 case FETCH_GET_PACK:
1438                         /* Check for shallow-info section */
1439                         if (process_section_header(&reader, "shallow-info", 1))
1440                                 receive_shallow_info(args, &reader);
1441
1442                         if (process_section_header(&reader, "wanted-refs", 1))
1443                                 receive_wanted_refs(&reader, ref);
1444
1445                         /* get the pack */
1446                         process_section_header(&reader, "packfile", 0);
1447                         if (get_pack(args, fd, pack_lockfile))
1448                                 die(_("git fetch-pack: fetch failed."));
1449
1450                         state = FETCH_DONE;
1451                         break;
1452                 case FETCH_DONE:
1453                         continue;
1454                 }
1455         }
1456
1457         oidset_clear(&common);
1458         return ref;
1459 }
1460
1461 static void fetch_pack_config(void)
1462 {
1463         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1464         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1465         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1466         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1467         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1468
1469         git_config(git_default_config, NULL);
1470 }
1471
1472 static void fetch_pack_setup(void)
1473 {
1474         static int did_setup;
1475         if (did_setup)
1476                 return;
1477         fetch_pack_config();
1478         if (0 <= transfer_unpack_limit)
1479                 unpack_limit = transfer_unpack_limit;
1480         else if (0 <= fetch_unpack_limit)
1481                 unpack_limit = fetch_unpack_limit;
1482         did_setup = 1;
1483 }
1484
1485 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1486 {
1487         struct string_list names = STRING_LIST_INIT_NODUP;
1488         int src, dst;
1489
1490         for (src = dst = 0; src < nr; src++) {
1491                 struct string_list_item *item;
1492                 item = string_list_insert(&names, ref[src]->name);
1493                 if (item->util)
1494                         continue; /* already have it */
1495                 item->util = ref[src];
1496                 if (src != dst)
1497                         ref[dst] = ref[src];
1498                 dst++;
1499         }
1500         for (src = dst; src < nr; src++)
1501                 ref[src] = NULL;
1502         string_list_clear(&names, 0);
1503         return dst;
1504 }
1505
1506 static void update_shallow(struct fetch_pack_args *args,
1507                            struct ref *refs,
1508                            struct shallow_info *si)
1509 {
1510         struct oid_array ref = OID_ARRAY_INIT;
1511         int *status;
1512         int i;
1513         struct ref *r;
1514
1515         if (args->deepen && alternate_shallow_file) {
1516                 if (*alternate_shallow_file == '\0') { /* --unshallow */
1517                         unlink_or_warn(git_path_shallow(the_repository));
1518                         rollback_lock_file(&shallow_lock);
1519                 } else
1520                         commit_lock_file(&shallow_lock);
1521                 return;
1522         }
1523
1524         if (!si->shallow || !si->shallow->nr)
1525                 return;
1526
1527         if (args->cloning) {
1528                 /*
1529                  * remote is shallow, but this is a clone, there are
1530                  * no objects in repo to worry about. Accept any
1531                  * shallow points that exist in the pack (iow in repo
1532                  * after get_pack() and reprepare_packed_git())
1533                  */
1534                 struct oid_array extra = OID_ARRAY_INIT;
1535                 struct object_id *oid = si->shallow->oid;
1536                 for (i = 0; i < si->shallow->nr; i++)
1537                         if (has_object_file(&oid[i]))
1538                                 oid_array_append(&extra, &oid[i]);
1539                 if (extra.nr) {
1540                         setup_alternate_shallow(&shallow_lock,
1541                                                 &alternate_shallow_file,
1542                                                 &extra);
1543                         commit_lock_file(&shallow_lock);
1544                 }
1545                 oid_array_clear(&extra);
1546                 return;
1547         }
1548
1549         if (!si->nr_ours && !si->nr_theirs)
1550                 return;
1551
1552         remove_nonexistent_theirs_shallow(si);
1553         if (!si->nr_ours && !si->nr_theirs)
1554                 return;
1555         for (r = refs; r; r = r->next)
1556                 oid_array_append(&ref, &r->old_oid);
1557         si->ref = &ref;
1558
1559         if (args->update_shallow) {
1560                 /*
1561                  * remote is also shallow, .git/shallow may be updated
1562                  * so all refs can be accepted. Make sure we only add
1563                  * shallow roots that are actually reachable from new
1564                  * refs.
1565                  */
1566                 struct oid_array extra = OID_ARRAY_INIT;
1567                 struct object_id *oid = si->shallow->oid;
1568                 assign_shallow_commits_to_refs(si, NULL, NULL);
1569                 if (!si->nr_ours && !si->nr_theirs) {
1570                         oid_array_clear(&ref);
1571                         return;
1572                 }
1573                 for (i = 0; i < si->nr_ours; i++)
1574                         oid_array_append(&extra, &oid[si->ours[i]]);
1575                 for (i = 0; i < si->nr_theirs; i++)
1576                         oid_array_append(&extra, &oid[si->theirs[i]]);
1577                 setup_alternate_shallow(&shallow_lock,
1578                                         &alternate_shallow_file,
1579                                         &extra);
1580                 commit_lock_file(&shallow_lock);
1581                 oid_array_clear(&extra);
1582                 oid_array_clear(&ref);
1583                 return;
1584         }
1585
1586         /*
1587          * remote is also shallow, check what ref is safe to update
1588          * without updating .git/shallow
1589          */
1590         status = xcalloc(ref.nr, sizeof(*status));
1591         assign_shallow_commits_to_refs(si, NULL, status);
1592         if (si->nr_ours || si->nr_theirs) {
1593                 for (r = refs, i = 0; r; r = r->next, i++)
1594                         if (status[i])
1595                                 r->status = REF_STATUS_REJECT_SHALLOW;
1596         }
1597         free(status);
1598         oid_array_clear(&ref);
1599 }
1600
1601 static int iterate_ref_map(void *cb_data, struct object_id *oid)
1602 {
1603         struct ref **rm = cb_data;
1604         struct ref *ref = *rm;
1605
1606         if (!ref)
1607                 return -1; /* end of the list */
1608         *rm = ref->next;
1609         oidcpy(oid, &ref->old_oid);
1610         return 0;
1611 }
1612
1613 struct ref *fetch_pack(struct fetch_pack_args *args,
1614                        int fd[], struct child_process *conn,
1615                        const struct ref *ref,
1616                        const char *dest,
1617                        struct ref **sought, int nr_sought,
1618                        struct oid_array *shallow,
1619                        char **pack_lockfile,
1620                        enum protocol_version version)
1621 {
1622         struct ref *ref_cpy;
1623         struct shallow_info si;
1624
1625         fetch_pack_setup();
1626         if (nr_sought)
1627                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1628
1629         if (!ref) {
1630                 packet_flush(fd[1]);
1631                 die(_("no matching remote head"));
1632         }
1633         prepare_shallow_info(&si, shallow);
1634         if (version == protocol_v2)
1635                 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1636                                            pack_lockfile);
1637         else
1638                 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1639                                         &si, pack_lockfile);
1640         reprepare_packed_git(the_repository);
1641
1642         if (!args->cloning && args->deepen) {
1643                 struct check_connected_options opt = CHECK_CONNECTED_INIT;
1644                 struct ref *iterator = ref_cpy;
1645                 opt.shallow_file = alternate_shallow_file;
1646                 if (args->deepen)
1647                         opt.is_deepening_fetch = 1;
1648                 if (check_connected(iterate_ref_map, &iterator, &opt)) {
1649                         error(_("remote did not send all necessary objects"));
1650                         free_refs(ref_cpy);
1651                         ref_cpy = NULL;
1652                         rollback_lock_file(&shallow_lock);
1653                         goto cleanup;
1654                 }
1655                 args->connectivity_checked = 1;
1656         }
1657
1658         update_shallow(args, ref_cpy, &si);
1659 cleanup:
1660         clear_shallow_info(&si);
1661         return ref_cpy;
1662 }
1663
1664 int report_unmatched_refs(struct ref **sought, int nr_sought)
1665 {
1666         int i, ret = 0;
1667
1668         for (i = 0; i < nr_sought; i++) {
1669                 if (!sought[i])
1670                         continue;
1671                 switch (sought[i]->match_status) {
1672                 case REF_MATCHED:
1673                         continue;
1674                 case REF_NOT_MATCHED:
1675                         error(_("no such remote ref %s"), sought[i]->name);
1676                         break;
1677                 case REF_UNADVERTISED_NOT_ALLOWED:
1678                         error(_("Server does not allow request for unadvertised object %s"),
1679                               sought[i]->name);
1680                         break;
1681                 }
1682                 ret = 1;
1683         }
1684         return ret;
1685 }