git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "repository.h"
   3 #include "config.h"
   4 #include "lockfile.h"
   5 #include "refs.h"
   6 #include "pkt-line.h"
   7 #include "commit.h"
   8 #include "tag.h"
   9 #include "exec-cmd.h"
  10 #include "pack.h"
  11 #include "sideband.h"
  12 #include "fetch-pack.h"
  13 #include "remote.h"
  14 #include "run-command.h"
  15 #include "connect.h"
  16 #include "transport.h"
  17 #include "version.h"
  18 #include "prio-queue.h"
  19 #include "sha1-array.h"
  20 #include "oidset.h"
  21 #include "packfile.h"
  22
  23 static int transfer_unpack_limit = -1;
  24 static int fetch_unpack_limit = -1;
  25 static int unpack_limit = 100;
  26 static int prefer_ofs_delta = 1;
  27 static int no_done;
  28 static int deepen_since_ok;
  29 static int deepen_not_ok;
  30 static int fetch_fsck_objects = -1;
  31 static int transfer_fsck_objects = -1;
  32 static int agent_supported;
  33 static int server_supports_filtering;
  34 static struct lock_file shallow_lock;
  35 static const char *alternate_shallow_file;
  36
  37 /* Remember to update object flag allocation in object.h */
  38 #define COMPLETE        (1U << 0)
  39 #define COMMON          (1U << 1)
  40 #define COMMON_REF      (1U << 2)
  41 #define SEEN            (1U << 3)
  42 #define POPPED          (1U << 4)
  43 #define ALTERNATE       (1U << 5)
  44
  45 static int marked;
  46
  47 /*
  48  * After sending this many "have"s if we do not get any new ACK , we
  49  * give up traversing our history.
  50  */
  51 #define MAX_IN_VAIN 256
  52
  53 struct negotiation_state {
  54         struct prio_queue rev_list;
  55         int non_common_revs;
  56 };
  57
  58 static int multi_ack, use_sideband;
  59 /* Allow specifying sha1 if it is a ref tip. */
  60 #define ALLOW_TIP_SHA1  01
  61 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  62 #define ALLOW_REACHABLE_SHA1    02
  63 static unsigned int allow_unadvertised_object_request;
  64
  65 __attribute__((format (printf, 2, 3)))
  66 static inline void print_verbose(const struct fetch_pack_args *args,
  67                                  const char *fmt, ...)
  68 {
  69         va_list params;
  70
  71         if (!args->verbose)
  72                 return;
  73
  74         va_start(params, fmt);
  75         vfprintf(stderr, fmt, params);
  76         va_end(params);
  77         fputc('\n', stderr);
  78 }
  79
  80 struct alternate_object_cache {
  81         struct object **items;
  82         size_t nr, alloc;
  83 };
  84
  85 static void cache_one_alternate(const char *refname,
  86                                 const struct object_id *oid,
  87                                 void *vcache)
  88 {
  89         struct alternate_object_cache *cache = vcache;
  90         struct object *obj = parse_object(oid);
  91
  92         if (!obj || (obj->flags & ALTERNATE))
  93                 return;
  94
  95         obj->flags |= ALTERNATE;
  96         ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  97         cache->items[cache->nr++] = obj;
  98 }
  99
 100 static void for_each_cached_alternate(struct negotiation_state *ns,
 101                                       void (*cb)(struct negotiation_state *,
 102                                                  struct object *))
 103 {
 104         static int initialized;
 105         static struct alternate_object_cache cache;
 106         size_t i;
 107
 108         if (!initialized) {
 109                 for_each_alternate_ref(cache_one_alternate, &cache);
 110                 initialized = 1;
 111         }
 112
 113         for (i = 0; i < cache.nr; i++)
 114                 cb(ns, cache.items[i]);
 115 }
 116
 117 static void rev_list_push(struct negotiation_state *ns,
 118                           struct commit *commit, int mark)
 119 {
 120         if (!(commit->object.flags & mark)) {
 121                 commit->object.flags |= mark;
 122
 123                 if (parse_commit(commit))
 124                         return;
 125
 126                 prio_queue_put(&ns->rev_list, commit);
 127
 128                 if (!(commit->object.flags & COMMON))
 129                         ns->non_common_revs++;
 130         }
 131 }
 132
 133 static int rev_list_insert_ref(struct negotiation_state *ns,
 134                                const char *refname,
 135                                const struct object_id *oid)
 136 {
 137         struct object *o = deref_tag(parse_object(oid), refname, 0);
 138
 139         if (o && o->type == OBJ_COMMIT)
 140                 rev_list_push(ns, (struct commit *)o, SEEN);
 141
 142         return 0;
 143 }
 144
 145 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 146                                    int flag, void *cb_data)
 147 {
 148         return rev_list_insert_ref(cb_data, refname, oid);
 149 }
 150
 151 static int clear_marks(const char *refname, const struct object_id *oid,
 152                        int flag, void *cb_data)
 153 {
 154         struct object *o = deref_tag(parse_object(oid), refname, 0);
 155
 156         if (o && o->type == OBJ_COMMIT)
 157                 clear_commit_marks((struct commit *)o,
 158                                    COMMON | COMMON_REF | SEEN | POPPED);
 159         return 0;
 160 }
 161
 162 /*
 163    This function marks a rev and its ancestors as common.
 164    In some cases, it is desirable to mark only the ancestors (for example
 165    when only the server does not yet know that they are common).
 166 */
 167
 168 static void mark_common(struct negotiation_state *ns, struct commit *commit,
 169                 int ancestors_only, int dont_parse)
 170 {
 171         if (commit != NULL && !(commit->object.flags & COMMON)) {
 172                 struct object *o = (struct object *)commit;
 173
 174                 if (!ancestors_only)
 175                         o->flags |= COMMON;
 176
 177                 if (!(o->flags & SEEN))
 178                         rev_list_push(ns, commit, SEEN);
 179                 else {
 180                         struct commit_list *parents;
 181
 182                         if (!ancestors_only && !(o->flags & POPPED))
 183                                 ns->non_common_revs--;
 184                         if (!o->parsed && !dont_parse)
 185                                 if (parse_commit(commit))
 186                                         return;
 187
 188                         for (parents = commit->parents;
 189                                         parents;
 190                                         parents = parents->next)
 191                                 mark_common(ns, parents->item, 0,
 192                                             dont_parse);
 193                 }
 194         }
 195 }
 196
 197 /*
 198   Get the next rev to send, ignoring the common.
 199 */
 200
 201 static const struct object_id *get_rev(struct negotiation_state *ns)
 202 {
 203         struct commit *commit = NULL;
 204
 205         while (commit == NULL) {
 206                 unsigned int mark;
 207                 struct commit_list *parents;
 208
 209                 if (ns->rev_list.nr == 0 || ns->non_common_revs == 0)
 210                         return NULL;
 211
 212                 commit = prio_queue_get(&ns->rev_list);
 213                 parse_commit(commit);
 214                 parents = commit->parents;
 215
 216                 commit->object.flags |= POPPED;
 217                 if (!(commit->object.flags & COMMON))
 218                         ns->non_common_revs--;
 219
 220                 if (commit->object.flags & COMMON) {
 221                         /* do not send "have", and ignore ancestors */
 222                         commit = NULL;
 223                         mark = COMMON | SEEN;
 224                 } else if (commit->object.flags & COMMON_REF)
 225                         /* send "have", and ignore ancestors */
 226                         mark = COMMON | SEEN;
 227                 else
 228                         /* send "have", also for its ancestors */
 229                         mark = SEEN;
 230
 231                 while (parents) {
 232                         if (!(parents->item->object.flags & SEEN))
 233                                 rev_list_push(ns, parents->item, mark);
 234                         if (mark & COMMON)
 235                                 mark_common(ns, parents->item, 1, 0);
 236                         parents = parents->next;
 237                 }
 238         }
 239
 240         return &commit->object.oid;
 241 }
 242
 243 enum ack_type {
 244         NAK = 0,
 245         ACK,
 246         ACK_continue,
 247         ACK_common,
 248         ACK_ready
 249 };
 250
 251 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 252 {
 253         if (args->stateless_rpc && args->deepen) {
 254                 /* If we sent a depth we will get back "duplicate"
 255                  * shallow and unshallow commands every time there
 256                  * is a block of have lines exchanged.
 257                  */
 258                 char *line;
 259                 while ((line = packet_read_line(fd, NULL))) {
 260                         if (starts_with(line, "shallow "))
 261                                 continue;
 262                         if (starts_with(line, "unshallow "))
 263                                 continue;
 264                         die(_("git fetch-pack: expected shallow list"));
 265                 }
 266         }
 267 }
 268
 269 static enum ack_type get_ack(int fd, struct object_id *result_oid)
 270 {
 271         int len;
 272         char *line = packet_read_line(fd, &len);
 273         const char *arg;
 274
 275         if (!line)
 276                 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
 277         if (!strcmp(line, "NAK"))
 278                 return NAK;
 279         if (skip_prefix(line, "ACK ", &arg)) {
 280                 if (!get_oid_hex(arg, result_oid)) {
 281                         arg += 40;
 282                         len -= arg - line;
 283                         if (len < 1)
 284                                 return ACK;
 285                         if (strstr(arg, "continue"))
 286                                 return ACK_continue;
 287                         if (strstr(arg, "common"))
 288                                 return ACK_common;
 289                         if (strstr(arg, "ready"))
 290                                 return ACK_ready;
 291                         return ACK;
 292                 }
 293         }
 294         if (skip_prefix(line, "ERR ", &arg))
 295                 die(_("remote error: %s"), arg);
 296         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 297 }
 298
 299 static void send_request(struct fetch_pack_args *args,
 300                          int fd, struct strbuf *buf)
 301 {
 302         if (args->stateless_rpc) {
 303                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 304                 packet_flush(fd);
 305         } else
 306                 write_or_die(fd, buf->buf, buf->len);
 307 }
 308
 309 static void insert_one_alternate_object(struct negotiation_state *ns,
 310                                         struct object *obj)
 311 {
 312         rev_list_insert_ref(ns, NULL, &obj->oid);
 313 }
 314
 315 #define INITIAL_FLUSH 16
 316 #define PIPESAFE_FLUSH 32
 317 #define LARGE_FLUSH 16384
 318
 319 static int next_flush(int stateless_rpc, int count)
 320 {
 321         if (stateless_rpc) {
 322                 if (count < LARGE_FLUSH)
 323                         count <<= 1;
 324                 else
 325                         count = count * 11 / 10;
 326         } else {
 327                 if (count < PIPESAFE_FLUSH)
 328                         count <<= 1;
 329                 else
 330                         count += PIPESAFE_FLUSH;
 331         }
 332         return count;
 333 }
 334
 335 static int find_common(struct negotiation_state *ns,
 336                        struct fetch_pack_args *args,
 337                        int fd[2], struct object_id *result_oid,
 338                        struct ref *refs)
 339 {
 340         int fetching;
 341         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 342         const struct object_id *oid;
 343         unsigned in_vain = 0;
 344         int got_continue = 0;
 345         int got_ready = 0;
 346         struct strbuf req_buf = STRBUF_INIT;
 347         size_t state_len = 0;
 348
 349         if (args->stateless_rpc && multi_ack == 1)
 350                 die(_("--stateless-rpc requires multi_ack_detailed"));
 351
 352         for_each_ref(rev_list_insert_ref_oid, ns);
 353         for_each_cached_alternate(ns, insert_one_alternate_object);
 354
 355         fetching = 0;
 356         for ( ; refs ; refs = refs->next) {
 357                 struct object_id *remote = &refs->old_oid;
 358                 const char *remote_hex;
 359                 struct object *o;
 360
 361                 /*
 362                  * If that object is complete (i.e. it is an ancestor of a
 363                  * local ref), we tell them we have it but do not have to
 364                  * tell them about its ancestors, which they already know
 365                  * about.
 366                  *
 367                  * We use lookup_object here because we are only
 368                  * interested in the case we *know* the object is
 369                  * reachable and we have already scanned it.
 370                  */
 371                 if (((o = lookup_object(remote->hash)) != NULL) &&
 372                                 (o->flags & COMPLETE)) {
 373                         continue;
 374                 }
 375
 376                 remote_hex = oid_to_hex(remote);
 377                 if (!fetching) {
 378                         struct strbuf c = STRBUF_INIT;
 379                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 380                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 381                         if (no_done)            strbuf_addstr(&c, " no-done");
 382                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 383                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 384                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 385                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 386                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 387                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 388                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 389                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 390                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 391                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 392                                                             git_user_agent_sanitized());
 393                         if (args->filter_options.choice)
 394                                 strbuf_addstr(&c, " filter");
 395                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 396                         strbuf_release(&c);
 397                 } else
 398                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 399                 fetching++;
 400         }
 401
 402         if (!fetching) {
 403                 strbuf_release(&req_buf);
 404                 packet_flush(fd[1]);
 405                 return 1;
 406         }
 407
 408         if (is_repository_shallow())
 409                 write_shallow_commits(&req_buf, 1, NULL);
 410         if (args->depth > 0)
 411                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 412         if (args->deepen_since) {
 413                 timestamp_t max_age = approxidate(args->deepen_since);
 414                 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 415         }
 416         if (args->deepen_not) {
 417                 int i;
 418                 for (i = 0; i < args->deepen_not->nr; i++) {
 419                         struct string_list_item *s = args->deepen_not->items + i;
 420                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 421                 }
 422         }
 423         if (server_supports_filtering && args->filter_options.choice)
 424                 packet_buf_write(&req_buf, "filter %s",
 425                                  args->filter_options.filter_spec);
 426         packet_buf_flush(&req_buf);
 427         state_len = req_buf.len;
 428
 429         if (args->deepen) {
 430                 char *line;
 431                 const char *arg;
 432                 struct object_id oid;
 433
 434                 send_request(args, fd[1], &req_buf);
 435                 while ((line = packet_read_line(fd[0], NULL))) {
 436                         if (skip_prefix(line, "shallow ", &arg)) {
 437                                 if (get_oid_hex(arg, &oid))
 438                                         die(_("invalid shallow line: %s"), line);
 439                                 register_shallow(&oid);
 440                                 continue;
 441                         }
 442                         if (skip_prefix(line, "unshallow ", &arg)) {
 443                                 if (get_oid_hex(arg, &oid))
 444                                         die(_("invalid unshallow line: %s"), line);
 445                                 if (!lookup_object(oid.hash))
 446                                         die(_("object not found: %s"), line);
 447                                 /* make sure that it is parsed as shallow */
 448                                 if (!parse_object(&oid))
 449                                         die(_("error in object: %s"), line);
 450                                 if (unregister_shallow(&oid))
 451                                         die(_("no shallow found: %s"), line);
 452                                 continue;
 453                         }
 454                         die(_("expected shallow/unshallow, got %s"), line);
 455                 }
 456         } else if (!args->stateless_rpc)
 457                 send_request(args, fd[1], &req_buf);
 458
 459         if (!args->stateless_rpc) {
 460                 /* If we aren't using the stateless-rpc interface
 461                  * we don't need to retain the headers.
 462                  */
 463                 strbuf_setlen(&req_buf, 0);
 464                 state_len = 0;
 465         }
 466
 467         flushes = 0;
 468         retval = -1;
 469         if (args->no_dependents)
 470                 goto done;
 471         while ((oid = get_rev(ns))) {
 472                 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 473                 print_verbose(args, "have %s", oid_to_hex(oid));
 474                 in_vain++;
 475                 if (flush_at <= ++count) {
 476                         int ack;
 477
 478                         packet_buf_flush(&req_buf);
 479                         send_request(args, fd[1], &req_buf);
 480                         strbuf_setlen(&req_buf, state_len);
 481                         flushes++;
 482                         flush_at = next_flush(args->stateless_rpc, count);
 483
 484                         /*
 485                          * We keep one window "ahead" of the other side, and
 486                          * will wait for an ACK only on the next one
 487                          */
 488                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 489                                 continue;
 490
 491                         consume_shallow_list(args, fd[0]);
 492                         do {
 493                                 ack = get_ack(fd[0], result_oid);
 494                                 if (ack)
 495                                         print_verbose(args, _("got %s %d %s"), "ack",
 496                                                       ack, oid_to_hex(result_oid));
 497                                 switch (ack) {
 498                                 case ACK:
 499                                         flushes = 0;
 500                                         multi_ack = 0;
 501                                         retval = 0;
 502                                         goto done;
 503                                 case ACK_common:
 504                                 case ACK_ready:
 505                                 case ACK_continue: {
 506                                         struct commit *commit =
 507                                                 lookup_commit(result_oid);
 508                                         if (!commit)
 509                                                 die(_("invalid commit %s"), oid_to_hex(result_oid));
 510                                         if (args->stateless_rpc
 511                                          && ack == ACK_common
 512                                          && !(commit->object.flags & COMMON)) {
 513                                                 /* We need to replay the have for this object
 514                                                  * on the next RPC request so the peer knows
 515                                                  * it is in common with us.
 516                                                  */
 517                                                 const char *hex = oid_to_hex(result_oid);
 518                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 519                                                 state_len = req_buf.len;
 520                                                 /*
 521                                                  * Reset in_vain because an ack
 522                                                  * for this commit has not been
 523                                                  * seen.
 524                                                  */
 525                                                 in_vain = 0;
 526                                         } else if (!args->stateless_rpc
 527                                                    || ack != ACK_common)
 528                                                 in_vain = 0;
 529                                         mark_common(ns, commit, 0, 1);
 530                                         retval = 0;
 531                                         got_continue = 1;
 532                                         if (ack == ACK_ready)
 533                                                 got_ready = 1;
 534                                         break;
 535                                         }
 536                                 }
 537                         } while (ack);
 538                         flushes--;
 539                         if (got_continue && MAX_IN_VAIN < in_vain) {
 540                                 print_verbose(args, _("giving up"));
 541                                 break; /* give up */
 542                         }
 543                         if (got_ready)
 544                                 break;
 545                 }
 546         }
 547 done:
 548         if (!got_ready || !no_done) {
 549                 packet_buf_write(&req_buf, "done\n");
 550                 send_request(args, fd[1], &req_buf);
 551         }
 552         print_verbose(args, _("done"));
 553         if (retval != 0) {
 554                 multi_ack = 0;
 555                 flushes++;
 556         }
 557         strbuf_release(&req_buf);
 558
 559         if (!got_ready || !no_done)
 560                 consume_shallow_list(args, fd[0]);
 561         while (flushes || multi_ack) {
 562                 int ack = get_ack(fd[0], result_oid);
 563                 if (ack) {
 564                         print_verbose(args, _("got %s (%d) %s"), "ack",
 565                                       ack, oid_to_hex(result_oid));
 566                         if (ack == ACK)
 567                                 return 0;
 568                         multi_ack = 1;
 569                         continue;
 570                 }
 571                 flushes--;
 572         }
 573         /* it is no error to fetch into a completely empty repo */
 574         return count ? retval : 0;
 575 }
 576
 577 static struct commit_list *complete;
 578
 579 static int mark_complete(const struct object_id *oid)
 580 {
 581         struct object *o = parse_object(oid);
 582
 583         while (o && o->type == OBJ_TAG) {
 584                 struct tag *t = (struct tag *) o;
 585                 if (!t->tagged)
 586                         break; /* broken repository */
 587                 o->flags |= COMPLETE;
 588                 o = parse_object(&t->tagged->oid);
 589         }
 590         if (o && o->type == OBJ_COMMIT) {
 591                 struct commit *commit = (struct commit *)o;
 592                 if (!(commit->object.flags & COMPLETE)) {
 593                         commit->object.flags |= COMPLETE;
 594                         commit_list_insert(commit, &complete);
 595                 }
 596         }
 597         return 0;
 598 }
 599
 600 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 601                              int flag, void *cb_data)
 602 {
 603         return mark_complete(oid);
 604 }
 605
 606 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 607                                          timestamp_t cutoff)
 608 {
 609         while (complete && cutoff <= complete->item->date) {
 610                 print_verbose(args, _("Marking %s as complete"),
 611                               oid_to_hex(&complete->item->object.oid));
 612                 pop_most_recent_commit(&complete, COMPLETE);
 613         }
 614 }
 615
 616 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 617 {
 618         for (; refs; refs = refs->next)
 619                 oidset_insert(oids, &refs->old_oid);
 620 }
 621
 622 static int tip_oids_contain(struct oidset *tip_oids,
 623                             struct ref *unmatched, struct ref *newlist,
 624                             const struct object_id *id)
 625 {
 626         /*
 627          * Note that this only looks at the ref lists the first time it's
 628          * called. This works out in filter_refs() because even though it may
 629          * add to "newlist" between calls, the additions will always be for
 630          * oids that are already in the set.
 631          */
 632         if (!tip_oids->map.map.tablesize) {
 633                 add_refs_to_oidset(tip_oids, unmatched);
 634                 add_refs_to_oidset(tip_oids, newlist);
 635         }
 636         return oidset_contains(tip_oids, id);
 637 }
 638
 639 static void filter_refs(struct fetch_pack_args *args,
 640                         struct ref **refs,
 641                         struct ref **sought, int nr_sought)
 642 {
 643         struct ref *newlist = NULL;
 644         struct ref **newtail = &newlist;
 645         struct ref *unmatched = NULL;
 646         struct ref *ref, *next;
 647         struct oidset tip_oids = OIDSET_INIT;
 648         int i;
 649
 650         i = 0;
 651         for (ref = *refs; ref; ref = next) {
 652                 int keep = 0;
 653                 next = ref->next;
 654
 655                 if (starts_with(ref->name, "refs/") &&
 656                     check_refname_format(ref->name, 0))
 657                         ; /* trash */
 658                 else {
 659                         while (i < nr_sought) {
 660                                 int cmp = strcmp(ref->name, sought[i]->name);
 661                                 if (cmp < 0)
 662                                         break; /* definitely do not have it */
 663                                 else if (cmp == 0) {
 664                                         keep = 1; /* definitely have it */
 665                                         sought[i]->match_status = REF_MATCHED;
 666                                 }
 667                                 i++;
 668                         }
 669                 }
 670
 671                 if (!keep && args->fetch_all &&
 672                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 673                         keep = 1;
 674
 675                 if (keep) {
 676                         *newtail = ref;
 677                         ref->next = NULL;
 678                         newtail = &ref->next;
 679                 } else {
 680                         ref->next = unmatched;
 681                         unmatched = ref;
 682                 }
 683         }
 684
 685         /* Append unmatched requests to the list */
 686         for (i = 0; i < nr_sought; i++) {
 687                 struct object_id oid;
 688                 const char *p;
 689
 690                 ref = sought[i];
 691                 if (ref->match_status != REF_NOT_MATCHED)
 692                         continue;
 693                 if (parse_oid_hex(ref->name, &oid, &p) ||
 694                     *p != '\0' ||
 695                     oidcmp(&oid, &ref->old_oid))
 696                         continue;
 697
 698                 if ((allow_unadvertised_object_request &
 699                      (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 700                     tip_oids_contain(&tip_oids, unmatched, newlist,
 701                                      &ref->old_oid)) {
 702                         ref->match_status = REF_MATCHED;
 703                         *newtail = copy_ref(ref);
 704                         newtail = &(*newtail)->next;
 705                 } else {
 706                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 707                 }
 708         }
 709
 710         oidset_clear(&tip_oids);
 711         for (ref = unmatched; ref; ref = next) {
 712                 next = ref->next;
 713                 free(ref);
 714         }
 715
 716         *refs = newlist;
 717 }
 718
 719 static void mark_alternate_complete(struct negotiation_state *unused,
 720                                     struct object *obj)
 721 {
 722         mark_complete(&obj->oid);
 723 }
 724
 725 struct loose_object_iter {
 726         struct oidset *loose_object_set;
 727         struct ref *refs;
 728 };
 729
 730 /*
 731  *  If the number of refs is not larger than the number of loose objects,
 732  *  this function stops inserting.
 733  */
 734 static int add_loose_objects_to_set(const struct object_id *oid,
 735                                     const char *path,
 736                                     void *data)
 737 {
 738         struct loose_object_iter *iter = data;
 739         oidset_insert(iter->loose_object_set, oid);
 740         if (iter->refs == NULL)
 741                 return 1;
 742
 743         iter->refs = iter->refs->next;
 744         return 0;
 745 }
 746
 747 /*
 748  * Mark recent commits available locally and reachable from a local ref as
 749  * COMPLETE. If args->no_dependents is false, also mark COMPLETE remote refs as
 750  * COMMON_REF (otherwise, we are not planning to participate in negotiation, and
 751  * thus do not need COMMON_REF marks).
 752  *
 753  * The cutoff time for recency is determined by this heuristic: it is the
 754  * earliest commit time of the objects in refs that are commits and that we know
 755  * the commit time of.
 756  */
 757 static void mark_complete_and_common_ref(struct negotiation_state *ns,
 758                                          struct fetch_pack_args *args,
 759                                          struct ref **refs)
 760 {
 761         struct ref *ref;
 762         int old_save_commit_buffer = save_commit_buffer;
 763         timestamp_t cutoff = 0;
 764         struct oidset loose_oid_set = OIDSET_INIT;
 765         int use_oidset = 0;
 766         struct loose_object_iter iter = {&loose_oid_set, *refs};
 767
 768         /* Enumerate all loose objects or know refs are not so many. */
 769         use_oidset = !for_each_loose_object(add_loose_objects_to_set,
 770                                             &iter, 0);
 771
 772         save_commit_buffer = 0;
 773
 774         for (ref = *refs; ref; ref = ref->next) {
 775                 struct object *o;
 776                 unsigned int flags = OBJECT_INFO_QUICK;
 777
 778                 if (use_oidset &&
 779                     !oidset_contains(&loose_oid_set, &ref->old_oid)) {
 780                         /*
 781                          * I know this does not exist in the loose form,
 782                          * so check if it exists in a non-loose form.
 783                          */
 784                         flags |= OBJECT_INFO_IGNORE_LOOSE;
 785                 }
 786
 787                 if (!has_object_file_with_flags(&ref->old_oid, flags))
 788                         continue;
 789                 o = parse_object(&ref->old_oid);
 790                 if (!o)
 791                         continue;
 792
 793                 /* We already have it -- which may mean that we were
 794                  * in sync with the other side at some time after
 795                  * that (it is OK if we guess wrong here).
 796                  */
 797                 if (o->type == OBJ_COMMIT) {
 798                         struct commit *commit = (struct commit *)o;
 799                         if (!cutoff || cutoff < commit->date)
 800                                 cutoff = commit->date;
 801                 }
 802         }
 803
 804         oidset_clear(&loose_oid_set);
 805
 806         if (!args->no_dependents) {
 807                 if (!args->deepen) {
 808                         for_each_ref(mark_complete_oid, NULL);
 809                         for_each_cached_alternate(NULL, mark_alternate_complete);
 810                         commit_list_sort_by_date(&complete);
 811                         if (cutoff)
 812                                 mark_recent_complete_commits(args, cutoff);
 813                 }
 814
 815                 /*
 816                  * Mark all complete remote refs as common refs.
 817                  * Don't mark them common yet; the server has to be told so first.
 818                  */
 819                 for (ref = *refs; ref; ref = ref->next) {
 820                         struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 821                                                      NULL, 0);
 822
 823                         if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 824                                 continue;
 825
 826                         if (!(o->flags & SEEN)) {
 827                                 rev_list_push(ns, (struct commit *)o,
 828                                               COMMON_REF | SEEN);
 829
 830                                 mark_common(ns, (struct commit *)o, 1, 1);
 831                         }
 832                 }
 833         }
 834
 835         save_commit_buffer = old_save_commit_buffer;
 836 }
 837
 838 /*
 839  * Returns 1 if every object pointed to by the given remote refs is available
 840  * locally and reachable from a local ref, and 0 otherwise.
 841  */
 842 static int everything_local(struct fetch_pack_args *args,
 843                             struct ref **refs)
 844 {
 845         struct ref *ref;
 846         int retval;
 847
 848         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 849                 const struct object_id *remote = &ref->old_oid;
 850                 struct object *o;
 851
 852                 o = lookup_object(remote->hash);
 853                 if (!o || !(o->flags & COMPLETE)) {
 854                         retval = 0;
 855                         print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 856                                       ref->name);
 857                         continue;
 858                 }
 859                 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 860                               ref->name);
 861         }
 862
 863         return retval;
 864 }
 865
 866 static int sideband_demux(int in, int out, void *data)
 867 {
 868         int *xd = data;
 869         int ret;
 870
 871         ret = recv_sideband("fetch-pack", xd[0], out);
 872         close(out);
 873         return ret;
 874 }
 875
 876 static int get_pack(struct fetch_pack_args *args,
 877                     int xd[2], char **pack_lockfile)
 878 {
 879         struct async demux;
 880         int do_keep = args->keep_pack;
 881         const char *cmd_name;
 882         struct pack_header header;
 883         int pass_header = 0;
 884         struct child_process cmd = CHILD_PROCESS_INIT;
 885         int ret;
 886
 887         memset(&demux, 0, sizeof(demux));
 888         if (use_sideband) {
 889                 /* xd[] is talking with upload-pack; subprocess reads from
 890                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 891                  * through demux->out.
 892                  */
 893                 demux.proc = sideband_demux;
 894                 demux.data = xd;
 895                 demux.out = -1;
 896                 demux.isolate_sigpipe = 1;
 897                 if (start_async(&demux))
 898                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 899         }
 900         else
 901                 demux.out = xd[0];
 902
 903         if (!args->keep_pack && unpack_limit) {
 904
 905                 if (read_pack_header(demux.out, &header))
 906                         die(_("protocol error: bad pack header"));
 907                 pass_header = 1;
 908                 if (ntohl(header.hdr_entries) < unpack_limit)
 909                         do_keep = 0;
 910                 else
 911                         do_keep = 1;
 912         }
 913
 914         if (alternate_shallow_file) {
 915                 argv_array_push(&cmd.args, "--shallow-file");
 916                 argv_array_push(&cmd.args, alternate_shallow_file);
 917         }
 918
 919         if (do_keep || args->from_promisor) {
 920                 if (pack_lockfile)
 921                         cmd.out = -1;
 922                 cmd_name = "index-pack";
 923                 argv_array_push(&cmd.args, cmd_name);
 924                 argv_array_push(&cmd.args, "--stdin");
 925                 if (!args->quiet && !args->no_progress)
 926                         argv_array_push(&cmd.args, "-v");
 927                 if (args->use_thin_pack)
 928                         argv_array_push(&cmd.args, "--fix-thin");
 929                 if (do_keep && (args->lock_pack || unpack_limit)) {
 930                         char hostname[HOST_NAME_MAX + 1];
 931                         if (xgethostname(hostname, sizeof(hostname)))
 932                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 933                         argv_array_pushf(&cmd.args,
 934                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 935                                         (uintmax_t)getpid(), hostname);
 936                 }
 937                 if (args->check_self_contained_and_connected)
 938                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 939                 if (args->from_promisor)
 940                         argv_array_push(&cmd.args, "--promisor");
 941         }
 942         else {
 943                 cmd_name = "unpack-objects";
 944                 argv_array_push(&cmd.args, cmd_name);
 945                 if (args->quiet || args->no_progress)
 946                         argv_array_push(&cmd.args, "-q");
 947                 args->check_self_contained_and_connected = 0;
 948         }
 949
 950         if (pass_header)
 951                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 952                                  ntohl(header.hdr_version),
 953                                  ntohl(header.hdr_entries));
 954         if (fetch_fsck_objects >= 0
 955             ? fetch_fsck_objects
 956             : transfer_fsck_objects >= 0
 957             ? transfer_fsck_objects
 958             : 0) {
 959                 if (args->from_promisor)
 960                         /*
 961                          * We cannot use --strict in index-pack because it
 962                          * checks both broken objects and links, but we only
 963                          * want to check for broken objects.
 964                          */
 965                         argv_array_push(&cmd.args, "--fsck-objects");
 966                 else
 967                         argv_array_push(&cmd.args, "--strict");
 968         }
 969
 970         cmd.in = demux.out;
 971         cmd.git_cmd = 1;
 972         if (start_command(&cmd))
 973                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 974         if (do_keep && pack_lockfile) {
 975                 *pack_lockfile = index_pack_lockfile(cmd.out);
 976                 close(cmd.out);
 977         }
 978
 979         if (!use_sideband)
 980                 /* Closed by start_command() */
 981                 xd[0] = -1;
 982
 983         ret = finish_command(&cmd);
 984         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 985                 args->self_contained_and_connected =
 986                         args->check_self_contained_and_connected &&
 987                         ret == 0;
 988         else
 989                 die(_("%s failed"), cmd_name);
 990         if (use_sideband && finish_async(&demux))
 991                 die(_("error in sideband demultiplexer"));
 992         return 0;
 993 }
 994
 995 static int cmp_ref_by_name(const void *a_, const void *b_)
 996 {
 997         const struct ref *a = *((const struct ref **)a_);
 998         const struct ref *b = *((const struct ref **)b_);
 999         return strcmp(a->name, b->name);
1000 }
1001
1002 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
1003                                  int fd[2],
1004                                  const struct ref *orig_ref,
1005                                  struct ref **sought, int nr_sought,
1006                                  struct shallow_info *si,
1007                                  char **pack_lockfile)
1008 {
1009         struct ref *ref = copy_ref_list(orig_ref);
1010         struct object_id oid;
1011         const char *agent_feature;
1012         int agent_len;
1013         struct negotiation_state ns = { { compare_commits_by_commit_date } };
1014
1015         sort_ref_list(&ref, ref_compare_name);
1016         QSORT(sought, nr_sought, cmp_ref_by_name);
1017
1018         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
1019                 die(_("Server does not support shallow clients"));
1020         if (args->depth > 0 || args->deepen_since || args->deepen_not)
1021                 args->deepen = 1;
1022         if (server_supports("multi_ack_detailed")) {
1023                 print_verbose(args, _("Server supports multi_ack_detailed"));
1024                 multi_ack = 2;
1025                 if (server_supports("no-done")) {
1026                         print_verbose(args, _("Server supports no-done"));
1027                         if (args->stateless_rpc)
1028                                 no_done = 1;
1029                 }
1030         }
1031         else if (server_supports("multi_ack")) {
1032                 print_verbose(args, _("Server supports multi_ack"));
1033                 multi_ack = 1;
1034         }
1035         if (server_supports("side-band-64k")) {
1036                 print_verbose(args, _("Server supports side-band-64k"));
1037                 use_sideband = 2;
1038         }
1039         else if (server_supports("side-band")) {
1040                 print_verbose(args, _("Server supports side-band"));
1041                 use_sideband = 1;
1042         }
1043         if (server_supports("allow-tip-sha1-in-want")) {
1044                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
1045                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1046         }
1047         if (server_supports("allow-reachable-sha1-in-want")) {
1048                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
1049                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1050         }
1051         if (!server_supports("thin-pack"))
1052                 args->use_thin_pack = 0;
1053         if (!server_supports("no-progress"))
1054                 args->no_progress = 0;
1055         if (!server_supports("include-tag"))
1056                 args->include_tag = 0;
1057         if (server_supports("ofs-delta"))
1058                 print_verbose(args, _("Server supports ofs-delta"));
1059         else
1060                 prefer_ofs_delta = 0;
1061
1062         if (server_supports("filter")) {
1063                 server_supports_filtering = 1;
1064                 print_verbose(args, _("Server supports filter"));
1065         } else if (args->filter_options.choice) {
1066                 warning("filtering not recognized by server, ignoring");
1067         }
1068
1069         if ((agent_feature = server_feature_value("agent", &agent_len))) {
1070                 agent_supported = 1;
1071                 if (agent_len)
1072                         print_verbose(args, _("Server version is %.*s"),
1073                                       agent_len, agent_feature);
1074         }
1075         if (server_supports("deepen-since"))
1076                 deepen_since_ok = 1;
1077         else if (args->deepen_since)
1078                 die(_("Server does not support --shallow-since"));
1079         if (server_supports("deepen-not"))
1080                 deepen_not_ok = 1;
1081         else if (args->deepen_not)
1082                 die(_("Server does not support --shallow-exclude"));
1083         if (!server_supports("deepen-relative") && args->deepen_relative)
1084                 die(_("Server does not support --deepen"));
1085
1086         if (marked)
1087                 for_each_ref(clear_marks, NULL);
1088         marked = 1;
1089         mark_complete_and_common_ref(&ns, args, &ref);
1090         filter_refs(args, &ref, sought, nr_sought);
1091         if (everything_local(args, &ref)) {
1092                 packet_flush(fd[1]);
1093                 goto all_done;
1094         }
1095         if (find_common(&ns, args, fd, &oid, ref) < 0)
1096                 if (!args->keep_pack)
1097                         /* When cloning, it is not unusual to have
1098                          * no common commit.
1099                          */
1100                         warning(_("no common commits"));
1101
1102         if (args->stateless_rpc)
1103                 packet_flush(fd[1]);
1104         if (args->deepen)
1105                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1106                                         NULL);
1107         else if (si->nr_ours || si->nr_theirs)
1108                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1109         else
1110                 alternate_shallow_file = NULL;
1111         if (get_pack(args, fd, pack_lockfile))
1112                 die(_("git fetch-pack: fetch failed."));
1113
1114  all_done:
1115         clear_prio_queue(&ns.rev_list);
1116         return ref;
1117 }
1118
1119 static void add_shallow_requests(struct strbuf *req_buf,
1120                                  const struct fetch_pack_args *args)
1121 {
1122         if (is_repository_shallow())
1123                 write_shallow_commits(req_buf, 1, NULL);
1124         if (args->depth > 0)
1125                 packet_buf_write(req_buf, "deepen %d", args->depth);
1126         if (args->deepen_since) {
1127                 timestamp_t max_age = approxidate(args->deepen_since);
1128                 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1129         }
1130         if (args->deepen_not) {
1131                 int i;
1132                 for (i = 0; i < args->deepen_not->nr; i++) {
1133                         struct string_list_item *s = args->deepen_not->items + i;
1134                         packet_buf_write(req_buf, "deepen-not %s", s->string);
1135                 }
1136         }
1137 }
1138
1139 static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1140 {
1141         for ( ; wants ; wants = wants->next) {
1142                 const struct object_id *remote = &wants->old_oid;
1143                 const char *remote_hex;
1144                 struct object *o;
1145
1146                 /*
1147                  * If that object is complete (i.e. it is an ancestor of a
1148                  * local ref), we tell them we have it but do not have to
1149                  * tell them about its ancestors, which they already know
1150                  * about.
1151                  *
1152                  * We use lookup_object here because we are only
1153                  * interested in the case we *know* the object is
1154                  * reachable and we have already scanned it.
1155                  */
1156                 if (((o = lookup_object(remote->hash)) != NULL) &&
1157                     (o->flags & COMPLETE)) {
1158                         continue;
1159                 }
1160
1161                 remote_hex = oid_to_hex(remote);
1162                 packet_buf_write(req_buf, "want %s\n", remote_hex);
1163         }
1164 }
1165
1166 static void add_common(struct strbuf *req_buf, struct oidset *common)
1167 {
1168         struct oidset_iter iter;
1169         const struct object_id *oid;
1170         oidset_iter_init(common, &iter);
1171
1172         while ((oid = oidset_iter_next(&iter))) {
1173                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1174         }
1175 }
1176
1177 static int add_haves(struct negotiation_state *ns, struct strbuf *req_buf,
1178                      int *haves_to_send, int *in_vain)
1179 {
1180         int ret = 0;
1181         int haves_added = 0;
1182         const struct object_id *oid;
1183
1184         while ((oid = get_rev(ns))) {
1185                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1186                 if (++haves_added >= *haves_to_send)
1187                         break;
1188         }
1189
1190         *in_vain += haves_added;
1191         if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1192                 /* Send Done */
1193                 packet_buf_write(req_buf, "done\n");
1194                 ret = 1;
1195         }
1196
1197         /* Increase haves to send on next round */
1198         *haves_to_send = next_flush(1, *haves_to_send);
1199
1200         return ret;
1201 }
1202
1203 static int send_fetch_request(struct negotiation_state *ns, int fd_out,
1204                               const struct fetch_pack_args *args,
1205                               const struct ref *wants, struct oidset *common,
1206                               int *haves_to_send, int *in_vain)
1207 {
1208         int ret = 0;
1209         struct strbuf req_buf = STRBUF_INIT;
1210
1211         if (server_supports_v2("fetch", 1))
1212                 packet_buf_write(&req_buf, "command=fetch");
1213         if (server_supports_v2("agent", 0))
1214                 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1215         if (args->server_options && args->server_options->nr &&
1216             server_supports_v2("server-option", 1)) {
1217                 int i;
1218                 for (i = 0; i < args->server_options->nr; i++)
1219                         packet_write_fmt(fd_out, "server-option=%s",
1220                                          args->server_options->items[i].string);
1221         }
1222
1223         packet_buf_delim(&req_buf);
1224         if (args->use_thin_pack)
1225                 packet_buf_write(&req_buf, "thin-pack");
1226         if (args->no_progress)
1227                 packet_buf_write(&req_buf, "no-progress");
1228         if (args->include_tag)
1229                 packet_buf_write(&req_buf, "include-tag");
1230         if (prefer_ofs_delta)
1231                 packet_buf_write(&req_buf, "ofs-delta");
1232
1233         /* Add shallow-info and deepen request */
1234         if (server_supports_feature("fetch", "shallow", 0))
1235                 add_shallow_requests(&req_buf, args);
1236         else if (is_repository_shallow() || args->deepen)
1237                 die(_("Server does not support shallow requests"));
1238
1239         /* Add filter */
1240         if (server_supports_feature("fetch", "filter", 0) &&
1241             args->filter_options.choice) {
1242                 print_verbose(args, _("Server supports filter"));
1243                 packet_buf_write(&req_buf, "filter %s",
1244                                  args->filter_options.filter_spec);
1245         } else if (args->filter_options.choice) {
1246                 warning("filtering not recognized by server, ignoring");
1247         }
1248
1249         /* add wants */
1250         add_wants(wants, &req_buf);
1251
1252         if (args->no_dependents) {
1253                 packet_buf_write(&req_buf, "done");
1254                 ret = 1;
1255         } else {
1256                 /* Add all of the common commits we've found in previous rounds */
1257                 add_common(&req_buf, common);
1258
1259                 /* Add initial haves */
1260                 ret = add_haves(ns, &req_buf, haves_to_send, in_vain);
1261         }
1262
1263         /* Send request */
1264         packet_buf_flush(&req_buf);
1265         write_or_die(fd_out, req_buf.buf, req_buf.len);
1266
1267         strbuf_release(&req_buf);
1268         return ret;
1269 }
1270
1271 /*
1272  * Processes a section header in a server's response and checks if it matches
1273  * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1274  * not consumed); if 0, the line will be consumed and the function will die if
1275  * the section header doesn't match what was expected.
1276  */
1277 static int process_section_header(struct packet_reader *reader,
1278                                   const char *section, int peek)
1279 {
1280         int ret;
1281
1282         if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1283                 die("error reading section header '%s'", section);
1284
1285         ret = !strcmp(reader->line, section);
1286
1287         if (!peek) {
1288                 if (!ret)
1289                         die("expected '%s', received '%s'",
1290                             section, reader->line);
1291                 packet_reader_read(reader);
1292         }
1293
1294         return ret;
1295 }
1296
1297 static int process_acks(struct negotiation_state *ns,
1298                         struct packet_reader *reader,
1299                         struct oidset *common)
1300 {
1301         /* received */
1302         int received_ready = 0;
1303         int received_ack = 0;
1304
1305         process_section_header(reader, "acknowledgments", 0);
1306         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1307                 const char *arg;
1308
1309                 if (!strcmp(reader->line, "NAK"))
1310                         continue;
1311
1312                 if (skip_prefix(reader->line, "ACK ", &arg)) {
1313                         struct object_id oid;
1314                         if (!get_oid_hex(arg, &oid)) {
1315                                 struct commit *commit;
1316                                 oidset_insert(common, &oid);
1317                                 commit = lookup_commit(&oid);
1318                                 mark_common(ns, commit, 0, 1);
1319                         }
1320                         continue;
1321                 }
1322
1323                 if (!strcmp(reader->line, "ready")) {
1324                         received_ready = 1;
1325                         continue;
1326                 }
1327
1328                 die("unexpected acknowledgment line: '%s'", reader->line);
1329         }
1330
1331         if (reader->status != PACKET_READ_FLUSH &&
1332             reader->status != PACKET_READ_DELIM)
1333                 die("error processing acks: %d", reader->status);
1334
1335         /* return 0 if no common, 1 if there are common, or 2 if ready */
1336         return received_ready ? 2 : (received_ack ? 1 : 0);
1337 }
1338
1339 static void receive_shallow_info(struct fetch_pack_args *args,
1340                                  struct packet_reader *reader)
1341 {
1342         process_section_header(reader, "shallow-info", 0);
1343         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1344                 const char *arg;
1345                 struct object_id oid;
1346
1347                 if (skip_prefix(reader->line, "shallow ", &arg)) {
1348                         if (get_oid_hex(arg, &oid))
1349                                 die(_("invalid shallow line: %s"), reader->line);
1350                         register_shallow(&oid);
1351                         continue;
1352                 }
1353                 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1354                         if (get_oid_hex(arg, &oid))
1355                                 die(_("invalid unshallow line: %s"), reader->line);
1356                         if (!lookup_object(oid.hash))
1357                                 die(_("object not found: %s"), reader->line);
1358                         /* make sure that it is parsed as shallow */
1359                         if (!parse_object(&oid))
1360                                 die(_("error in object: %s"), reader->line);
1361                         if (unregister_shallow(&oid))
1362                                 die(_("no shallow found: %s"), reader->line);
1363                         continue;
1364                 }
1365                 die(_("expected shallow/unshallow, got %s"), reader->line);
1366         }
1367
1368         if (reader->status != PACKET_READ_FLUSH &&
1369             reader->status != PACKET_READ_DELIM)
1370                 die("error processing shallow info: %d", reader->status);
1371
1372         setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1373         args->deepen = 1;
1374 }
1375
1376 enum fetch_state {
1377         FETCH_CHECK_LOCAL = 0,
1378         FETCH_SEND_REQUEST,
1379         FETCH_PROCESS_ACKS,
1380         FETCH_GET_PACK,
1381         FETCH_DONE,
1382 };
1383
1384 static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1385                                     int fd[2],
1386                                     const struct ref *orig_ref,
1387                                     struct ref **sought, int nr_sought,
1388                                     char **pack_lockfile)
1389 {
1390         struct ref *ref = copy_ref_list(orig_ref);
1391         enum fetch_state state = FETCH_CHECK_LOCAL;
1392         struct oidset common = OIDSET_INIT;
1393         struct packet_reader reader;
1394         int in_vain = 0;
1395         int haves_to_send = INITIAL_FLUSH;
1396         struct negotiation_state ns = { { compare_commits_by_commit_date } };
1397         packet_reader_init(&reader, fd[0], NULL, 0,
1398                            PACKET_READ_CHOMP_NEWLINE);
1399
1400         while (state != FETCH_DONE) {
1401                 switch (state) {
1402                 case FETCH_CHECK_LOCAL:
1403                         sort_ref_list(&ref, ref_compare_name);
1404                         QSORT(sought, nr_sought, cmp_ref_by_name);
1405
1406                         /* v2 supports these by default */
1407                         allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1408                         use_sideband = 2;
1409                         if (args->depth > 0 || args->deepen_since || args->deepen_not)
1410                                 args->deepen = 1;
1411
1412                         if (marked)
1413                                 for_each_ref(clear_marks, NULL);
1414                         marked = 1;
1415
1416                         /* Filter 'ref' by 'sought' and those that aren't local */
1417                         mark_complete_and_common_ref(&ns, args, &ref);
1418                         filter_refs(args, &ref, sought, nr_sought);
1419                         if (everything_local(args, &ref))
1420                                 state = FETCH_DONE;
1421                         else
1422                                 state = FETCH_SEND_REQUEST;
1423
1424                         for_each_ref(rev_list_insert_ref_oid, &ns);
1425                         for_each_cached_alternate(&ns,
1426                                                   insert_one_alternate_object);
1427                         break;
1428                 case FETCH_SEND_REQUEST:
1429                         if (send_fetch_request(&ns, fd[1], args, ref, &common,
1430                                                &haves_to_send, &in_vain))
1431                                 state = FETCH_GET_PACK;
1432                         else
1433                                 state = FETCH_PROCESS_ACKS;
1434                         break;
1435                 case FETCH_PROCESS_ACKS:
1436                         /* Process ACKs/NAKs */
1437                         switch (process_acks(&ns, &reader, &common)) {
1438                         case 2:
1439                                 state = FETCH_GET_PACK;
1440                                 break;
1441                         case 1:
1442                                 in_vain = 0;
1443                                 /* fallthrough */
1444                         default:
1445                                 state = FETCH_SEND_REQUEST;
1446                                 break;
1447                         }
1448                         break;
1449                 case FETCH_GET_PACK:
1450                         /* Check for shallow-info section */
1451                         if (process_section_header(&reader, "shallow-info", 1))
1452                                 receive_shallow_info(args, &reader);
1453
1454                         /* get the pack */
1455                         process_section_header(&reader, "packfile", 0);
1456                         if (get_pack(args, fd, pack_lockfile))
1457                                 die(_("git fetch-pack: fetch failed."));
1458
1459                         state = FETCH_DONE;
1460                         break;
1461                 case FETCH_DONE:
1462                         continue;
1463                 }
1464         }
1465
1466         clear_prio_queue(&ns.rev_list);
1467         oidset_clear(&common);
1468         return ref;
1469 }
1470
1471 static void fetch_pack_config(void)
1472 {
1473         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1474         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1475         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1476         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1477         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1478
1479         git_config(git_default_config, NULL);
1480 }
1481
1482 static void fetch_pack_setup(void)
1483 {
1484         static int did_setup;
1485         if (did_setup)
1486                 return;
1487         fetch_pack_config();
1488         if (0 <= transfer_unpack_limit)
1489                 unpack_limit = transfer_unpack_limit;
1490         else if (0 <= fetch_unpack_limit)
1491                 unpack_limit = fetch_unpack_limit;
1492         did_setup = 1;
1493 }
1494
1495 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1496 {
1497         struct string_list names = STRING_LIST_INIT_NODUP;
1498         int src, dst;
1499
1500         for (src = dst = 0; src < nr; src++) {
1501                 struct string_list_item *item;
1502                 item = string_list_insert(&names, ref[src]->name);
1503                 if (item->util)
1504                         continue; /* already have it */
1505                 item->util = ref[src];
1506                 if (src != dst)
1507                         ref[dst] = ref[src];
1508                 dst++;
1509         }
1510         for (src = dst; src < nr; src++)
1511                 ref[src] = NULL;
1512         string_list_clear(&names, 0);
1513         return dst;
1514 }
1515
1516 static void update_shallow(struct fetch_pack_args *args,
1517                            struct ref **sought, int nr_sought,
1518                            struct shallow_info *si)
1519 {
1520         struct oid_array ref = OID_ARRAY_INIT;
1521         int *status;
1522         int i;
1523
1524         if (args->deepen && alternate_shallow_file) {
1525                 if (*alternate_shallow_file == '\0') { /* --unshallow */
1526                         unlink_or_warn(git_path_shallow());
1527                         rollback_lock_file(&shallow_lock);
1528                 } else
1529                         commit_lock_file(&shallow_lock);
1530                 return;
1531         }
1532
1533         if (!si->shallow || !si->shallow->nr)
1534                 return;
1535
1536         if (args->cloning) {
1537                 /*
1538                  * remote is shallow, but this is a clone, there are
1539                  * no objects in repo to worry about. Accept any
1540                  * shallow points that exist in the pack (iow in repo
1541                  * after get_pack() and reprepare_packed_git())
1542                  */
1543                 struct oid_array extra = OID_ARRAY_INIT;
1544                 struct object_id *oid = si->shallow->oid;
1545                 for (i = 0; i < si->shallow->nr; i++)
1546                         if (has_object_file(&oid[i]))
1547                                 oid_array_append(&extra, &oid[i]);
1548                 if (extra.nr) {
1549                         setup_alternate_shallow(&shallow_lock,
1550                                                 &alternate_shallow_file,
1551                                                 &extra);
1552                         commit_lock_file(&shallow_lock);
1553                 }
1554                 oid_array_clear(&extra);
1555                 return;
1556         }
1557
1558         if (!si->nr_ours && !si->nr_theirs)
1559                 return;
1560
1561         remove_nonexistent_theirs_shallow(si);
1562         if (!si->nr_ours && !si->nr_theirs)
1563                 return;
1564         for (i = 0; i < nr_sought; i++)
1565                 oid_array_append(&ref, &sought[i]->old_oid);
1566         si->ref = &ref;
1567
1568         if (args->update_shallow) {
1569                 /*
1570                  * remote is also shallow, .git/shallow may be updated
1571                  * so all refs can be accepted. Make sure we only add
1572                  * shallow roots that are actually reachable from new
1573                  * refs.
1574                  */
1575                 struct oid_array extra = OID_ARRAY_INIT;
1576                 struct object_id *oid = si->shallow->oid;
1577                 assign_shallow_commits_to_refs(si, NULL, NULL);
1578                 if (!si->nr_ours && !si->nr_theirs) {
1579                         oid_array_clear(&ref);
1580                         return;
1581                 }
1582                 for (i = 0; i < si->nr_ours; i++)
1583                         oid_array_append(&extra, &oid[si->ours[i]]);
1584                 for (i = 0; i < si->nr_theirs; i++)
1585                         oid_array_append(&extra, &oid[si->theirs[i]]);
1586                 setup_alternate_shallow(&shallow_lock,
1587                                         &alternate_shallow_file,
1588                                         &extra);
1589                 commit_lock_file(&shallow_lock);
1590                 oid_array_clear(&extra);
1591                 oid_array_clear(&ref);
1592                 return;
1593         }
1594
1595         /*
1596          * remote is also shallow, check what ref is safe to update
1597          * without updating .git/shallow
1598          */
1599         status = xcalloc(nr_sought, sizeof(*status));
1600         assign_shallow_commits_to_refs(si, NULL, status);
1601         if (si->nr_ours || si->nr_theirs) {
1602                 for (i = 0; i < nr_sought; i++)
1603                         if (status[i])
1604                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1605         }
1606         free(status);
1607         oid_array_clear(&ref);
1608 }
1609
1610 struct ref *fetch_pack(struct fetch_pack_args *args,
1611                        int fd[], struct child_process *conn,
1612                        const struct ref *ref,
1613                        const char *dest,
1614                        struct ref **sought, int nr_sought,
1615                        struct oid_array *shallow,
1616                        char **pack_lockfile,
1617                        enum protocol_version version)
1618 {
1619         struct ref *ref_cpy;
1620         struct shallow_info si;
1621
1622         fetch_pack_setup();
1623         if (nr_sought)
1624                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1625
1626         if (!ref) {
1627                 packet_flush(fd[1]);
1628                 die(_("no matching remote head"));
1629         }
1630         prepare_shallow_info(&si, shallow);
1631         if (version == protocol_v2)
1632                 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1633                                            pack_lockfile);
1634         else
1635                 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1636                                         &si, pack_lockfile);
1637         reprepare_packed_git(the_repository);
1638         update_shallow(args, sought, nr_sought, &si);
1639         clear_shallow_info(&si);
1640         return ref_cpy;
1641 }
1642
1643 int report_unmatched_refs(struct ref **sought, int nr_sought)
1644 {
1645         int i, ret = 0;
1646
1647         for (i = 0; i < nr_sought; i++) {
1648                 if (!sought[i])
1649                         continue;
1650                 switch (sought[i]->match_status) {
1651                 case REF_MATCHED:
1652                         continue;
1653                 case REF_NOT_MATCHED:
1654                         error(_("no such remote ref %s"), sought[i]->name);
1655                         break;
1656                 case REF_UNADVERTISED_NOT_ALLOWED:
1657                         error(_("Server does not allow request for unadvertised object %s"),
1658                               sought[i]->name);
1659                         break;
1660                 }
1661                 ret = 1;
1662         }
1663         return ret;
1664 }