git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "repository.h"
   3 #include "config.h"
   4 #include "lockfile.h"
   5 #include "refs.h"
   6 #include "pkt-line.h"
   7 #include "commit.h"
   8 #include "tag.h"
   9 #include "exec-cmd.h"
  10 #include "pack.h"
  11 #include "sideband.h"
  12 #include "fetch-pack.h"
  13 #include "remote.h"
  14 #include "run-command.h"
  15 #include "connect.h"
  16 #include "transport.h"
  17 #include "version.h"
  18 #include "prio-queue.h"
  19 #include "sha1-array.h"
  20 #include "oidset.h"
  21 #include "packfile.h"
  22 #include "object-store.h"
  23
  24 static int transfer_unpack_limit = -1;
  25 static int fetch_unpack_limit = -1;
  26 static int unpack_limit = 100;
  27 static int prefer_ofs_delta = 1;
  28 static int no_done;
  29 static int deepen_since_ok;
  30 static int deepen_not_ok;
  31 static int fetch_fsck_objects = -1;
  32 static int transfer_fsck_objects = -1;
  33 static int agent_supported;
  34 static int server_supports_filtering;
  35 static struct lock_file shallow_lock;
  36 static const char *alternate_shallow_file;
  37
  38 /* Remember to update object flag allocation in object.h */
  39 #define COMPLETE        (1U << 0)
  40 #define COMMON          (1U << 1)
  41 #define COMMON_REF      (1U << 2)
  42 #define SEEN            (1U << 3)
  43 #define POPPED          (1U << 4)
  44 #define ALTERNATE       (1U << 5)
  45
  46 static int marked;
  47
  48 /*
  49  * After sending this many "have"s if we do not get any new ACK , we
  50  * give up traversing our history.
  51  */
  52 #define MAX_IN_VAIN 256
  53
  54 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  55 static int non_common_revs, multi_ack, use_sideband;
  56 /* Allow specifying sha1 if it is a ref tip. */
  57 #define ALLOW_TIP_SHA1  01
  58 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  59 #define ALLOW_REACHABLE_SHA1    02
  60 static unsigned int allow_unadvertised_object_request;
  61
  62 __attribute__((format (printf, 2, 3)))
  63 static inline void print_verbose(const struct fetch_pack_args *args,
  64                                  const char *fmt, ...)
  65 {
  66         va_list params;
  67
  68         if (!args->verbose)
  69                 return;
  70
  71         va_start(params, fmt);
  72         vfprintf(stderr, fmt, params);
  73         va_end(params);
  74         fputc('\n', stderr);
  75 }
  76
  77 struct alternate_object_cache {
  78         struct object **items;
  79         size_t nr, alloc;
  80 };
  81
  82 static void cache_one_alternate(const char *refname,
  83                                 const struct object_id *oid,
  84                                 void *vcache)
  85 {
  86         struct alternate_object_cache *cache = vcache;
  87         struct object *obj = parse_object(the_repository, oid);
  88
  89         if (!obj || (obj->flags & ALTERNATE))
  90                 return;
  91
  92         obj->flags |= ALTERNATE;
  93         ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  94         cache->items[cache->nr++] = obj;
  95 }
  96
  97 static void for_each_cached_alternate(void (*cb)(struct object *))
  98 {
  99         static int initialized;
 100         static struct alternate_object_cache cache;
 101         size_t i;
 102
 103         if (!initialized) {
 104                 for_each_alternate_ref(cache_one_alternate, &cache);
 105                 initialized = 1;
 106         }
 107
 108         for (i = 0; i < cache.nr; i++)
 109                 cb(cache.items[i]);
 110 }
 111
 112 static void rev_list_push(struct commit *commit, int mark)
 113 {
 114         if (!(commit->object.flags & mark)) {
 115                 commit->object.flags |= mark;
 116
 117                 if (parse_commit(commit))
 118                         return;
 119
 120                 prio_queue_put(&rev_list, commit);
 121
 122                 if (!(commit->object.flags & COMMON))
 123                         non_common_revs++;
 124         }
 125 }
 126
 127 static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 128 {
 129         struct object *o = deref_tag(parse_object(the_repository, oid),
 130                                      refname, 0);
 131
 132         if (o && o->type == OBJ_COMMIT)
 133                 rev_list_push((struct commit *)o, SEEN);
 134
 135         return 0;
 136 }
 137
 138 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 139                                    int flag, void *cb_data)
 140 {
 141         return rev_list_insert_ref(refname, oid);
 142 }
 143
 144 static int clear_marks(const char *refname, const struct object_id *oid,
 145                        int flag, void *cb_data)
 146 {
 147         struct object *o = deref_tag(parse_object(the_repository, oid),
 148                                      refname, 0);
 149
 150         if (o && o->type == OBJ_COMMIT)
 151                 clear_commit_marks((struct commit *)o,
 152                                    COMMON | COMMON_REF | SEEN | POPPED);
 153         return 0;
 154 }
 155
 156 /*
 157    This function marks a rev and its ancestors as common.
 158    In some cases, it is desirable to mark only the ancestors (for example
 159    when only the server does not yet know that they are common).
 160 */
 161
 162 static void mark_common(struct commit *commit,
 163                 int ancestors_only, int dont_parse)
 164 {
 165         if (commit != NULL && !(commit->object.flags & COMMON)) {
 166                 struct object *o = (struct object *)commit;
 167
 168                 if (!ancestors_only)
 169                         o->flags |= COMMON;
 170
 171                 if (!(o->flags & SEEN))
 172                         rev_list_push(commit, SEEN);
 173                 else {
 174                         struct commit_list *parents;
 175
 176                         if (!ancestors_only && !(o->flags & POPPED))
 177                                 non_common_revs--;
 178                         if (!o->parsed && !dont_parse)
 179                                 if (parse_commit(commit))
 180                                         return;
 181
 182                         for (parents = commit->parents;
 183                                         parents;
 184                                         parents = parents->next)
 185                                 mark_common(parents->item, 0, dont_parse);
 186                 }
 187         }
 188 }
 189
 190 /*
 191   Get the next rev to send, ignoring the common.
 192 */
 193
 194 static const struct object_id *get_rev(void)
 195 {
 196         struct commit *commit = NULL;
 197
 198         while (commit == NULL) {
 199                 unsigned int mark;
 200                 struct commit_list *parents;
 201
 202                 if (rev_list.nr == 0 || non_common_revs == 0)
 203                         return NULL;
 204
 205                 commit = prio_queue_get(&rev_list);
 206                 parse_commit(commit);
 207                 parents = commit->parents;
 208
 209                 commit->object.flags |= POPPED;
 210                 if (!(commit->object.flags & COMMON))
 211                         non_common_revs--;
 212
 213                 if (commit->object.flags & COMMON) {
 214                         /* do not send "have", and ignore ancestors */
 215                         commit = NULL;
 216                         mark = COMMON | SEEN;
 217                 } else if (commit->object.flags & COMMON_REF)
 218                         /* send "have", and ignore ancestors */
 219                         mark = COMMON | SEEN;
 220                 else
 221                         /* send "have", also for its ancestors */
 222                         mark = SEEN;
 223
 224                 while (parents) {
 225                         if (!(parents->item->object.flags & SEEN))
 226                                 rev_list_push(parents->item, mark);
 227                         if (mark & COMMON)
 228                                 mark_common(parents->item, 1, 0);
 229                         parents = parents->next;
 230                 }
 231         }
 232
 233         return &commit->object.oid;
 234 }
 235
 236 enum ack_type {
 237         NAK = 0,
 238         ACK,
 239         ACK_continue,
 240         ACK_common,
 241         ACK_ready
 242 };
 243
 244 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 245 {
 246         if (args->stateless_rpc && args->deepen) {
 247                 /* If we sent a depth we will get back "duplicate"
 248                  * shallow and unshallow commands every time there
 249                  * is a block of have lines exchanged.
 250                  */
 251                 char *line;
 252                 while ((line = packet_read_line(fd, NULL))) {
 253                         if (starts_with(line, "shallow "))
 254                                 continue;
 255                         if (starts_with(line, "unshallow "))
 256                                 continue;
 257                         die(_("git fetch-pack: expected shallow list"));
 258                 }
 259         }
 260 }
 261
 262 static enum ack_type get_ack(int fd, struct object_id *result_oid)
 263 {
 264         int len;
 265         char *line = packet_read_line(fd, &len);
 266         const char *arg;
 267
 268         if (!line)
 269                 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
 270         if (!strcmp(line, "NAK"))
 271                 return NAK;
 272         if (skip_prefix(line, "ACK ", &arg)) {
 273                 if (!get_oid_hex(arg, result_oid)) {
 274                         arg += 40;
 275                         len -= arg - line;
 276                         if (len < 1)
 277                                 return ACK;
 278                         if (strstr(arg, "continue"))
 279                                 return ACK_continue;
 280                         if (strstr(arg, "common"))
 281                                 return ACK_common;
 282                         if (strstr(arg, "ready"))
 283                                 return ACK_ready;
 284                         return ACK;
 285                 }
 286         }
 287         if (skip_prefix(line, "ERR ", &arg))
 288                 die(_("remote error: %s"), arg);
 289         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 290 }
 291
 292 static void send_request(struct fetch_pack_args *args,
 293                          int fd, struct strbuf *buf)
 294 {
 295         if (args->stateless_rpc) {
 296                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 297                 packet_flush(fd);
 298         } else
 299                 write_or_die(fd, buf->buf, buf->len);
 300 }
 301
 302 static void insert_one_alternate_object(struct object *obj)
 303 {
 304         rev_list_insert_ref(NULL, &obj->oid);
 305 }
 306
 307 #define INITIAL_FLUSH 16
 308 #define PIPESAFE_FLUSH 32
 309 #define LARGE_FLUSH 16384
 310
 311 static int next_flush(int stateless_rpc, int count)
 312 {
 313         if (stateless_rpc) {
 314                 if (count < LARGE_FLUSH)
 315                         count <<= 1;
 316                 else
 317                         count = count * 11 / 10;
 318         } else {
 319                 if (count < PIPESAFE_FLUSH)
 320                         count <<= 1;
 321                 else
 322                         count += PIPESAFE_FLUSH;
 323         }
 324         return count;
 325 }
 326
 327 static int find_common(struct fetch_pack_args *args,
 328                        int fd[2], struct object_id *result_oid,
 329                        struct ref *refs)
 330 {
 331         int fetching;
 332         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 333         const struct object_id *oid;
 334         unsigned in_vain = 0;
 335         int got_continue = 0;
 336         int got_ready = 0;
 337         struct strbuf req_buf = STRBUF_INIT;
 338         size_t state_len = 0;
 339
 340         if (args->stateless_rpc && multi_ack == 1)
 341                 die(_("--stateless-rpc requires multi_ack_detailed"));
 342         if (marked)
 343                 for_each_ref(clear_marks, NULL);
 344         marked = 1;
 345
 346         for_each_ref(rev_list_insert_ref_oid, NULL);
 347         for_each_cached_alternate(insert_one_alternate_object);
 348
 349         fetching = 0;
 350         for ( ; refs ; refs = refs->next) {
 351                 struct object_id *remote = &refs->old_oid;
 352                 const char *remote_hex;
 353                 struct object *o;
 354
 355                 /*
 356                  * If that object is complete (i.e. it is an ancestor of a
 357                  * local ref), we tell them we have it but do not have to
 358                  * tell them about its ancestors, which they already know
 359                  * about.
 360                  *
 361                  * We use lookup_object here because we are only
 362                  * interested in the case we *know* the object is
 363                  * reachable and we have already scanned it.
 364                  */
 365                 if (((o = lookup_object(the_repository, remote->hash)) != NULL) &&
 366                                 (o->flags & COMPLETE)) {
 367                         continue;
 368                 }
 369
 370                 remote_hex = oid_to_hex(remote);
 371                 if (!fetching) {
 372                         struct strbuf c = STRBUF_INIT;
 373                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 374                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 375                         if (no_done)            strbuf_addstr(&c, " no-done");
 376                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 377                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 378                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 379                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 380                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 381                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 382                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 383                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 384                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 385                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 386                                                             git_user_agent_sanitized());
 387                         if (args->filter_options.choice)
 388                                 strbuf_addstr(&c, " filter");
 389                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 390                         strbuf_release(&c);
 391                 } else
 392                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 393                 fetching++;
 394         }
 395
 396         if (!fetching) {
 397                 strbuf_release(&req_buf);
 398                 packet_flush(fd[1]);
 399                 return 1;
 400         }
 401
 402         if (is_repository_shallow(the_repository))
 403                 write_shallow_commits(&req_buf, 1, NULL);
 404         if (args->depth > 0)
 405                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 406         if (args->deepen_since) {
 407                 timestamp_t max_age = approxidate(args->deepen_since);
 408                 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 409         }
 410         if (args->deepen_not) {
 411                 int i;
 412                 for (i = 0; i < args->deepen_not->nr; i++) {
 413                         struct string_list_item *s = args->deepen_not->items + i;
 414                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 415                 }
 416         }
 417         if (server_supports_filtering && args->filter_options.choice)
 418                 packet_buf_write(&req_buf, "filter %s",
 419                                  args->filter_options.filter_spec);
 420         packet_buf_flush(&req_buf);
 421         state_len = req_buf.len;
 422
 423         if (args->deepen) {
 424                 char *line;
 425                 const char *arg;
 426                 struct object_id oid;
 427
 428                 send_request(args, fd[1], &req_buf);
 429                 while ((line = packet_read_line(fd[0], NULL))) {
 430                         if (skip_prefix(line, "shallow ", &arg)) {
 431                                 if (get_oid_hex(arg, &oid))
 432                                         die(_("invalid shallow line: %s"), line);
 433                                 register_shallow(the_repository, &oid);
 434                                 continue;
 435                         }
 436                         if (skip_prefix(line, "unshallow ", &arg)) {
 437                                 if (get_oid_hex(arg, &oid))
 438                                         die(_("invalid unshallow line: %s"), line);
 439                                 if (!lookup_object(the_repository, oid.hash))
 440                                         die(_("object not found: %s"), line);
 441                                 /* make sure that it is parsed as shallow */
 442                                 if (!parse_object(the_repository, &oid))
 443                                         die(_("error in object: %s"), line);
 444                                 if (unregister_shallow(&oid))
 445                                         die(_("no shallow found: %s"), line);
 446                                 continue;
 447                         }
 448                         die(_("expected shallow/unshallow, got %s"), line);
 449                 }
 450         } else if (!args->stateless_rpc)
 451                 send_request(args, fd[1], &req_buf);
 452
 453         if (!args->stateless_rpc) {
 454                 /* If we aren't using the stateless-rpc interface
 455                  * we don't need to retain the headers.
 456                  */
 457                 strbuf_setlen(&req_buf, 0);
 458                 state_len = 0;
 459         }
 460
 461         flushes = 0;
 462         retval = -1;
 463         if (args->no_dependents)
 464                 goto done;
 465         while ((oid = get_rev())) {
 466                 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 467                 print_verbose(args, "have %s", oid_to_hex(oid));
 468                 in_vain++;
 469                 if (flush_at <= ++count) {
 470                         int ack;
 471
 472                         packet_buf_flush(&req_buf);
 473                         send_request(args, fd[1], &req_buf);
 474                         strbuf_setlen(&req_buf, state_len);
 475                         flushes++;
 476                         flush_at = next_flush(args->stateless_rpc, count);
 477
 478                         /*
 479                          * We keep one window "ahead" of the other side, and
 480                          * will wait for an ACK only on the next one
 481                          */
 482                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 483                                 continue;
 484
 485                         consume_shallow_list(args, fd[0]);
 486                         do {
 487                                 ack = get_ack(fd[0], result_oid);
 488                                 if (ack)
 489                                         print_verbose(args, _("got %s %d %s"), "ack",
 490                                                       ack, oid_to_hex(result_oid));
 491                                 switch (ack) {
 492                                 case ACK:
 493                                         flushes = 0;
 494                                         multi_ack = 0;
 495                                         retval = 0;
 496                                         goto done;
 497                                 case ACK_common:
 498                                 case ACK_ready:
 499                                 case ACK_continue: {
 500                                         struct commit *commit =
 501                                                 lookup_commit(the_repository,
 502                                                               result_oid);
 503                                         if (!commit)
 504                                                 die(_("invalid commit %s"), oid_to_hex(result_oid));
 505                                         if (args->stateless_rpc
 506                                          && ack == ACK_common
 507                                          && !(commit->object.flags & COMMON)) {
 508                                                 /* We need to replay the have for this object
 509                                                  * on the next RPC request so the peer knows
 510                                                  * it is in common with us.
 511                                                  */
 512                                                 const char *hex = oid_to_hex(result_oid);
 513                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 514                                                 state_len = req_buf.len;
 515                                                 /*
 516                                                  * Reset in_vain because an ack
 517                                                  * for this commit has not been
 518                                                  * seen.
 519                                                  */
 520                                                 in_vain = 0;
 521                                         } else if (!args->stateless_rpc
 522                                                    || ack != ACK_common)
 523                                                 in_vain = 0;
 524                                         mark_common(commit, 0, 1);
 525                                         retval = 0;
 526                                         got_continue = 1;
 527                                         if (ack == ACK_ready) {
 528                                                 clear_prio_queue(&rev_list);
 529                                                 got_ready = 1;
 530                                         }
 531                                         break;
 532                                         }
 533                                 }
 534                         } while (ack);
 535                         flushes--;
 536                         if (got_continue && MAX_IN_VAIN < in_vain) {
 537                                 print_verbose(args, _("giving up"));
 538                                 break; /* give up */
 539                         }
 540                 }
 541         }
 542 done:
 543         if (!got_ready || !no_done) {
 544                 packet_buf_write(&req_buf, "done\n");
 545                 send_request(args, fd[1], &req_buf);
 546         }
 547         print_verbose(args, _("done"));
 548         if (retval != 0) {
 549                 multi_ack = 0;
 550                 flushes++;
 551         }
 552         strbuf_release(&req_buf);
 553
 554         if (!got_ready || !no_done)
 555                 consume_shallow_list(args, fd[0]);
 556         while (flushes || multi_ack) {
 557                 int ack = get_ack(fd[0], result_oid);
 558                 if (ack) {
 559                         print_verbose(args, _("got %s (%d) %s"), "ack",
 560                                       ack, oid_to_hex(result_oid));
 561                         if (ack == ACK)
 562                                 return 0;
 563                         multi_ack = 1;
 564                         continue;
 565                 }
 566                 flushes--;
 567         }
 568         /* it is no error to fetch into a completely empty repo */
 569         return count ? retval : 0;
 570 }
 571
 572 static struct commit_list *complete;
 573
 574 static int mark_complete(const struct object_id *oid)
 575 {
 576         struct object *o = parse_object(the_repository, oid);
 577
 578         while (o && o->type == OBJ_TAG) {
 579                 struct tag *t = (struct tag *) o;
 580                 if (!t->tagged)
 581                         break; /* broken repository */
 582                 o->flags |= COMPLETE;
 583                 o = parse_object(the_repository, &t->tagged->oid);
 584         }
 585         if (o && o->type == OBJ_COMMIT) {
 586                 struct commit *commit = (struct commit *)o;
 587                 if (!(commit->object.flags & COMPLETE)) {
 588                         commit->object.flags |= COMPLETE;
 589                         commit_list_insert(commit, &complete);
 590                 }
 591         }
 592         return 0;
 593 }
 594
 595 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 596                              int flag, void *cb_data)
 597 {
 598         return mark_complete(oid);
 599 }
 600
 601 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 602                                          timestamp_t cutoff)
 603 {
 604         while (complete && cutoff <= complete->item->date) {
 605                 print_verbose(args, _("Marking %s as complete"),
 606                               oid_to_hex(&complete->item->object.oid));
 607                 pop_most_recent_commit(&complete, COMPLETE);
 608         }
 609 }
 610
 611 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 612 {
 613         for (; refs; refs = refs->next)
 614                 oidset_insert(oids, &refs->old_oid);
 615 }
 616
 617 static int tip_oids_contain(struct oidset *tip_oids,
 618                             struct ref *unmatched, struct ref *newlist,
 619                             const struct object_id *id)
 620 {
 621         /*
 622          * Note that this only looks at the ref lists the first time it's
 623          * called. This works out in filter_refs() because even though it may
 624          * add to "newlist" between calls, the additions will always be for
 625          * oids that are already in the set.
 626          */
 627         if (!tip_oids->map.map.tablesize) {
 628                 add_refs_to_oidset(tip_oids, unmatched);
 629                 add_refs_to_oidset(tip_oids, newlist);
 630         }
 631         return oidset_contains(tip_oids, id);
 632 }
 633
 634 static void filter_refs(struct fetch_pack_args *args,
 635                         struct ref **refs,
 636                         struct ref **sought, int nr_sought)
 637 {
 638         struct ref *newlist = NULL;
 639         struct ref **newtail = &newlist;
 640         struct ref *unmatched = NULL;
 641         struct ref *ref, *next;
 642         struct oidset tip_oids = OIDSET_INIT;
 643         int i;
 644
 645         i = 0;
 646         for (ref = *refs; ref; ref = next) {
 647                 int keep = 0;
 648                 next = ref->next;
 649
 650                 if (starts_with(ref->name, "refs/") &&
 651                     check_refname_format(ref->name, 0))
 652                         ; /* trash */
 653                 else {
 654                         while (i < nr_sought) {
 655                                 int cmp = strcmp(ref->name, sought[i]->name);
 656                                 if (cmp < 0)
 657                                         break; /* definitely do not have it */
 658                                 else if (cmp == 0) {
 659                                         keep = 1; /* definitely have it */
 660                                         sought[i]->match_status = REF_MATCHED;
 661                                 }
 662                                 i++;
 663                         }
 664
 665                         if (!keep && args->fetch_all &&
 666                             (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 667                                 keep = 1;
 668                 }
 669
 670                 if (keep) {
 671                         *newtail = ref;
 672                         ref->next = NULL;
 673                         newtail = &ref->next;
 674                 } else {
 675                         ref->next = unmatched;
 676                         unmatched = ref;
 677                 }
 678         }
 679
 680         /* Append unmatched requests to the list */
 681         for (i = 0; i < nr_sought; i++) {
 682                 struct object_id oid;
 683                 const char *p;
 684
 685                 ref = sought[i];
 686                 if (ref->match_status != REF_NOT_MATCHED)
 687                         continue;
 688                 if (parse_oid_hex(ref->name, &oid, &p) ||
 689                     *p != '\0' ||
 690                     oidcmp(&oid, &ref->old_oid))
 691                         continue;
 692
 693                 if ((allow_unadvertised_object_request &
 694                      (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 695                     tip_oids_contain(&tip_oids, unmatched, newlist,
 696                                      &ref->old_oid)) {
 697                         ref->match_status = REF_MATCHED;
 698                         *newtail = copy_ref(ref);
 699                         newtail = &(*newtail)->next;
 700                 } else {
 701                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 702                 }
 703         }
 704
 705         oidset_clear(&tip_oids);
 706         for (ref = unmatched; ref; ref = next) {
 707                 next = ref->next;
 708                 free(ref);
 709         }
 710
 711         *refs = newlist;
 712 }
 713
 714 static void mark_alternate_complete(struct object *obj)
 715 {
 716         mark_complete(&obj->oid);
 717 }
 718
 719 struct loose_object_iter {
 720         struct oidset *loose_object_set;
 721         struct ref *refs;
 722 };
 723
 724 /*
 725  *  If the number of refs is not larger than the number of loose objects,
 726  *  this function stops inserting.
 727  */
 728 static int add_loose_objects_to_set(const struct object_id *oid,
 729                                     const char *path,
 730                                     void *data)
 731 {
 732         struct loose_object_iter *iter = data;
 733         oidset_insert(iter->loose_object_set, oid);
 734         if (iter->refs == NULL)
 735                 return 1;
 736
 737         iter->refs = iter->refs->next;
 738         return 0;
 739 }
 740
 741 static int everything_local(struct fetch_pack_args *args,
 742                             struct ref **refs,
 743                             struct ref **sought, int nr_sought)
 744 {
 745         struct ref *ref;
 746         int retval;
 747         int old_save_commit_buffer = save_commit_buffer;
 748         timestamp_t cutoff = 0;
 749         struct oidset loose_oid_set = OIDSET_INIT;
 750         int use_oidset = 0;
 751         struct loose_object_iter iter = {&loose_oid_set, *refs};
 752
 753         /* Enumerate all loose objects or know refs are not so many. */
 754         use_oidset = !for_each_loose_object(add_loose_objects_to_set,
 755                                             &iter, 0);
 756
 757         save_commit_buffer = 0;
 758
 759         for (ref = *refs; ref; ref = ref->next) {
 760                 struct object *o;
 761                 unsigned int flags = OBJECT_INFO_QUICK;
 762
 763                 if (use_oidset &&
 764                     !oidset_contains(&loose_oid_set, &ref->old_oid)) {
 765                         /*
 766                          * I know this does not exist in the loose form,
 767                          * so check if it exists in a non-loose form.
 768                          */
 769                         flags |= OBJECT_INFO_IGNORE_LOOSE;
 770                 }
 771
 772                 if (!has_object_file_with_flags(&ref->old_oid, flags))
 773                         continue;
 774                 o = parse_object(the_repository, &ref->old_oid);
 775                 if (!o)
 776                         continue;
 777
 778                 /* We already have it -- which may mean that we were
 779                  * in sync with the other side at some time after
 780                  * that (it is OK if we guess wrong here).
 781                  */
 782                 if (o->type == OBJ_COMMIT) {
 783                         struct commit *commit = (struct commit *)o;
 784                         if (!cutoff || cutoff < commit->date)
 785                                 cutoff = commit->date;
 786                 }
 787         }
 788
 789         oidset_clear(&loose_oid_set);
 790
 791         if (!args->no_dependents) {
 792                 if (!args->deepen) {
 793                         for_each_ref(mark_complete_oid, NULL);
 794                         for_each_cached_alternate(mark_alternate_complete);
 795                         commit_list_sort_by_date(&complete);
 796                         if (cutoff)
 797                                 mark_recent_complete_commits(args, cutoff);
 798                 }
 799
 800                 /*
 801                  * Mark all complete remote refs as common refs.
 802                  * Don't mark them common yet; the server has to be told so first.
 803                  */
 804                 for (ref = *refs; ref; ref = ref->next) {
 805                         struct object *o = deref_tag(lookup_object(the_repository,
 806                                                      ref->old_oid.hash),
 807                                                      NULL, 0);
 808
 809                         if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 810                                 continue;
 811
 812                         if (!(o->flags & SEEN)) {
 813                                 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 814
 815                                 mark_common((struct commit *)o, 1, 1);
 816                         }
 817                 }
 818         }
 819
 820         filter_refs(args, refs, sought, nr_sought);
 821
 822         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 823                 const struct object_id *remote = &ref->old_oid;
 824                 struct object *o;
 825
 826                 o = lookup_object(the_repository, remote->hash);
 827                 if (!o || !(o->flags & COMPLETE)) {
 828                         retval = 0;
 829                         print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 830                                       ref->name);
 831                         continue;
 832                 }
 833                 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 834                               ref->name);
 835         }
 836
 837         save_commit_buffer = old_save_commit_buffer;
 838
 839         return retval;
 840 }
 841
 842 static int sideband_demux(int in, int out, void *data)
 843 {
 844         int *xd = data;
 845         int ret;
 846
 847         ret = recv_sideband("fetch-pack", xd[0], out);
 848         close(out);
 849         return ret;
 850 }
 851
 852 static int get_pack(struct fetch_pack_args *args,
 853                     int xd[2], char **pack_lockfile)
 854 {
 855         struct async demux;
 856         int do_keep = args->keep_pack;
 857         const char *cmd_name;
 858         struct pack_header header;
 859         int pass_header = 0;
 860         struct child_process cmd = CHILD_PROCESS_INIT;
 861         int ret;
 862
 863         memset(&demux, 0, sizeof(demux));
 864         if (use_sideband) {
 865                 /* xd[] is talking with upload-pack; subprocess reads from
 866                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 867                  * through demux->out.
 868                  */
 869                 demux.proc = sideband_demux;
 870                 demux.data = xd;
 871                 demux.out = -1;
 872                 demux.isolate_sigpipe = 1;
 873                 if (start_async(&demux))
 874                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 875         }
 876         else
 877                 demux.out = xd[0];
 878
 879         if (!args->keep_pack && unpack_limit) {
 880
 881                 if (read_pack_header(demux.out, &header))
 882                         die(_("protocol error: bad pack header"));
 883                 pass_header = 1;
 884                 if (ntohl(header.hdr_entries) < unpack_limit)
 885                         do_keep = 0;
 886                 else
 887                         do_keep = 1;
 888         }
 889
 890         if (alternate_shallow_file) {
 891                 argv_array_push(&cmd.args, "--shallow-file");
 892                 argv_array_push(&cmd.args, alternate_shallow_file);
 893         }
 894
 895         if (do_keep || args->from_promisor) {
 896                 if (pack_lockfile)
 897                         cmd.out = -1;
 898                 cmd_name = "index-pack";
 899                 argv_array_push(&cmd.args, cmd_name);
 900                 argv_array_push(&cmd.args, "--stdin");
 901                 if (!args->quiet && !args->no_progress)
 902                         argv_array_push(&cmd.args, "-v");
 903                 if (args->use_thin_pack)
 904                         argv_array_push(&cmd.args, "--fix-thin");
 905                 if (do_keep && (args->lock_pack || unpack_limit)) {
 906                         char hostname[HOST_NAME_MAX + 1];
 907                         if (xgethostname(hostname, sizeof(hostname)))
 908                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 909                         argv_array_pushf(&cmd.args,
 910                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 911                                         (uintmax_t)getpid(), hostname);
 912                 }
 913                 if (args->check_self_contained_and_connected)
 914                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 915                 if (args->from_promisor)
 916                         argv_array_push(&cmd.args, "--promisor");
 917         }
 918         else {
 919                 cmd_name = "unpack-objects";
 920                 argv_array_push(&cmd.args, cmd_name);
 921                 if (args->quiet || args->no_progress)
 922                         argv_array_push(&cmd.args, "-q");
 923                 args->check_self_contained_and_connected = 0;
 924         }
 925
 926         if (pass_header)
 927                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 928                                  ntohl(header.hdr_version),
 929                                  ntohl(header.hdr_entries));
 930         if (fetch_fsck_objects >= 0
 931             ? fetch_fsck_objects
 932             : transfer_fsck_objects >= 0
 933             ? transfer_fsck_objects
 934             : 0) {
 935                 if (args->from_promisor)
 936                         /*
 937                          * We cannot use --strict in index-pack because it
 938                          * checks both broken objects and links, but we only
 939                          * want to check for broken objects.
 940                          */
 941                         argv_array_push(&cmd.args, "--fsck-objects");
 942                 else
 943                         argv_array_push(&cmd.args, "--strict");
 944         }
 945
 946         cmd.in = demux.out;
 947         cmd.git_cmd = 1;
 948         if (start_command(&cmd))
 949                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 950         if (do_keep && pack_lockfile) {
 951                 *pack_lockfile = index_pack_lockfile(cmd.out);
 952                 close(cmd.out);
 953         }
 954
 955         if (!use_sideband)
 956                 /* Closed by start_command() */
 957                 xd[0] = -1;
 958
 959         ret = finish_command(&cmd);
 960         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 961                 args->self_contained_and_connected =
 962                         args->check_self_contained_and_connected &&
 963                         ret == 0;
 964         else
 965                 die(_("%s failed"), cmd_name);
 966         if (use_sideband && finish_async(&demux))
 967                 die(_("error in sideband demultiplexer"));
 968         return 0;
 969 }
 970
 971 static int cmp_ref_by_name(const void *a_, const void *b_)
 972 {
 973         const struct ref *a = *((const struct ref **)a_);
 974         const struct ref *b = *((const struct ref **)b_);
 975         return strcmp(a->name, b->name);
 976 }
 977
 978 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 979                                  int fd[2],
 980                                  const struct ref *orig_ref,
 981                                  struct ref **sought, int nr_sought,
 982                                  struct shallow_info *si,
 983                                  char **pack_lockfile)
 984 {
 985         struct ref *ref = copy_ref_list(orig_ref);
 986         struct object_id oid;
 987         const char *agent_feature;
 988         int agent_len;
 989
 990         sort_ref_list(&ref, ref_compare_name);
 991         QSORT(sought, nr_sought, cmp_ref_by_name);
 992
 993         if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
 994                 die(_("Server does not support shallow clients"));
 995         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 996                 args->deepen = 1;
 997         if (server_supports("multi_ack_detailed")) {
 998                 print_verbose(args, _("Server supports multi_ack_detailed"));
 999                 multi_ack = 2;
1000                 if (server_supports("no-done")) {
1001                         print_verbose(args, _("Server supports no-done"));
1002                         if (args->stateless_rpc)
1003                                 no_done = 1;
1004                 }
1005         }
1006         else if (server_supports("multi_ack")) {
1007                 print_verbose(args, _("Server supports multi_ack"));
1008                 multi_ack = 1;
1009         }
1010         if (server_supports("side-band-64k")) {
1011                 print_verbose(args, _("Server supports side-band-64k"));
1012                 use_sideband = 2;
1013         }
1014         else if (server_supports("side-band")) {
1015                 print_verbose(args, _("Server supports side-band"));
1016                 use_sideband = 1;
1017         }
1018         if (server_supports("allow-tip-sha1-in-want")) {
1019                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
1020                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1021         }
1022         if (server_supports("allow-reachable-sha1-in-want")) {
1023                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
1024                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1025         }
1026         if (!server_supports("thin-pack"))
1027                 args->use_thin_pack = 0;
1028         if (!server_supports("no-progress"))
1029                 args->no_progress = 0;
1030         if (!server_supports("include-tag"))
1031                 args->include_tag = 0;
1032         if (server_supports("ofs-delta"))
1033                 print_verbose(args, _("Server supports ofs-delta"));
1034         else
1035                 prefer_ofs_delta = 0;
1036
1037         if (server_supports("filter")) {
1038                 server_supports_filtering = 1;
1039                 print_verbose(args, _("Server supports filter"));
1040         } else if (args->filter_options.choice) {
1041                 warning("filtering not recognized by server, ignoring");
1042         }
1043
1044         if ((agent_feature = server_feature_value("agent", &agent_len))) {
1045                 agent_supported = 1;
1046                 if (agent_len)
1047                         print_verbose(args, _("Server version is %.*s"),
1048                                       agent_len, agent_feature);
1049         }
1050         if (server_supports("deepen-since"))
1051                 deepen_since_ok = 1;
1052         else if (args->deepen_since)
1053                 die(_("Server does not support --shallow-since"));
1054         if (server_supports("deepen-not"))
1055                 deepen_not_ok = 1;
1056         else if (args->deepen_not)
1057                 die(_("Server does not support --shallow-exclude"));
1058         if (!server_supports("deepen-relative") && args->deepen_relative)
1059                 die(_("Server does not support --deepen"));
1060
1061         if (everything_local(args, &ref, sought, nr_sought)) {
1062                 packet_flush(fd[1]);
1063                 goto all_done;
1064         }
1065         if (find_common(args, fd, &oid, ref) < 0)
1066                 if (!args->keep_pack)
1067                         /* When cloning, it is not unusual to have
1068                          * no common commit.
1069                          */
1070                         warning(_("no common commits"));
1071
1072         if (args->stateless_rpc)
1073                 packet_flush(fd[1]);
1074         if (args->deepen)
1075                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1076                                         NULL);
1077         else if (si->nr_ours || si->nr_theirs)
1078                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1079         else
1080                 alternate_shallow_file = NULL;
1081         if (get_pack(args, fd, pack_lockfile))
1082                 die(_("git fetch-pack: fetch failed."));
1083
1084  all_done:
1085         return ref;
1086 }
1087
1088 static void add_shallow_requests(struct strbuf *req_buf,
1089                                  const struct fetch_pack_args *args)
1090 {
1091         if (is_repository_shallow(the_repository))
1092                 write_shallow_commits(req_buf, 1, NULL);
1093         if (args->depth > 0)
1094                 packet_buf_write(req_buf, "deepen %d", args->depth);
1095         if (args->deepen_since) {
1096                 timestamp_t max_age = approxidate(args->deepen_since);
1097                 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1098         }
1099         if (args->deepen_not) {
1100                 int i;
1101                 for (i = 0; i < args->deepen_not->nr; i++) {
1102                         struct string_list_item *s = args->deepen_not->items + i;
1103                         packet_buf_write(req_buf, "deepen-not %s", s->string);
1104                 }
1105         }
1106 }
1107
1108 static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1109 {
1110         for ( ; wants ; wants = wants->next) {
1111                 const struct object_id *remote = &wants->old_oid;
1112                 const char *remote_hex;
1113                 struct object *o;
1114
1115                 /*
1116                  * If that object is complete (i.e. it is an ancestor of a
1117                  * local ref), we tell them we have it but do not have to
1118                  * tell them about its ancestors, which they already know
1119                  * about.
1120                  *
1121                  * We use lookup_object here because we are only
1122                  * interested in the case we *know* the object is
1123                  * reachable and we have already scanned it.
1124                  */
1125                 if (((o = lookup_object(the_repository, remote->hash)) != NULL) &&
1126                     (o->flags & COMPLETE)) {
1127                         continue;
1128                 }
1129
1130                 remote_hex = oid_to_hex(remote);
1131                 packet_buf_write(req_buf, "want %s\n", remote_hex);
1132         }
1133 }
1134
1135 static void add_common(struct strbuf *req_buf, struct oidset *common)
1136 {
1137         struct oidset_iter iter;
1138         const struct object_id *oid;
1139         oidset_iter_init(common, &iter);
1140
1141         while ((oid = oidset_iter_next(&iter))) {
1142                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1143         }
1144 }
1145
1146 static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
1147 {
1148         int ret = 0;
1149         int haves_added = 0;
1150         const struct object_id *oid;
1151
1152         while ((oid = get_rev())) {
1153                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1154                 if (++haves_added >= *haves_to_send)
1155                         break;
1156         }
1157
1158         *in_vain += haves_added;
1159         if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1160                 /* Send Done */
1161                 packet_buf_write(req_buf, "done\n");
1162                 ret = 1;
1163         }
1164
1165         /* Increase haves to send on next round */
1166         *haves_to_send = next_flush(1, *haves_to_send);
1167
1168         return ret;
1169 }
1170
1171 static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
1172                               const struct ref *wants, struct oidset *common,
1173                               int *haves_to_send, int *in_vain)
1174 {
1175         int ret = 0;
1176         struct strbuf req_buf = STRBUF_INIT;
1177
1178         if (server_supports_v2("fetch", 1))
1179                 packet_buf_write(&req_buf, "command=fetch");
1180         if (server_supports_v2("agent", 0))
1181                 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1182         if (args->server_options && args->server_options->nr &&
1183             server_supports_v2("server-option", 1)) {
1184                 int i;
1185                 for (i = 0; i < args->server_options->nr; i++)
1186                         packet_write_fmt(fd_out, "server-option=%s",
1187                                          args->server_options->items[i].string);
1188         }
1189
1190         packet_buf_delim(&req_buf);
1191         if (args->use_thin_pack)
1192                 packet_buf_write(&req_buf, "thin-pack");
1193         if (args->no_progress)
1194                 packet_buf_write(&req_buf, "no-progress");
1195         if (args->include_tag)
1196                 packet_buf_write(&req_buf, "include-tag");
1197         if (prefer_ofs_delta)
1198                 packet_buf_write(&req_buf, "ofs-delta");
1199
1200         /* Add shallow-info and deepen request */
1201         if (server_supports_feature("fetch", "shallow", 0))
1202                 add_shallow_requests(&req_buf, args);
1203         else if (is_repository_shallow(the_repository) || args->deepen)
1204                 die(_("Server does not support shallow requests"));
1205
1206         /* Add filter */
1207         if (server_supports_feature("fetch", "filter", 0) &&
1208             args->filter_options.choice) {
1209                 print_verbose(args, _("Server supports filter"));
1210                 packet_buf_write(&req_buf, "filter %s",
1211                                  args->filter_options.filter_spec);
1212         } else if (args->filter_options.choice) {
1213                 warning("filtering not recognized by server, ignoring");
1214         }
1215
1216         /* add wants */
1217         add_wants(wants, &req_buf);
1218
1219         if (args->no_dependents) {
1220                 packet_buf_write(&req_buf, "done");
1221                 ret = 1;
1222         } else {
1223                 /* Add all of the common commits we've found in previous rounds */
1224                 add_common(&req_buf, common);
1225
1226                 /* Add initial haves */
1227                 ret = add_haves(&req_buf, haves_to_send, in_vain);
1228         }
1229
1230         /* Send request */
1231         packet_buf_flush(&req_buf);
1232         write_or_die(fd_out, req_buf.buf, req_buf.len);
1233
1234         strbuf_release(&req_buf);
1235         return ret;
1236 }
1237
1238 /*
1239  * Processes a section header in a server's response and checks if it matches
1240  * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1241  * not consumed); if 0, the line will be consumed and the function will die if
1242  * the section header doesn't match what was expected.
1243  */
1244 static int process_section_header(struct packet_reader *reader,
1245                                   const char *section, int peek)
1246 {
1247         int ret;
1248
1249         if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1250                 die("error reading section header '%s'", section);
1251
1252         ret = !strcmp(reader->line, section);
1253
1254         if (!peek) {
1255                 if (!ret)
1256                         die("expected '%s', received '%s'",
1257                             section, reader->line);
1258                 packet_reader_read(reader);
1259         }
1260
1261         return ret;
1262 }
1263
1264 static int process_acks(struct packet_reader *reader, struct oidset *common)
1265 {
1266         /* received */
1267         int received_ready = 0;
1268         int received_ack = 0;
1269
1270         process_section_header(reader, "acknowledgments", 0);
1271         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1272                 const char *arg;
1273
1274                 if (!strcmp(reader->line, "NAK"))
1275                         continue;
1276
1277                 if (skip_prefix(reader->line, "ACK ", &arg)) {
1278                         struct object_id oid;
1279                         if (!get_oid_hex(arg, &oid)) {
1280                                 struct commit *commit;
1281                                 oidset_insert(common, &oid);
1282                                 commit = lookup_commit(the_repository, &oid);
1283                                 mark_common(commit, 0, 1);
1284                         }
1285                         continue;
1286                 }
1287
1288                 if (!strcmp(reader->line, "ready")) {
1289                         clear_prio_queue(&rev_list);
1290                         received_ready = 1;
1291                         continue;
1292                 }
1293
1294                 die("unexpected acknowledgment line: '%s'", reader->line);
1295         }
1296
1297         if (reader->status != PACKET_READ_FLUSH &&
1298             reader->status != PACKET_READ_DELIM)
1299                 die("error processing acks: %d", reader->status);
1300
1301         /* return 0 if no common, 1 if there are common, or 2 if ready */
1302         return received_ready ? 2 : (received_ack ? 1 : 0);
1303 }
1304
1305 static void receive_shallow_info(struct fetch_pack_args *args,
1306                                  struct packet_reader *reader)
1307 {
1308         process_section_header(reader, "shallow-info", 0);
1309         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1310                 const char *arg;
1311                 struct object_id oid;
1312
1313                 if (skip_prefix(reader->line, "shallow ", &arg)) {
1314                         if (get_oid_hex(arg, &oid))
1315                                 die(_("invalid shallow line: %s"), reader->line);
1316                         register_shallow(the_repository, &oid);
1317                         continue;
1318                 }
1319                 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1320                         if (get_oid_hex(arg, &oid))
1321                                 die(_("invalid unshallow line: %s"), reader->line);
1322                         if (!lookup_object(the_repository, oid.hash))
1323                                 die(_("object not found: %s"), reader->line);
1324                         /* make sure that it is parsed as shallow */
1325                         if (!parse_object(the_repository, &oid))
1326                                 die(_("error in object: %s"), reader->line);
1327                         if (unregister_shallow(&oid))
1328                                 die(_("no shallow found: %s"), reader->line);
1329                         continue;
1330                 }
1331                 die(_("expected shallow/unshallow, got %s"), reader->line);
1332         }
1333
1334         if (reader->status != PACKET_READ_FLUSH &&
1335             reader->status != PACKET_READ_DELIM)
1336                 die("error processing shallow info: %d", reader->status);
1337
1338         setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1339         args->deepen = 1;
1340 }
1341
1342 enum fetch_state {
1343         FETCH_CHECK_LOCAL = 0,
1344         FETCH_SEND_REQUEST,
1345         FETCH_PROCESS_ACKS,
1346         FETCH_GET_PACK,
1347         FETCH_DONE,
1348 };
1349
1350 static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1351                                     int fd[2],
1352                                     const struct ref *orig_ref,
1353                                     struct ref **sought, int nr_sought,
1354                                     char **pack_lockfile)
1355 {
1356         struct ref *ref = copy_ref_list(orig_ref);
1357         enum fetch_state state = FETCH_CHECK_LOCAL;
1358         struct oidset common = OIDSET_INIT;
1359         struct packet_reader reader;
1360         int in_vain = 0;
1361         int haves_to_send = INITIAL_FLUSH;
1362         packet_reader_init(&reader, fd[0], NULL, 0,
1363                            PACKET_READ_CHOMP_NEWLINE);
1364
1365         while (state != FETCH_DONE) {
1366                 switch (state) {
1367                 case FETCH_CHECK_LOCAL:
1368                         sort_ref_list(&ref, ref_compare_name);
1369                         QSORT(sought, nr_sought, cmp_ref_by_name);
1370
1371                         /* v2 supports these by default */
1372                         allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1373                         use_sideband = 2;
1374                         if (args->depth > 0 || args->deepen_since || args->deepen_not)
1375                                 args->deepen = 1;
1376
1377                         if (marked)
1378                                 for_each_ref(clear_marks, NULL);
1379                         marked = 1;
1380
1381                         for_each_ref(rev_list_insert_ref_oid, NULL);
1382                         for_each_cached_alternate(insert_one_alternate_object);
1383
1384                         /* Filter 'ref' by 'sought' and those that aren't local */
1385                         if (everything_local(args, &ref, sought, nr_sought))
1386                                 state = FETCH_DONE;
1387                         else
1388                                 state = FETCH_SEND_REQUEST;
1389                         break;
1390                 case FETCH_SEND_REQUEST:
1391                         if (send_fetch_request(fd[1], args, ref, &common,
1392                                                &haves_to_send, &in_vain))
1393                                 state = FETCH_GET_PACK;
1394                         else
1395                                 state = FETCH_PROCESS_ACKS;
1396                         break;
1397                 case FETCH_PROCESS_ACKS:
1398                         /* Process ACKs/NAKs */
1399                         switch (process_acks(&reader, &common)) {
1400                         case 2:
1401                                 state = FETCH_GET_PACK;
1402                                 break;
1403                         case 1:
1404                                 in_vain = 0;
1405                                 /* fallthrough */
1406                         default:
1407                                 state = FETCH_SEND_REQUEST;
1408                                 break;
1409                         }
1410                         break;
1411                 case FETCH_GET_PACK:
1412                         /* Check for shallow-info section */
1413                         if (process_section_header(&reader, "shallow-info", 1))
1414                                 receive_shallow_info(args, &reader);
1415
1416                         /* get the pack */
1417                         process_section_header(&reader, "packfile", 0);
1418                         if (get_pack(args, fd, pack_lockfile))
1419                                 die(_("git fetch-pack: fetch failed."));
1420
1421                         state = FETCH_DONE;
1422                         break;
1423                 case FETCH_DONE:
1424                         continue;
1425                 }
1426         }
1427
1428         oidset_clear(&common);
1429         return ref;
1430 }
1431
1432 static void fetch_pack_config(void)
1433 {
1434         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1435         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1436         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1437         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1438         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1439
1440         git_config(git_default_config, NULL);
1441 }
1442
1443 static void fetch_pack_setup(void)
1444 {
1445         static int did_setup;
1446         if (did_setup)
1447                 return;
1448         fetch_pack_config();
1449         if (0 <= transfer_unpack_limit)
1450                 unpack_limit = transfer_unpack_limit;
1451         else if (0 <= fetch_unpack_limit)
1452                 unpack_limit = fetch_unpack_limit;
1453         did_setup = 1;
1454 }
1455
1456 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1457 {
1458         struct string_list names = STRING_LIST_INIT_NODUP;
1459         int src, dst;
1460
1461         for (src = dst = 0; src < nr; src++) {
1462                 struct string_list_item *item;
1463                 item = string_list_insert(&names, ref[src]->name);
1464                 if (item->util)
1465                         continue; /* already have it */
1466                 item->util = ref[src];
1467                 if (src != dst)
1468                         ref[dst] = ref[src];
1469                 dst++;
1470         }
1471         for (src = dst; src < nr; src++)
1472                 ref[src] = NULL;
1473         string_list_clear(&names, 0);
1474         return dst;
1475 }
1476
1477 static void update_shallow(struct fetch_pack_args *args,
1478                            struct ref **sought, int nr_sought,
1479                            struct shallow_info *si)
1480 {
1481         struct oid_array ref = OID_ARRAY_INIT;
1482         int *status;
1483         int i;
1484
1485         if (args->deepen && alternate_shallow_file) {
1486                 if (*alternate_shallow_file == '\0') { /* --unshallow */
1487                         unlink_or_warn(git_path_shallow(the_repository));
1488                         rollback_lock_file(&shallow_lock);
1489                 } else
1490                         commit_lock_file(&shallow_lock);
1491                 return;
1492         }
1493
1494         if (!si->shallow || !si->shallow->nr)
1495                 return;
1496
1497         if (args->cloning) {
1498                 /*
1499                  * remote is shallow, but this is a clone, there are
1500                  * no objects in repo to worry about. Accept any
1501                  * shallow points that exist in the pack (iow in repo
1502                  * after get_pack() and reprepare_packed_git())
1503                  */
1504                 struct oid_array extra = OID_ARRAY_INIT;
1505                 struct object_id *oid = si->shallow->oid;
1506                 for (i = 0; i < si->shallow->nr; i++)
1507                         if (has_object_file(&oid[i]))
1508                                 oid_array_append(&extra, &oid[i]);
1509                 if (extra.nr) {
1510                         setup_alternate_shallow(&shallow_lock,
1511                                                 &alternate_shallow_file,
1512                                                 &extra);
1513                         commit_lock_file(&shallow_lock);
1514                 }
1515                 oid_array_clear(&extra);
1516                 return;
1517         }
1518
1519         if (!si->nr_ours && !si->nr_theirs)
1520                 return;
1521
1522         remove_nonexistent_theirs_shallow(si);
1523         if (!si->nr_ours && !si->nr_theirs)
1524                 return;
1525         for (i = 0; i < nr_sought; i++)
1526                 oid_array_append(&ref, &sought[i]->old_oid);
1527         si->ref = &ref;
1528
1529         if (args->update_shallow) {
1530                 /*
1531                  * remote is also shallow, .git/shallow may be updated
1532                  * so all refs can be accepted. Make sure we only add
1533                  * shallow roots that are actually reachable from new
1534                  * refs.
1535                  */
1536                 struct oid_array extra = OID_ARRAY_INIT;
1537                 struct object_id *oid = si->shallow->oid;
1538                 assign_shallow_commits_to_refs(si, NULL, NULL);
1539                 if (!si->nr_ours && !si->nr_theirs) {
1540                         oid_array_clear(&ref);
1541                         return;
1542                 }
1543                 for (i = 0; i < si->nr_ours; i++)
1544                         oid_array_append(&extra, &oid[si->ours[i]]);
1545                 for (i = 0; i < si->nr_theirs; i++)
1546                         oid_array_append(&extra, &oid[si->theirs[i]]);
1547                 setup_alternate_shallow(&shallow_lock,
1548                                         &alternate_shallow_file,
1549                                         &extra);
1550                 commit_lock_file(&shallow_lock);
1551                 oid_array_clear(&extra);
1552                 oid_array_clear(&ref);
1553                 return;
1554         }
1555
1556         /*
1557          * remote is also shallow, check what ref is safe to update
1558          * without updating .git/shallow
1559          */
1560         status = xcalloc(nr_sought, sizeof(*status));
1561         assign_shallow_commits_to_refs(si, NULL, status);
1562         if (si->nr_ours || si->nr_theirs) {
1563                 for (i = 0; i < nr_sought; i++)
1564                         if (status[i])
1565                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1566         }
1567         free(status);
1568         oid_array_clear(&ref);
1569 }
1570
1571 struct ref *fetch_pack(struct fetch_pack_args *args,
1572                        int fd[], struct child_process *conn,
1573                        const struct ref *ref,
1574                        const char *dest,
1575                        struct ref **sought, int nr_sought,
1576                        struct oid_array *shallow,
1577                        char **pack_lockfile,
1578                        enum protocol_version version)
1579 {
1580         struct ref *ref_cpy;
1581         struct shallow_info si;
1582
1583         fetch_pack_setup();
1584         if (nr_sought)
1585                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1586
1587         if (!ref) {
1588                 packet_flush(fd[1]);
1589                 die(_("no matching remote head"));
1590         }
1591         prepare_shallow_info(&si, shallow);
1592         if (version == protocol_v2)
1593                 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1594                                            pack_lockfile);
1595         else
1596                 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1597                                         &si, pack_lockfile);
1598         reprepare_packed_git(the_repository);
1599         update_shallow(args, sought, nr_sought, &si);
1600         clear_shallow_info(&si);
1601         return ref_cpy;
1602 }
1603
1604 int report_unmatched_refs(struct ref **sought, int nr_sought)
1605 {
1606         int i, ret = 0;
1607
1608         for (i = 0; i < nr_sought; i++) {
1609                 if (!sought[i])
1610                         continue;
1611                 switch (sought[i]->match_status) {
1612                 case REF_MATCHED:
1613                         continue;
1614                 case REF_NOT_MATCHED:
1615                         error(_("no such remote ref %s"), sought[i]->name);
1616                         break;
1617                 case REF_UNADVERTISED_NOT_ALLOWED:
1618                         error(_("Server does not allow request for unadvertised object %s"),
1619                               sought[i]->name);
1620                         break;
1621                 }
1622                 ret = 1;
1623         }
1624         return ret;
1625 }