git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "repository.h"
   3 #include "config.h"
   4 #include "lockfile.h"
   5 #include "refs.h"
   6 #include "pkt-line.h"
   7 #include "commit.h"
   8 #include "tag.h"
   9 #include "exec-cmd.h"
  10 #include "pack.h"
  11 #include "sideband.h"
  12 #include "fetch-pack.h"
  13 #include "remote.h"
  14 #include "run-command.h"
  15 #include "connect.h"
  16 #include "transport.h"
  17 #include "version.h"
  18 #include "prio-queue.h"
  19 #include "sha1-array.h"
  20 #include "oidset.h"
  21 #include "packfile.h"
  22 #include "object-store.h"
  23
  24 static int transfer_unpack_limit = -1;
  25 static int fetch_unpack_limit = -1;
  26 static int unpack_limit = 100;
  27 static int prefer_ofs_delta = 1;
  28 static int no_done;
  29 static int deepen_since_ok;
  30 static int deepen_not_ok;
  31 static int fetch_fsck_objects = -1;
  32 static int transfer_fsck_objects = -1;
  33 static int agent_supported;
  34 static int server_supports_filtering;
  35 static struct lock_file shallow_lock;
  36 static const char *alternate_shallow_file;
  37
  38 /* Remember to update object flag allocation in object.h */
  39 #define COMPLETE        (1U << 0)
  40 #define COMMON          (1U << 1)
  41 #define COMMON_REF      (1U << 2)
  42 #define SEEN            (1U << 3)
  43 #define POPPED          (1U << 4)
  44 #define ALTERNATE       (1U << 5)
  45
  46 static int marked;
  47
  48 /*
  49  * After sending this many "have"s if we do not get any new ACK , we
  50  * give up traversing our history.
  51  */
  52 #define MAX_IN_VAIN 256
  53
  54 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  55 static int non_common_revs, multi_ack, use_sideband;
  56 /* Allow specifying sha1 if it is a ref tip. */
  57 #define ALLOW_TIP_SHA1  01
  58 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  59 #define ALLOW_REACHABLE_SHA1    02
  60 static unsigned int allow_unadvertised_object_request;
  61
  62 __attribute__((format (printf, 2, 3)))
  63 static inline void print_verbose(const struct fetch_pack_args *args,
  64                                  const char *fmt, ...)
  65 {
  66         va_list params;
  67
  68         if (!args->verbose)
  69                 return;
  70
  71         va_start(params, fmt);
  72         vfprintf(stderr, fmt, params);
  73         va_end(params);
  74         fputc('\n', stderr);
  75 }
  76
  77 struct alternate_object_cache {
  78         struct object **items;
  79         size_t nr, alloc;
  80 };
  81
  82 static void cache_one_alternate(const char *refname,
  83                                 const struct object_id *oid,
  84                                 void *vcache)
  85 {
  86         struct alternate_object_cache *cache = vcache;
  87         struct object *obj = parse_object(the_repository, oid);
  88
  89         if (!obj || (obj->flags & ALTERNATE))
  90                 return;
  91
  92         obj->flags |= ALTERNATE;
  93         ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  94         cache->items[cache->nr++] = obj;
  95 }
  96
  97 static void for_each_cached_alternate(void (*cb)(struct object *))
  98 {
  99         static int initialized;
 100         static struct alternate_object_cache cache;
 101         size_t i;
 102
 103         if (!initialized) {
 104                 for_each_alternate_ref(cache_one_alternate, &cache);
 105                 initialized = 1;
 106         }
 107
 108         for (i = 0; i < cache.nr; i++)
 109                 cb(cache.items[i]);
 110 }
 111
 112 static void rev_list_push(struct commit *commit, int mark)
 113 {
 114         if (!(commit->object.flags & mark)) {
 115                 commit->object.flags |= mark;
 116
 117                 if (parse_commit(commit))
 118                         return;
 119
 120                 prio_queue_put(&rev_list, commit);
 121
 122                 if (!(commit->object.flags & COMMON))
 123                         non_common_revs++;
 124         }
 125 }
 126
 127 static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 128 {
 129         struct object *o = deref_tag(the_repository,
 130                                      parse_object(the_repository, oid),
 131                                      refname, 0);
 132
 133         if (o && o->type == OBJ_COMMIT)
 134                 rev_list_push((struct commit *)o, SEEN);
 135
 136         return 0;
 137 }
 138
 139 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 140                                    int flag, void *cb_data)
 141 {
 142         return rev_list_insert_ref(refname, oid);
 143 }
 144
 145 static int clear_marks(const char *refname, const struct object_id *oid,
 146                        int flag, void *cb_data)
 147 {
 148         struct object *o = deref_tag(the_repository,
 149                                      parse_object(the_repository, oid),
 150                                      refname, 0);
 151
 152         if (o && o->type == OBJ_COMMIT)
 153                 clear_commit_marks((struct commit *)o,
 154                                    COMMON | COMMON_REF | SEEN | POPPED);
 155         return 0;
 156 }
 157
 158 /*
 159    This function marks a rev and its ancestors as common.
 160    In some cases, it is desirable to mark only the ancestors (for example
 161    when only the server does not yet know that they are common).
 162 */
 163
 164 static void mark_common(struct commit *commit,
 165                 int ancestors_only, int dont_parse)
 166 {
 167         if (commit != NULL && !(commit->object.flags & COMMON)) {
 168                 struct object *o = (struct object *)commit;
 169
 170                 if (!ancestors_only)
 171                         o->flags |= COMMON;
 172
 173                 if (!(o->flags & SEEN))
 174                         rev_list_push(commit, SEEN);
 175                 else {
 176                         struct commit_list *parents;
 177
 178                         if (!ancestors_only && !(o->flags & POPPED))
 179                                 non_common_revs--;
 180                         if (!o->parsed && !dont_parse)
 181                                 if (parse_commit(commit))
 182                                         return;
 183
 184                         for (parents = commit->parents;
 185                                         parents;
 186                                         parents = parents->next)
 187                                 mark_common(parents->item, 0, dont_parse);
 188                 }
 189         }
 190 }
 191
 192 /*
 193   Get the next rev to send, ignoring the common.
 194 */
 195
 196 static const struct object_id *get_rev(void)
 197 {
 198         struct commit *commit = NULL;
 199
 200         while (commit == NULL) {
 201                 unsigned int mark;
 202                 struct commit_list *parents;
 203
 204                 if (rev_list.nr == 0 || non_common_revs == 0)
 205                         return NULL;
 206
 207                 commit = prio_queue_get(&rev_list);
 208                 parse_commit(commit);
 209                 parents = commit->parents;
 210
 211                 commit->object.flags |= POPPED;
 212                 if (!(commit->object.flags & COMMON))
 213                         non_common_revs--;
 214
 215                 if (commit->object.flags & COMMON) {
 216                         /* do not send "have", and ignore ancestors */
 217                         commit = NULL;
 218                         mark = COMMON | SEEN;
 219                 } else if (commit->object.flags & COMMON_REF)
 220                         /* send "have", and ignore ancestors */
 221                         mark = COMMON | SEEN;
 222                 else
 223                         /* send "have", also for its ancestors */
 224                         mark = SEEN;
 225
 226                 while (parents) {
 227                         if (!(parents->item->object.flags & SEEN))
 228                                 rev_list_push(parents->item, mark);
 229                         if (mark & COMMON)
 230                                 mark_common(parents->item, 1, 0);
 231                         parents = parents->next;
 232                 }
 233         }
 234
 235         return &commit->object.oid;
 236 }
 237
 238 enum ack_type {
 239         NAK = 0,
 240         ACK,
 241         ACK_continue,
 242         ACK_common,
 243         ACK_ready
 244 };
 245
 246 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 247 {
 248         if (args->stateless_rpc && args->deepen) {
 249                 /* If we sent a depth we will get back "duplicate"
 250                  * shallow and unshallow commands every time there
 251                  * is a block of have lines exchanged.
 252                  */
 253                 char *line;
 254                 while ((line = packet_read_line(fd, NULL))) {
 255                         if (starts_with(line, "shallow "))
 256                                 continue;
 257                         if (starts_with(line, "unshallow "))
 258                                 continue;
 259                         die(_("git fetch-pack: expected shallow list"));
 260                 }
 261         }
 262 }
 263
 264 static enum ack_type get_ack(int fd, struct object_id *result_oid)
 265 {
 266         int len;
 267         char *line = packet_read_line(fd, &len);
 268         const char *arg;
 269
 270         if (!line)
 271                 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
 272         if (!strcmp(line, "NAK"))
 273                 return NAK;
 274         if (skip_prefix(line, "ACK ", &arg)) {
 275                 if (!get_oid_hex(arg, result_oid)) {
 276                         arg += 40;
 277                         len -= arg - line;
 278                         if (len < 1)
 279                                 return ACK;
 280                         if (strstr(arg, "continue"))
 281                                 return ACK_continue;
 282                         if (strstr(arg, "common"))
 283                                 return ACK_common;
 284                         if (strstr(arg, "ready"))
 285                                 return ACK_ready;
 286                         return ACK;
 287                 }
 288         }
 289         if (skip_prefix(line, "ERR ", &arg))
 290                 die(_("remote error: %s"), arg);
 291         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 292 }
 293
 294 static void send_request(struct fetch_pack_args *args,
 295                          int fd, struct strbuf *buf)
 296 {
 297         if (args->stateless_rpc) {
 298                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 299                 packet_flush(fd);
 300         } else
 301                 write_or_die(fd, buf->buf, buf->len);
 302 }
 303
 304 static void insert_one_alternate_object(struct object *obj)
 305 {
 306         rev_list_insert_ref(NULL, &obj->oid);
 307 }
 308
 309 #define INITIAL_FLUSH 16
 310 #define PIPESAFE_FLUSH 32
 311 #define LARGE_FLUSH 16384
 312
 313 static int next_flush(int stateless_rpc, int count)
 314 {
 315         if (stateless_rpc) {
 316                 if (count < LARGE_FLUSH)
 317                         count <<= 1;
 318                 else
 319                         count = count * 11 / 10;
 320         } else {
 321                 if (count < PIPESAFE_FLUSH)
 322                         count <<= 1;
 323                 else
 324                         count += PIPESAFE_FLUSH;
 325         }
 326         return count;
 327 }
 328
 329 static int find_common(struct fetch_pack_args *args,
 330                        int fd[2], struct object_id *result_oid,
 331                        struct ref *refs)
 332 {
 333         int fetching;
 334         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 335         const struct object_id *oid;
 336         unsigned in_vain = 0;
 337         int got_continue = 0;
 338         int got_ready = 0;
 339         struct strbuf req_buf = STRBUF_INIT;
 340         size_t state_len = 0;
 341
 342         if (args->stateless_rpc && multi_ack == 1)
 343                 die(_("--stateless-rpc requires multi_ack_detailed"));
 344         if (marked)
 345                 for_each_ref(clear_marks, NULL);
 346         marked = 1;
 347
 348         for_each_ref(rev_list_insert_ref_oid, NULL);
 349         for_each_cached_alternate(insert_one_alternate_object);
 350
 351         fetching = 0;
 352         for ( ; refs ; refs = refs->next) {
 353                 struct object_id *remote = &refs->old_oid;
 354                 const char *remote_hex;
 355                 struct object *o;
 356
 357                 /*
 358                  * If that object is complete (i.e. it is an ancestor of a
 359                  * local ref), we tell them we have it but do not have to
 360                  * tell them about its ancestors, which they already know
 361                  * about.
 362                  *
 363                  * We use lookup_object here because we are only
 364                  * interested in the case we *know* the object is
 365                  * reachable and we have already scanned it.
 366                  */
 367                 if (((o = lookup_object(the_repository, remote->hash)) != NULL) &&
 368                                 (o->flags & COMPLETE)) {
 369                         continue;
 370                 }
 371
 372                 remote_hex = oid_to_hex(remote);
 373                 if (!fetching) {
 374                         struct strbuf c = STRBUF_INIT;
 375                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 376                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 377                         if (no_done)            strbuf_addstr(&c, " no-done");
 378                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 379                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 380                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 381                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 382                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 383                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 384                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 385                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 386                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 387                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 388                                                             git_user_agent_sanitized());
 389                         if (args->filter_options.choice)
 390                                 strbuf_addstr(&c, " filter");
 391                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 392                         strbuf_release(&c);
 393                 } else
 394                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 395                 fetching++;
 396         }
 397
 398         if (!fetching) {
 399                 strbuf_release(&req_buf);
 400                 packet_flush(fd[1]);
 401                 return 1;
 402         }
 403
 404         if (is_repository_shallow(the_repository))
 405                 write_shallow_commits(&req_buf, 1, NULL);
 406         if (args->depth > 0)
 407                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 408         if (args->deepen_since) {
 409                 timestamp_t max_age = approxidate(args->deepen_since);
 410                 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 411         }
 412         if (args->deepen_not) {
 413                 int i;
 414                 for (i = 0; i < args->deepen_not->nr; i++) {
 415                         struct string_list_item *s = args->deepen_not->items + i;
 416                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 417                 }
 418         }
 419         if (server_supports_filtering && args->filter_options.choice)
 420                 packet_buf_write(&req_buf, "filter %s",
 421                                  args->filter_options.filter_spec);
 422         packet_buf_flush(&req_buf);
 423         state_len = req_buf.len;
 424
 425         if (args->deepen) {
 426                 char *line;
 427                 const char *arg;
 428                 struct object_id oid;
 429
 430                 send_request(args, fd[1], &req_buf);
 431                 while ((line = packet_read_line(fd[0], NULL))) {
 432                         if (skip_prefix(line, "shallow ", &arg)) {
 433                                 if (get_oid_hex(arg, &oid))
 434                                         die(_("invalid shallow line: %s"), line);
 435                                 register_shallow(the_repository, &oid);
 436                                 continue;
 437                         }
 438                         if (skip_prefix(line, "unshallow ", &arg)) {
 439                                 if (get_oid_hex(arg, &oid))
 440                                         die(_("invalid unshallow line: %s"), line);
 441                                 if (!lookup_object(the_repository, oid.hash))
 442                                         die(_("object not found: %s"), line);
 443                                 /* make sure that it is parsed as shallow */
 444                                 if (!parse_object(the_repository, &oid))
 445                                         die(_("error in object: %s"), line);
 446                                 if (unregister_shallow(&oid))
 447                                         die(_("no shallow found: %s"), line);
 448                                 continue;
 449                         }
 450                         die(_("expected shallow/unshallow, got %s"), line);
 451                 }
 452         } else if (!args->stateless_rpc)
 453                 send_request(args, fd[1], &req_buf);
 454
 455         if (!args->stateless_rpc) {
 456                 /* If we aren't using the stateless-rpc interface
 457                  * we don't need to retain the headers.
 458                  */
 459                 strbuf_setlen(&req_buf, 0);
 460                 state_len = 0;
 461         }
 462
 463         flushes = 0;
 464         retval = -1;
 465         if (args->no_dependents)
 466                 goto done;
 467         while ((oid = get_rev())) {
 468                 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 469                 print_verbose(args, "have %s", oid_to_hex(oid));
 470                 in_vain++;
 471                 if (flush_at <= ++count) {
 472                         int ack;
 473
 474                         packet_buf_flush(&req_buf);
 475                         send_request(args, fd[1], &req_buf);
 476                         strbuf_setlen(&req_buf, state_len);
 477                         flushes++;
 478                         flush_at = next_flush(args->stateless_rpc, count);
 479
 480                         /*
 481                          * We keep one window "ahead" of the other side, and
 482                          * will wait for an ACK only on the next one
 483                          */
 484                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 485                                 continue;
 486
 487                         consume_shallow_list(args, fd[0]);
 488                         do {
 489                                 ack = get_ack(fd[0], result_oid);
 490                                 if (ack)
 491                                         print_verbose(args, _("got %s %d %s"), "ack",
 492                                                       ack, oid_to_hex(result_oid));
 493                                 switch (ack) {
 494                                 case ACK:
 495                                         flushes = 0;
 496                                         multi_ack = 0;
 497                                         retval = 0;
 498                                         goto done;
 499                                 case ACK_common:
 500                                 case ACK_ready:
 501                                 case ACK_continue: {
 502                                         struct commit *commit =
 503                                                 lookup_commit(the_repository,
 504                                                               result_oid);
 505                                         if (!commit)
 506                                                 die(_("invalid commit %s"), oid_to_hex(result_oid));
 507                                         if (args->stateless_rpc
 508                                          && ack == ACK_common
 509                                          && !(commit->object.flags & COMMON)) {
 510                                                 /* We need to replay the have for this object
 511                                                  * on the next RPC request so the peer knows
 512                                                  * it is in common with us.
 513                                                  */
 514                                                 const char *hex = oid_to_hex(result_oid);
 515                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 516                                                 state_len = req_buf.len;
 517                                                 /*
 518                                                  * Reset in_vain because an ack
 519                                                  * for this commit has not been
 520                                                  * seen.
 521                                                  */
 522                                                 in_vain = 0;
 523                                         } else if (!args->stateless_rpc
 524                                                    || ack != ACK_common)
 525                                                 in_vain = 0;
 526                                         mark_common(commit, 0, 1);
 527                                         retval = 0;
 528                                         got_continue = 1;
 529                                         if (ack == ACK_ready) {
 530                                                 clear_prio_queue(&rev_list);
 531                                                 got_ready = 1;
 532                                         }
 533                                         break;
 534                                         }
 535                                 }
 536                         } while (ack);
 537                         flushes--;
 538                         if (got_continue && MAX_IN_VAIN < in_vain) {
 539                                 print_verbose(args, _("giving up"));
 540                                 break; /* give up */
 541                         }
 542                 }
 543         }
 544 done:
 545         if (!got_ready || !no_done) {
 546                 packet_buf_write(&req_buf, "done\n");
 547                 send_request(args, fd[1], &req_buf);
 548         }
 549         print_verbose(args, _("done"));
 550         if (retval != 0) {
 551                 multi_ack = 0;
 552                 flushes++;
 553         }
 554         strbuf_release(&req_buf);
 555
 556         if (!got_ready || !no_done)
 557                 consume_shallow_list(args, fd[0]);
 558         while (flushes || multi_ack) {
 559                 int ack = get_ack(fd[0], result_oid);
 560                 if (ack) {
 561                         print_verbose(args, _("got %s (%d) %s"), "ack",
 562                                       ack, oid_to_hex(result_oid));
 563                         if (ack == ACK)
 564                                 return 0;
 565                         multi_ack = 1;
 566                         continue;
 567                 }
 568                 flushes--;
 569         }
 570         /* it is no error to fetch into a completely empty repo */
 571         return count ? retval : 0;
 572 }
 573
 574 static struct commit_list *complete;
 575
 576 static int mark_complete(const struct object_id *oid)
 577 {
 578         struct object *o = parse_object(the_repository, oid);
 579
 580         while (o && o->type == OBJ_TAG) {
 581                 struct tag *t = (struct tag *) o;
 582                 if (!t->tagged)
 583                         break; /* broken repository */
 584                 o->flags |= COMPLETE;
 585                 o = parse_object(the_repository, &t->tagged->oid);
 586         }
 587         if (o && o->type == OBJ_COMMIT) {
 588                 struct commit *commit = (struct commit *)o;
 589                 if (!(commit->object.flags & COMPLETE)) {
 590                         commit->object.flags |= COMPLETE;
 591                         commit_list_insert(commit, &complete);
 592                 }
 593         }
 594         return 0;
 595 }
 596
 597 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 598                              int flag, void *cb_data)
 599 {
 600         return mark_complete(oid);
 601 }
 602
 603 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 604                                          timestamp_t cutoff)
 605 {
 606         while (complete && cutoff <= complete->item->date) {
 607                 print_verbose(args, _("Marking %s as complete"),
 608                               oid_to_hex(&complete->item->object.oid));
 609                 pop_most_recent_commit(&complete, COMPLETE);
 610         }
 611 }
 612
 613 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 614 {
 615         for (; refs; refs = refs->next)
 616                 oidset_insert(oids, &refs->old_oid);
 617 }
 618
 619 static int tip_oids_contain(struct oidset *tip_oids,
 620                             struct ref *unmatched, struct ref *newlist,
 621                             const struct object_id *id)
 622 {
 623         /*
 624          * Note that this only looks at the ref lists the first time it's
 625          * called. This works out in filter_refs() because even though it may
 626          * add to "newlist" between calls, the additions will always be for
 627          * oids that are already in the set.
 628          */
 629         if (!tip_oids->map.map.tablesize) {
 630                 add_refs_to_oidset(tip_oids, unmatched);
 631                 add_refs_to_oidset(tip_oids, newlist);
 632         }
 633         return oidset_contains(tip_oids, id);
 634 }
 635
 636 static void filter_refs(struct fetch_pack_args *args,
 637                         struct ref **refs,
 638                         struct ref **sought, int nr_sought)
 639 {
 640         struct ref *newlist = NULL;
 641         struct ref **newtail = &newlist;
 642         struct ref *unmatched = NULL;
 643         struct ref *ref, *next;
 644         struct oidset tip_oids = OIDSET_INIT;
 645         int i;
 646
 647         i = 0;
 648         for (ref = *refs; ref; ref = next) {
 649                 int keep = 0;
 650                 next = ref->next;
 651
 652                 if (starts_with(ref->name, "refs/") &&
 653                     check_refname_format(ref->name, 0))
 654                         ; /* trash */
 655                 else {
 656                         while (i < nr_sought) {
 657                                 int cmp = strcmp(ref->name, sought[i]->name);
 658                                 if (cmp < 0)
 659                                         break; /* definitely do not have it */
 660                                 else if (cmp == 0) {
 661                                         keep = 1; /* definitely have it */
 662                                         sought[i]->match_status = REF_MATCHED;
 663                                 }
 664                                 i++;
 665                         }
 666
 667                         if (!keep && args->fetch_all &&
 668                             (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 669                                 keep = 1;
 670                 }
 671
 672                 if (keep) {
 673                         *newtail = ref;
 674                         ref->next = NULL;
 675                         newtail = &ref->next;
 676                 } else {
 677                         ref->next = unmatched;
 678                         unmatched = ref;
 679                 }
 680         }
 681
 682         /* Append unmatched requests to the list */
 683         for (i = 0; i < nr_sought; i++) {
 684                 struct object_id oid;
 685                 const char *p;
 686
 687                 ref = sought[i];
 688                 if (ref->match_status != REF_NOT_MATCHED)
 689                         continue;
 690                 if (parse_oid_hex(ref->name, &oid, &p) ||
 691                     *p != '\0' ||
 692                     oidcmp(&oid, &ref->old_oid))
 693                         continue;
 694
 695                 if ((allow_unadvertised_object_request &
 696                      (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 697                     tip_oids_contain(&tip_oids, unmatched, newlist,
 698                                      &ref->old_oid)) {
 699                         ref->match_status = REF_MATCHED;
 700                         *newtail = copy_ref(ref);
 701                         newtail = &(*newtail)->next;
 702                 } else {
 703                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 704                 }
 705         }
 706
 707         oidset_clear(&tip_oids);
 708         for (ref = unmatched; ref; ref = next) {
 709                 next = ref->next;
 710                 free(ref);
 711         }
 712
 713         *refs = newlist;
 714 }
 715
 716 static void mark_alternate_complete(struct object *obj)
 717 {
 718         mark_complete(&obj->oid);
 719 }
 720
 721 struct loose_object_iter {
 722         struct oidset *loose_object_set;
 723         struct ref *refs;
 724 };
 725
 726 /*
 727  *  If the number of refs is not larger than the number of loose objects,
 728  *  this function stops inserting.
 729  */
 730 static int add_loose_objects_to_set(const struct object_id *oid,
 731                                     const char *path,
 732                                     void *data)
 733 {
 734         struct loose_object_iter *iter = data;
 735         oidset_insert(iter->loose_object_set, oid);
 736         if (iter->refs == NULL)
 737                 return 1;
 738
 739         iter->refs = iter->refs->next;
 740         return 0;
 741 }
 742
 743 static int everything_local(struct fetch_pack_args *args,
 744                             struct ref **refs,
 745                             struct ref **sought, int nr_sought)
 746 {
 747         struct ref *ref;
 748         int retval;
 749         int old_save_commit_buffer = save_commit_buffer;
 750         timestamp_t cutoff = 0;
 751         struct oidset loose_oid_set = OIDSET_INIT;
 752         int use_oidset = 0;
 753         struct loose_object_iter iter = {&loose_oid_set, *refs};
 754
 755         /* Enumerate all loose objects or know refs are not so many. */
 756         use_oidset = !for_each_loose_object(add_loose_objects_to_set,
 757                                             &iter, 0);
 758
 759         save_commit_buffer = 0;
 760
 761         for (ref = *refs; ref; ref = ref->next) {
 762                 struct object *o;
 763                 unsigned int flags = OBJECT_INFO_QUICK;
 764
 765                 if (use_oidset &&
 766                     !oidset_contains(&loose_oid_set, &ref->old_oid)) {
 767                         /*
 768                          * I know this does not exist in the loose form,
 769                          * so check if it exists in a non-loose form.
 770                          */
 771                         flags |= OBJECT_INFO_IGNORE_LOOSE;
 772                 }
 773
 774                 if (!has_object_file_with_flags(&ref->old_oid, flags))
 775                         continue;
 776                 o = parse_object(the_repository, &ref->old_oid);
 777                 if (!o)
 778                         continue;
 779
 780                 /* We already have it -- which may mean that we were
 781                  * in sync with the other side at some time after
 782                  * that (it is OK if we guess wrong here).
 783                  */
 784                 if (o->type == OBJ_COMMIT) {
 785                         struct commit *commit = (struct commit *)o;
 786                         if (!cutoff || cutoff < commit->date)
 787                                 cutoff = commit->date;
 788                 }
 789         }
 790
 791         oidset_clear(&loose_oid_set);
 792
 793         if (!args->no_dependents) {
 794                 if (!args->deepen) {
 795                         for_each_ref(mark_complete_oid, NULL);
 796                         for_each_cached_alternate(mark_alternate_complete);
 797                         commit_list_sort_by_date(&complete);
 798                         if (cutoff)
 799                                 mark_recent_complete_commits(args, cutoff);
 800                 }
 801
 802                 /*
 803                  * Mark all complete remote refs as common refs.
 804                  * Don't mark them common yet; the server has to be told so first.
 805                  */
 806                 for (ref = *refs; ref; ref = ref->next) {
 807                         struct object *o = deref_tag(the_repository,
 808                                                      lookup_object(the_repository,
 809                                                      ref->old_oid.hash),
 810                                                      NULL, 0);
 811
 812                         if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 813                                 continue;
 814
 815                         if (!(o->flags & SEEN)) {
 816                                 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 817
 818                                 mark_common((struct commit *)o, 1, 1);
 819                         }
 820                 }
 821         }
 822
 823         filter_refs(args, refs, sought, nr_sought);
 824
 825         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 826                 const struct object_id *remote = &ref->old_oid;
 827                 struct object *o;
 828
 829                 o = lookup_object(the_repository, remote->hash);
 830                 if (!o || !(o->flags & COMPLETE)) {
 831                         retval = 0;
 832                         print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 833                                       ref->name);
 834                         continue;
 835                 }
 836                 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 837                               ref->name);
 838         }
 839
 840         save_commit_buffer = old_save_commit_buffer;
 841
 842         return retval;
 843 }
 844
 845 static int sideband_demux(int in, int out, void *data)
 846 {
 847         int *xd = data;
 848         int ret;
 849
 850         ret = recv_sideband("fetch-pack", xd[0], out);
 851         close(out);
 852         return ret;
 853 }
 854
 855 static int get_pack(struct fetch_pack_args *args,
 856                     int xd[2], char **pack_lockfile)
 857 {
 858         struct async demux;
 859         int do_keep = args->keep_pack;
 860         const char *cmd_name;
 861         struct pack_header header;
 862         int pass_header = 0;
 863         struct child_process cmd = CHILD_PROCESS_INIT;
 864         int ret;
 865
 866         memset(&demux, 0, sizeof(demux));
 867         if (use_sideband) {
 868                 /* xd[] is talking with upload-pack; subprocess reads from
 869                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 870                  * through demux->out.
 871                  */
 872                 demux.proc = sideband_demux;
 873                 demux.data = xd;
 874                 demux.out = -1;
 875                 demux.isolate_sigpipe = 1;
 876                 if (start_async(&demux))
 877                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 878         }
 879         else
 880                 demux.out = xd[0];
 881
 882         if (!args->keep_pack && unpack_limit) {
 883
 884                 if (read_pack_header(demux.out, &header))
 885                         die(_("protocol error: bad pack header"));
 886                 pass_header = 1;
 887                 if (ntohl(header.hdr_entries) < unpack_limit)
 888                         do_keep = 0;
 889                 else
 890                         do_keep = 1;
 891         }
 892
 893         if (alternate_shallow_file) {
 894                 argv_array_push(&cmd.args, "--shallow-file");
 895                 argv_array_push(&cmd.args, alternate_shallow_file);
 896         }
 897
 898         if (do_keep || args->from_promisor) {
 899                 if (pack_lockfile)
 900                         cmd.out = -1;
 901                 cmd_name = "index-pack";
 902                 argv_array_push(&cmd.args, cmd_name);
 903                 argv_array_push(&cmd.args, "--stdin");
 904                 if (!args->quiet && !args->no_progress)
 905                         argv_array_push(&cmd.args, "-v");
 906                 if (args->use_thin_pack)
 907                         argv_array_push(&cmd.args, "--fix-thin");
 908                 if (do_keep && (args->lock_pack || unpack_limit)) {
 909                         char hostname[HOST_NAME_MAX + 1];
 910                         if (xgethostname(hostname, sizeof(hostname)))
 911                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 912                         argv_array_pushf(&cmd.args,
 913                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 914                                         (uintmax_t)getpid(), hostname);
 915                 }
 916                 if (args->check_self_contained_and_connected)
 917                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 918                 if (args->from_promisor)
 919                         argv_array_push(&cmd.args, "--promisor");
 920         }
 921         else {
 922                 cmd_name = "unpack-objects";
 923                 argv_array_push(&cmd.args, cmd_name);
 924                 if (args->quiet || args->no_progress)
 925                         argv_array_push(&cmd.args, "-q");
 926                 args->check_self_contained_and_connected = 0;
 927         }
 928
 929         if (pass_header)
 930                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 931                                  ntohl(header.hdr_version),
 932                                  ntohl(header.hdr_entries));
 933         if (fetch_fsck_objects >= 0
 934             ? fetch_fsck_objects
 935             : transfer_fsck_objects >= 0
 936             ? transfer_fsck_objects
 937             : 0) {
 938                 if (args->from_promisor)
 939                         /*
 940                          * We cannot use --strict in index-pack because it
 941                          * checks both broken objects and links, but we only
 942                          * want to check for broken objects.
 943                          */
 944                         argv_array_push(&cmd.args, "--fsck-objects");
 945                 else
 946                         argv_array_push(&cmd.args, "--strict");
 947         }
 948
 949         cmd.in = demux.out;
 950         cmd.git_cmd = 1;
 951         if (start_command(&cmd))
 952                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 953         if (do_keep && pack_lockfile) {
 954                 *pack_lockfile = index_pack_lockfile(cmd.out);
 955                 close(cmd.out);
 956         }
 957
 958         if (!use_sideband)
 959                 /* Closed by start_command() */
 960                 xd[0] = -1;
 961
 962         ret = finish_command(&cmd);
 963         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 964                 args->self_contained_and_connected =
 965                         args->check_self_contained_and_connected &&
 966                         ret == 0;
 967         else
 968                 die(_("%s failed"), cmd_name);
 969         if (use_sideband && finish_async(&demux))
 970                 die(_("error in sideband demultiplexer"));
 971         return 0;
 972 }
 973
 974 static int cmp_ref_by_name(const void *a_, const void *b_)
 975 {
 976         const struct ref *a = *((const struct ref **)a_);
 977         const struct ref *b = *((const struct ref **)b_);
 978         return strcmp(a->name, b->name);
 979 }
 980
 981 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 982                                  int fd[2],
 983                                  const struct ref *orig_ref,
 984                                  struct ref **sought, int nr_sought,
 985                                  struct shallow_info *si,
 986                                  char **pack_lockfile)
 987 {
 988         struct ref *ref = copy_ref_list(orig_ref);
 989         struct object_id oid;
 990         const char *agent_feature;
 991         int agent_len;
 992
 993         sort_ref_list(&ref, ref_compare_name);
 994         QSORT(sought, nr_sought, cmp_ref_by_name);
 995
 996         if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
 997                 die(_("Server does not support shallow clients"));
 998         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 999                 args->deepen = 1;
1000         if (server_supports("multi_ack_detailed")) {
1001                 print_verbose(args, _("Server supports multi_ack_detailed"));
1002                 multi_ack = 2;
1003                 if (server_supports("no-done")) {
1004                         print_verbose(args, _("Server supports no-done"));
1005                         if (args->stateless_rpc)
1006                                 no_done = 1;
1007                 }
1008         }
1009         else if (server_supports("multi_ack")) {
1010                 print_verbose(args, _("Server supports multi_ack"));
1011                 multi_ack = 1;
1012         }
1013         if (server_supports("side-band-64k")) {
1014                 print_verbose(args, _("Server supports side-band-64k"));
1015                 use_sideband = 2;
1016         }
1017         else if (server_supports("side-band")) {
1018                 print_verbose(args, _("Server supports side-band"));
1019                 use_sideband = 1;
1020         }
1021         if (server_supports("allow-tip-sha1-in-want")) {
1022                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
1023                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1024         }
1025         if (server_supports("allow-reachable-sha1-in-want")) {
1026                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
1027                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1028         }
1029         if (!server_supports("thin-pack"))
1030                 args->use_thin_pack = 0;
1031         if (!server_supports("no-progress"))
1032                 args->no_progress = 0;
1033         if (!server_supports("include-tag"))
1034                 args->include_tag = 0;
1035         if (server_supports("ofs-delta"))
1036                 print_verbose(args, _("Server supports ofs-delta"));
1037         else
1038                 prefer_ofs_delta = 0;
1039
1040         if (server_supports("filter")) {
1041                 server_supports_filtering = 1;
1042                 print_verbose(args, _("Server supports filter"));
1043         } else if (args->filter_options.choice) {
1044                 warning("filtering not recognized by server, ignoring");
1045         }
1046
1047         if ((agent_feature = server_feature_value("agent", &agent_len))) {
1048                 agent_supported = 1;
1049                 if (agent_len)
1050                         print_verbose(args, _("Server version is %.*s"),
1051                                       agent_len, agent_feature);
1052         }
1053         if (server_supports("deepen-since"))
1054                 deepen_since_ok = 1;
1055         else if (args->deepen_since)
1056                 die(_("Server does not support --shallow-since"));
1057         if (server_supports("deepen-not"))
1058                 deepen_not_ok = 1;
1059         else if (args->deepen_not)
1060                 die(_("Server does not support --shallow-exclude"));
1061         if (!server_supports("deepen-relative") && args->deepen_relative)
1062                 die(_("Server does not support --deepen"));
1063
1064         if (everything_local(args, &ref, sought, nr_sought)) {
1065                 packet_flush(fd[1]);
1066                 goto all_done;
1067         }
1068         if (find_common(args, fd, &oid, ref) < 0)
1069                 if (!args->keep_pack)
1070                         /* When cloning, it is not unusual to have
1071                          * no common commit.
1072                          */
1073                         warning(_("no common commits"));
1074
1075         if (args->stateless_rpc)
1076                 packet_flush(fd[1]);
1077         if (args->deepen)
1078                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1079                                         NULL);
1080         else if (si->nr_ours || si->nr_theirs)
1081                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1082         else
1083                 alternate_shallow_file = NULL;
1084         if (get_pack(args, fd, pack_lockfile))
1085                 die(_("git fetch-pack: fetch failed."));
1086
1087  all_done:
1088         return ref;
1089 }
1090
1091 static void add_shallow_requests(struct strbuf *req_buf,
1092                                  const struct fetch_pack_args *args)
1093 {
1094         if (is_repository_shallow(the_repository))
1095                 write_shallow_commits(req_buf, 1, NULL);
1096         if (args->depth > 0)
1097                 packet_buf_write(req_buf, "deepen %d", args->depth);
1098         if (args->deepen_since) {
1099                 timestamp_t max_age = approxidate(args->deepen_since);
1100                 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1101         }
1102         if (args->deepen_not) {
1103                 int i;
1104                 for (i = 0; i < args->deepen_not->nr; i++) {
1105                         struct string_list_item *s = args->deepen_not->items + i;
1106                         packet_buf_write(req_buf, "deepen-not %s", s->string);
1107                 }
1108         }
1109 }
1110
1111 static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1112 {
1113         for ( ; wants ; wants = wants->next) {
1114                 const struct object_id *remote = &wants->old_oid;
1115                 const char *remote_hex;
1116                 struct object *o;
1117
1118                 /*
1119                  * If that object is complete (i.e. it is an ancestor of a
1120                  * local ref), we tell them we have it but do not have to
1121                  * tell them about its ancestors, which they already know
1122                  * about.
1123                  *
1124                  * We use lookup_object here because we are only
1125                  * interested in the case we *know* the object is
1126                  * reachable and we have already scanned it.
1127                  */
1128                 if (((o = lookup_object(the_repository, remote->hash)) != NULL) &&
1129                     (o->flags & COMPLETE)) {
1130                         continue;
1131                 }
1132
1133                 remote_hex = oid_to_hex(remote);
1134                 packet_buf_write(req_buf, "want %s\n", remote_hex);
1135         }
1136 }
1137
1138 static void add_common(struct strbuf *req_buf, struct oidset *common)
1139 {
1140         struct oidset_iter iter;
1141         const struct object_id *oid;
1142         oidset_iter_init(common, &iter);
1143
1144         while ((oid = oidset_iter_next(&iter))) {
1145                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1146         }
1147 }
1148
1149 static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
1150 {
1151         int ret = 0;
1152         int haves_added = 0;
1153         const struct object_id *oid;
1154
1155         while ((oid = get_rev())) {
1156                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1157                 if (++haves_added >= *haves_to_send)
1158                         break;
1159         }
1160
1161         *in_vain += haves_added;
1162         if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1163                 /* Send Done */
1164                 packet_buf_write(req_buf, "done\n");
1165                 ret = 1;
1166         }
1167
1168         /* Increase haves to send on next round */
1169         *haves_to_send = next_flush(1, *haves_to_send);
1170
1171         return ret;
1172 }
1173
1174 static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
1175                               const struct ref *wants, struct oidset *common,
1176                               int *haves_to_send, int *in_vain)
1177 {
1178         int ret = 0;
1179         struct strbuf req_buf = STRBUF_INIT;
1180
1181         if (server_supports_v2("fetch", 1))
1182                 packet_buf_write(&req_buf, "command=fetch");
1183         if (server_supports_v2("agent", 0))
1184                 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1185         if (args->server_options && args->server_options->nr &&
1186             server_supports_v2("server-option", 1)) {
1187                 int i;
1188                 for (i = 0; i < args->server_options->nr; i++)
1189                         packet_write_fmt(fd_out, "server-option=%s",
1190                                          args->server_options->items[i].string);
1191         }
1192
1193         packet_buf_delim(&req_buf);
1194         if (args->use_thin_pack)
1195                 packet_buf_write(&req_buf, "thin-pack");
1196         if (args->no_progress)
1197                 packet_buf_write(&req_buf, "no-progress");
1198         if (args->include_tag)
1199                 packet_buf_write(&req_buf, "include-tag");
1200         if (prefer_ofs_delta)
1201                 packet_buf_write(&req_buf, "ofs-delta");
1202
1203         /* Add shallow-info and deepen request */
1204         if (server_supports_feature("fetch", "shallow", 0))
1205                 add_shallow_requests(&req_buf, args);
1206         else if (is_repository_shallow(the_repository) || args->deepen)
1207                 die(_("Server does not support shallow requests"));
1208
1209         /* Add filter */
1210         if (server_supports_feature("fetch", "filter", 0) &&
1211             args->filter_options.choice) {
1212                 print_verbose(args, _("Server supports filter"));
1213                 packet_buf_write(&req_buf, "filter %s",
1214                                  args->filter_options.filter_spec);
1215         } else if (args->filter_options.choice) {
1216                 warning("filtering not recognized by server, ignoring");
1217         }
1218
1219         /* add wants */
1220         add_wants(wants, &req_buf);
1221
1222         if (args->no_dependents) {
1223                 packet_buf_write(&req_buf, "done");
1224                 ret = 1;
1225         } else {
1226                 /* Add all of the common commits we've found in previous rounds */
1227                 add_common(&req_buf, common);
1228
1229                 /* Add initial haves */
1230                 ret = add_haves(&req_buf, haves_to_send, in_vain);
1231         }
1232
1233         /* Send request */
1234         packet_buf_flush(&req_buf);
1235         write_or_die(fd_out, req_buf.buf, req_buf.len);
1236
1237         strbuf_release(&req_buf);
1238         return ret;
1239 }
1240
1241 /*
1242  * Processes a section header in a server's response and checks if it matches
1243  * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1244  * not consumed); if 0, the line will be consumed and the function will die if
1245  * the section header doesn't match what was expected.
1246  */
1247 static int process_section_header(struct packet_reader *reader,
1248                                   const char *section, int peek)
1249 {
1250         int ret;
1251
1252         if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1253                 die("error reading section header '%s'", section);
1254
1255         ret = !strcmp(reader->line, section);
1256
1257         if (!peek) {
1258                 if (!ret)
1259                         die("expected '%s', received '%s'",
1260                             section, reader->line);
1261                 packet_reader_read(reader);
1262         }
1263
1264         return ret;
1265 }
1266
1267 static int process_acks(struct packet_reader *reader, struct oidset *common)
1268 {
1269         /* received */
1270         int received_ready = 0;
1271         int received_ack = 0;
1272
1273         process_section_header(reader, "acknowledgments", 0);
1274         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1275                 const char *arg;
1276
1277                 if (!strcmp(reader->line, "NAK"))
1278                         continue;
1279
1280                 if (skip_prefix(reader->line, "ACK ", &arg)) {
1281                         struct object_id oid;
1282                         if (!get_oid_hex(arg, &oid)) {
1283                                 struct commit *commit;
1284                                 oidset_insert(common, &oid);
1285                                 commit = lookup_commit(the_repository, &oid);
1286                                 mark_common(commit, 0, 1);
1287                         }
1288                         continue;
1289                 }
1290
1291                 if (!strcmp(reader->line, "ready")) {
1292                         clear_prio_queue(&rev_list);
1293                         received_ready = 1;
1294                         continue;
1295                 }
1296
1297                 die("unexpected acknowledgment line: '%s'", reader->line);
1298         }
1299
1300         if (reader->status != PACKET_READ_FLUSH &&
1301             reader->status != PACKET_READ_DELIM)
1302                 die("error processing acks: %d", reader->status);
1303
1304         /* return 0 if no common, 1 if there are common, or 2 if ready */
1305         return received_ready ? 2 : (received_ack ? 1 : 0);
1306 }
1307
1308 static void receive_shallow_info(struct fetch_pack_args *args,
1309                                  struct packet_reader *reader)
1310 {
1311         process_section_header(reader, "shallow-info", 0);
1312         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1313                 const char *arg;
1314                 struct object_id oid;
1315
1316                 if (skip_prefix(reader->line, "shallow ", &arg)) {
1317                         if (get_oid_hex(arg, &oid))
1318                                 die(_("invalid shallow line: %s"), reader->line);
1319                         register_shallow(the_repository, &oid);
1320                         continue;
1321                 }
1322                 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1323                         if (get_oid_hex(arg, &oid))
1324                                 die(_("invalid unshallow line: %s"), reader->line);
1325                         if (!lookup_object(the_repository, oid.hash))
1326                                 die(_("object not found: %s"), reader->line);
1327                         /* make sure that it is parsed as shallow */
1328                         if (!parse_object(the_repository, &oid))
1329                                 die(_("error in object: %s"), reader->line);
1330                         if (unregister_shallow(&oid))
1331                                 die(_("no shallow found: %s"), reader->line);
1332                         continue;
1333                 }
1334                 die(_("expected shallow/unshallow, got %s"), reader->line);
1335         }
1336
1337         if (reader->status != PACKET_READ_FLUSH &&
1338             reader->status != PACKET_READ_DELIM)
1339                 die("error processing shallow info: %d", reader->status);
1340
1341         setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1342         args->deepen = 1;
1343 }
1344
1345 enum fetch_state {
1346         FETCH_CHECK_LOCAL = 0,
1347         FETCH_SEND_REQUEST,
1348         FETCH_PROCESS_ACKS,
1349         FETCH_GET_PACK,
1350         FETCH_DONE,
1351 };
1352
1353 static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1354                                     int fd[2],
1355                                     const struct ref *orig_ref,
1356                                     struct ref **sought, int nr_sought,
1357                                     char **pack_lockfile)
1358 {
1359         struct ref *ref = copy_ref_list(orig_ref);
1360         enum fetch_state state = FETCH_CHECK_LOCAL;
1361         struct oidset common = OIDSET_INIT;
1362         struct packet_reader reader;
1363         int in_vain = 0;
1364         int haves_to_send = INITIAL_FLUSH;
1365         packet_reader_init(&reader, fd[0], NULL, 0,
1366                            PACKET_READ_CHOMP_NEWLINE);
1367
1368         while (state != FETCH_DONE) {
1369                 switch (state) {
1370                 case FETCH_CHECK_LOCAL:
1371                         sort_ref_list(&ref, ref_compare_name);
1372                         QSORT(sought, nr_sought, cmp_ref_by_name);
1373
1374                         /* v2 supports these by default */
1375                         allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1376                         use_sideband = 2;
1377                         if (args->depth > 0 || args->deepen_since || args->deepen_not)
1378                                 args->deepen = 1;
1379
1380                         if (marked)
1381                                 for_each_ref(clear_marks, NULL);
1382                         marked = 1;
1383
1384                         for_each_ref(rev_list_insert_ref_oid, NULL);
1385                         for_each_cached_alternate(insert_one_alternate_object);
1386
1387                         /* Filter 'ref' by 'sought' and those that aren't local */
1388                         if (everything_local(args, &ref, sought, nr_sought))
1389                                 state = FETCH_DONE;
1390                         else
1391                                 state = FETCH_SEND_REQUEST;
1392                         break;
1393                 case FETCH_SEND_REQUEST:
1394                         if (send_fetch_request(fd[1], args, ref, &common,
1395                                                &haves_to_send, &in_vain))
1396                                 state = FETCH_GET_PACK;
1397                         else
1398                                 state = FETCH_PROCESS_ACKS;
1399                         break;
1400                 case FETCH_PROCESS_ACKS:
1401                         /* Process ACKs/NAKs */
1402                         switch (process_acks(&reader, &common)) {
1403                         case 2:
1404                                 state = FETCH_GET_PACK;
1405                                 break;
1406                         case 1:
1407                                 in_vain = 0;
1408                                 /* fallthrough */
1409                         default:
1410                                 state = FETCH_SEND_REQUEST;
1411                                 break;
1412                         }
1413                         break;
1414                 case FETCH_GET_PACK:
1415                         /* Check for shallow-info section */
1416                         if (process_section_header(&reader, "shallow-info", 1))
1417                                 receive_shallow_info(args, &reader);
1418
1419                         /* get the pack */
1420                         process_section_header(&reader, "packfile", 0);
1421                         if (get_pack(args, fd, pack_lockfile))
1422                                 die(_("git fetch-pack: fetch failed."));
1423
1424                         state = FETCH_DONE;
1425                         break;
1426                 case FETCH_DONE:
1427                         continue;
1428                 }
1429         }
1430
1431         oidset_clear(&common);
1432         return ref;
1433 }
1434
1435 static void fetch_pack_config(void)
1436 {
1437         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1438         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1439         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1440         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1441         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1442
1443         git_config(git_default_config, NULL);
1444 }
1445
1446 static void fetch_pack_setup(void)
1447 {
1448         static int did_setup;
1449         if (did_setup)
1450                 return;
1451         fetch_pack_config();
1452         if (0 <= transfer_unpack_limit)
1453                 unpack_limit = transfer_unpack_limit;
1454         else if (0 <= fetch_unpack_limit)
1455                 unpack_limit = fetch_unpack_limit;
1456         did_setup = 1;
1457 }
1458
1459 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1460 {
1461         struct string_list names = STRING_LIST_INIT_NODUP;
1462         int src, dst;
1463
1464         for (src = dst = 0; src < nr; src++) {
1465                 struct string_list_item *item;
1466                 item = string_list_insert(&names, ref[src]->name);
1467                 if (item->util)
1468                         continue; /* already have it */
1469                 item->util = ref[src];
1470                 if (src != dst)
1471                         ref[dst] = ref[src];
1472                 dst++;
1473         }
1474         for (src = dst; src < nr; src++)
1475                 ref[src] = NULL;
1476         string_list_clear(&names, 0);
1477         return dst;
1478 }
1479
1480 static void update_shallow(struct fetch_pack_args *args,
1481                            struct ref **sought, int nr_sought,
1482                            struct shallow_info *si)
1483 {
1484         struct oid_array ref = OID_ARRAY_INIT;
1485         int *status;
1486         int i;
1487
1488         if (args->deepen && alternate_shallow_file) {
1489                 if (*alternate_shallow_file == '\0') { /* --unshallow */
1490                         unlink_or_warn(git_path_shallow(the_repository));
1491                         rollback_lock_file(&shallow_lock);
1492                 } else
1493                         commit_lock_file(&shallow_lock);
1494                 return;
1495         }
1496
1497         if (!si->shallow || !si->shallow->nr)
1498                 return;
1499
1500         if (args->cloning) {
1501                 /*
1502                  * remote is shallow, but this is a clone, there are
1503                  * no objects in repo to worry about. Accept any
1504                  * shallow points that exist in the pack (iow in repo
1505                  * after get_pack() and reprepare_packed_git())
1506                  */
1507                 struct oid_array extra = OID_ARRAY_INIT;
1508                 struct object_id *oid = si->shallow->oid;
1509                 for (i = 0; i < si->shallow->nr; i++)
1510                         if (has_object_file(&oid[i]))
1511                                 oid_array_append(&extra, &oid[i]);
1512                 if (extra.nr) {
1513                         setup_alternate_shallow(&shallow_lock,
1514                                                 &alternate_shallow_file,
1515                                                 &extra);
1516                         commit_lock_file(&shallow_lock);
1517                 }
1518                 oid_array_clear(&extra);
1519                 return;
1520         }
1521
1522         if (!si->nr_ours && !si->nr_theirs)
1523                 return;
1524
1525         remove_nonexistent_theirs_shallow(si);
1526         if (!si->nr_ours && !si->nr_theirs)
1527                 return;
1528         for (i = 0; i < nr_sought; i++)
1529                 oid_array_append(&ref, &sought[i]->old_oid);
1530         si->ref = &ref;
1531
1532         if (args->update_shallow) {
1533                 /*
1534                  * remote is also shallow, .git/shallow may be updated
1535                  * so all refs can be accepted. Make sure we only add
1536                  * shallow roots that are actually reachable from new
1537                  * refs.
1538                  */
1539                 struct oid_array extra = OID_ARRAY_INIT;
1540                 struct object_id *oid = si->shallow->oid;
1541                 assign_shallow_commits_to_refs(si, NULL, NULL);
1542                 if (!si->nr_ours && !si->nr_theirs) {
1543                         oid_array_clear(&ref);
1544                         return;
1545                 }
1546                 for (i = 0; i < si->nr_ours; i++)
1547                         oid_array_append(&extra, &oid[si->ours[i]]);
1548                 for (i = 0; i < si->nr_theirs; i++)
1549                         oid_array_append(&extra, &oid[si->theirs[i]]);
1550                 setup_alternate_shallow(&shallow_lock,
1551                                         &alternate_shallow_file,
1552                                         &extra);
1553                 commit_lock_file(&shallow_lock);
1554                 oid_array_clear(&extra);
1555                 oid_array_clear(&ref);
1556                 return;
1557         }
1558
1559         /*
1560          * remote is also shallow, check what ref is safe to update
1561          * without updating .git/shallow
1562          */
1563         status = xcalloc(nr_sought, sizeof(*status));
1564         assign_shallow_commits_to_refs(si, NULL, status);
1565         if (si->nr_ours || si->nr_theirs) {
1566                 for (i = 0; i < nr_sought; i++)
1567                         if (status[i])
1568                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1569         }
1570         free(status);
1571         oid_array_clear(&ref);
1572 }
1573
1574 struct ref *fetch_pack(struct fetch_pack_args *args,
1575                        int fd[], struct child_process *conn,
1576                        const struct ref *ref,
1577                        const char *dest,
1578                        struct ref **sought, int nr_sought,
1579                        struct oid_array *shallow,
1580                        char **pack_lockfile,
1581                        enum protocol_version version)
1582 {
1583         struct ref *ref_cpy;
1584         struct shallow_info si;
1585
1586         fetch_pack_setup();
1587         if (nr_sought)
1588                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1589
1590         if (!ref) {
1591                 packet_flush(fd[1]);
1592                 die(_("no matching remote head"));
1593         }
1594         prepare_shallow_info(&si, shallow);
1595         if (version == protocol_v2)
1596                 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1597                                            pack_lockfile);
1598         else
1599                 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1600                                         &si, pack_lockfile);
1601         reprepare_packed_git(the_repository);
1602         update_shallow(args, sought, nr_sought, &si);
1603         clear_shallow_info(&si);
1604         return ref_cpy;
1605 }
1606
1607 int report_unmatched_refs(struct ref **sought, int nr_sought)
1608 {
1609         int i, ret = 0;
1610
1611         for (i = 0; i < nr_sought; i++) {
1612                 if (!sought[i])
1613                         continue;
1614                 switch (sought[i]->match_status) {
1615                 case REF_MATCHED:
1616                         continue;
1617                 case REF_NOT_MATCHED:
1618                         error(_("no such remote ref %s"), sought[i]->name);
1619                         break;
1620                 case REF_UNADVERTISED_NOT_ALLOWED:
1621                         error(_("Server does not allow request for unadvertised object %s"),
1622                               sought[i]->name);
1623                         break;
1624                 }
1625                 ret = 1;
1626         }
1627         return ret;
1628 }