git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "repository.h"
   3 #include "config.h"
   4 #include "lockfile.h"
   5 #include "refs.h"
   6 #include "pkt-line.h"
   7 #include "commit.h"
   8 #include "tag.h"
   9 #include "exec-cmd.h"
  10 #include "pack.h"
  11 #include "sideband.h"
  12 #include "fetch-pack.h"
  13 #include "remote.h"
  14 #include "run-command.h"
  15 #include "connect.h"
  16 #include "transport.h"
  17 #include "version.h"
  18 #include "prio-queue.h"
  19 #include "sha1-array.h"
  20 #include "oidset.h"
  21 #include "packfile.h"
  22 #include "object-store.h"
  23
  24 static int transfer_unpack_limit = -1;
  25 static int fetch_unpack_limit = -1;
  26 static int unpack_limit = 100;
  27 static int prefer_ofs_delta = 1;
  28 static int no_done;
  29 static int deepen_since_ok;
  30 static int deepen_not_ok;
  31 static int fetch_fsck_objects = -1;
  32 static int transfer_fsck_objects = -1;
  33 static int agent_supported;
  34 static int server_supports_filtering;
  35 static struct lock_file shallow_lock;
  36 static const char *alternate_shallow_file;
  37
  38 /* Remember to update object flag allocation in object.h */
  39 #define COMPLETE        (1U << 0)
  40 #define COMMON          (1U << 1)
  41 #define COMMON_REF      (1U << 2)
  42 #define SEEN            (1U << 3)
  43 #define POPPED          (1U << 4)
  44 #define ALTERNATE       (1U << 5)
  45
  46 static int marked;
  47
  48 /*
  49  * After sending this many "have"s if we do not get any new ACK , we
  50  * give up traversing our history.
  51  */
  52 #define MAX_IN_VAIN 256
  53
  54 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  55 static int non_common_revs, multi_ack, use_sideband;
  56 /* Allow specifying sha1 if it is a ref tip. */
  57 #define ALLOW_TIP_SHA1  01
  58 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  59 #define ALLOW_REACHABLE_SHA1    02
  60 static unsigned int allow_unadvertised_object_request;
  61
  62 __attribute__((format (printf, 2, 3)))
  63 static inline void print_verbose(const struct fetch_pack_args *args,
  64                                  const char *fmt, ...)
  65 {
  66         va_list params;
  67
  68         if (!args->verbose)
  69                 return;
  70
  71         va_start(params, fmt);
  72         vfprintf(stderr, fmt, params);
  73         va_end(params);
  74         fputc('\n', stderr);
  75 }
  76
  77 struct alternate_object_cache {
  78         struct object **items;
  79         size_t nr, alloc;
  80 };
  81
  82 static void cache_one_alternate(const char *refname,
  83                                 const struct object_id *oid,
  84                                 void *vcache)
  85 {
  86         struct alternate_object_cache *cache = vcache;
  87         struct object *obj = parse_object(oid);
  88
  89         if (!obj || (obj->flags & ALTERNATE))
  90                 return;
  91
  92         obj->flags |= ALTERNATE;
  93         ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  94         cache->items[cache->nr++] = obj;
  95 }
  96
  97 static void for_each_cached_alternate(void (*cb)(struct object *))
  98 {
  99         static int initialized;
 100         static struct alternate_object_cache cache;
 101         size_t i;
 102
 103         if (!initialized) {
 104                 for_each_alternate_ref(cache_one_alternate, &cache);
 105                 initialized = 1;
 106         }
 107
 108         for (i = 0; i < cache.nr; i++)
 109                 cb(cache.items[i]);
 110 }
 111
 112 static void rev_list_push(struct commit *commit, int mark)
 113 {
 114         if (!(commit->object.flags & mark)) {
 115                 commit->object.flags |= mark;
 116
 117                 if (parse_commit(commit))
 118                         return;
 119
 120                 prio_queue_put(&rev_list, commit);
 121
 122                 if (!(commit->object.flags & COMMON))
 123                         non_common_revs++;
 124         }
 125 }
 126
 127 static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 128 {
 129         struct object *o = deref_tag(parse_object(oid), refname, 0);
 130
 131         if (o && o->type == OBJ_COMMIT)
 132                 rev_list_push((struct commit *)o, SEEN);
 133
 134         return 0;
 135 }
 136
 137 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 138                                    int flag, void *cb_data)
 139 {
 140         return rev_list_insert_ref(refname, oid);
 141 }
 142
 143 static int clear_marks(const char *refname, const struct object_id *oid,
 144                        int flag, void *cb_data)
 145 {
 146         struct object *o = deref_tag(parse_object(oid), refname, 0);
 147
 148         if (o && o->type == OBJ_COMMIT)
 149                 clear_commit_marks((struct commit *)o,
 150                                    COMMON | COMMON_REF | SEEN | POPPED);
 151         return 0;
 152 }
 153
 154 /*
 155    This function marks a rev and its ancestors as common.
 156    In some cases, it is desirable to mark only the ancestors (for example
 157    when only the server does not yet know that they are common).
 158 */
 159
 160 static void mark_common(struct commit *commit,
 161                 int ancestors_only, int dont_parse)
 162 {
 163         if (commit != NULL && !(commit->object.flags & COMMON)) {
 164                 struct object *o = (struct object *)commit;
 165
 166                 if (!ancestors_only)
 167                         o->flags |= COMMON;
 168
 169                 if (!(o->flags & SEEN))
 170                         rev_list_push(commit, SEEN);
 171                 else {
 172                         struct commit_list *parents;
 173
 174                         if (!ancestors_only && !(o->flags & POPPED))
 175                                 non_common_revs--;
 176                         if (!o->parsed && !dont_parse)
 177                                 if (parse_commit(commit))
 178                                         return;
 179
 180                         for (parents = commit->parents;
 181                                         parents;
 182                                         parents = parents->next)
 183                                 mark_common(parents->item, 0, dont_parse);
 184                 }
 185         }
 186 }
 187
 188 /*
 189   Get the next rev to send, ignoring the common.
 190 */
 191
 192 static const struct object_id *get_rev(void)
 193 {
 194         struct commit *commit = NULL;
 195
 196         while (commit == NULL) {
 197                 unsigned int mark;
 198                 struct commit_list *parents;
 199
 200                 if (rev_list.nr == 0 || non_common_revs == 0)
 201                         return NULL;
 202
 203                 commit = prio_queue_get(&rev_list);
 204                 parse_commit(commit);
 205                 parents = commit->parents;
 206
 207                 commit->object.flags |= POPPED;
 208                 if (!(commit->object.flags & COMMON))
 209                         non_common_revs--;
 210
 211                 if (commit->object.flags & COMMON) {
 212                         /* do not send "have", and ignore ancestors */
 213                         commit = NULL;
 214                         mark = COMMON | SEEN;
 215                 } else if (commit->object.flags & COMMON_REF)
 216                         /* send "have", and ignore ancestors */
 217                         mark = COMMON | SEEN;
 218                 else
 219                         /* send "have", also for its ancestors */
 220                         mark = SEEN;
 221
 222                 while (parents) {
 223                         if (!(parents->item->object.flags & SEEN))
 224                                 rev_list_push(parents->item, mark);
 225                         if (mark & COMMON)
 226                                 mark_common(parents->item, 1, 0);
 227                         parents = parents->next;
 228                 }
 229         }
 230
 231         return &commit->object.oid;
 232 }
 233
 234 enum ack_type {
 235         NAK = 0,
 236         ACK,
 237         ACK_continue,
 238         ACK_common,
 239         ACK_ready
 240 };
 241
 242 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 243 {
 244         if (args->stateless_rpc && args->deepen) {
 245                 /* If we sent a depth we will get back "duplicate"
 246                  * shallow and unshallow commands every time there
 247                  * is a block of have lines exchanged.
 248                  */
 249                 char *line;
 250                 while ((line = packet_read_line(fd, NULL))) {
 251                         if (starts_with(line, "shallow "))
 252                                 continue;
 253                         if (starts_with(line, "unshallow "))
 254                                 continue;
 255                         die(_("git fetch-pack: expected shallow list"));
 256                 }
 257         }
 258 }
 259
 260 static enum ack_type get_ack(int fd, struct object_id *result_oid)
 261 {
 262         int len;
 263         char *line = packet_read_line(fd, &len);
 264         const char *arg;
 265
 266         if (!line)
 267                 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
 268         if (!strcmp(line, "NAK"))
 269                 return NAK;
 270         if (skip_prefix(line, "ACK ", &arg)) {
 271                 if (!get_oid_hex(arg, result_oid)) {
 272                         arg += 40;
 273                         len -= arg - line;
 274                         if (len < 1)
 275                                 return ACK;
 276                         if (strstr(arg, "continue"))
 277                                 return ACK_continue;
 278                         if (strstr(arg, "common"))
 279                                 return ACK_common;
 280                         if (strstr(arg, "ready"))
 281                                 return ACK_ready;
 282                         return ACK;
 283                 }
 284         }
 285         if (skip_prefix(line, "ERR ", &arg))
 286                 die(_("remote error: %s"), arg);
 287         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 288 }
 289
 290 static void send_request(struct fetch_pack_args *args,
 291                          int fd, struct strbuf *buf)
 292 {
 293         if (args->stateless_rpc) {
 294                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 295                 packet_flush(fd);
 296         } else
 297                 write_or_die(fd, buf->buf, buf->len);
 298 }
 299
 300 static void insert_one_alternate_object(struct object *obj)
 301 {
 302         rev_list_insert_ref(NULL, &obj->oid);
 303 }
 304
 305 #define INITIAL_FLUSH 16
 306 #define PIPESAFE_FLUSH 32
 307 #define LARGE_FLUSH 16384
 308
 309 static int next_flush(int stateless_rpc, int count)
 310 {
 311         if (stateless_rpc) {
 312                 if (count < LARGE_FLUSH)
 313                         count <<= 1;
 314                 else
 315                         count = count * 11 / 10;
 316         } else {
 317                 if (count < PIPESAFE_FLUSH)
 318                         count <<= 1;
 319                 else
 320                         count += PIPESAFE_FLUSH;
 321         }
 322         return count;
 323 }
 324
 325 static int find_common(struct fetch_pack_args *args,
 326                        int fd[2], struct object_id *result_oid,
 327                        struct ref *refs)
 328 {
 329         int fetching;
 330         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 331         const struct object_id *oid;
 332         unsigned in_vain = 0;
 333         int got_continue = 0;
 334         int got_ready = 0;
 335         struct strbuf req_buf = STRBUF_INIT;
 336         size_t state_len = 0;
 337
 338         if (args->stateless_rpc && multi_ack == 1)
 339                 die(_("--stateless-rpc requires multi_ack_detailed"));
 340         if (marked)
 341                 for_each_ref(clear_marks, NULL);
 342         marked = 1;
 343
 344         for_each_ref(rev_list_insert_ref_oid, NULL);
 345         for_each_cached_alternate(insert_one_alternate_object);
 346
 347         fetching = 0;
 348         for ( ; refs ; refs = refs->next) {
 349                 struct object_id *remote = &refs->old_oid;
 350                 const char *remote_hex;
 351                 struct object *o;
 352
 353                 /*
 354                  * If that object is complete (i.e. it is an ancestor of a
 355                  * local ref), we tell them we have it but do not have to
 356                  * tell them about its ancestors, which they already know
 357                  * about.
 358                  *
 359                  * We use lookup_object here because we are only
 360                  * interested in the case we *know* the object is
 361                  * reachable and we have already scanned it.
 362                  */
 363                 if (((o = lookup_object(remote->hash)) != NULL) &&
 364                                 (o->flags & COMPLETE)) {
 365                         continue;
 366                 }
 367
 368                 remote_hex = oid_to_hex(remote);
 369                 if (!fetching) {
 370                         struct strbuf c = STRBUF_INIT;
 371                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 372                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 373                         if (no_done)            strbuf_addstr(&c, " no-done");
 374                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 375                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 376                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 377                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 378                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 379                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 380                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 381                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 382                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 383                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 384                                                             git_user_agent_sanitized());
 385                         if (args->filter_options.choice)
 386                                 strbuf_addstr(&c, " filter");
 387                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 388                         strbuf_release(&c);
 389                 } else
 390                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 391                 fetching++;
 392         }
 393
 394         if (!fetching) {
 395                 strbuf_release(&req_buf);
 396                 packet_flush(fd[1]);
 397                 return 1;
 398         }
 399
 400         if (is_repository_shallow(the_repository))
 401                 write_shallow_commits(&req_buf, 1, NULL);
 402         if (args->depth > 0)
 403                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 404         if (args->deepen_since) {
 405                 timestamp_t max_age = approxidate(args->deepen_since);
 406                 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 407         }
 408         if (args->deepen_not) {
 409                 int i;
 410                 for (i = 0; i < args->deepen_not->nr; i++) {
 411                         struct string_list_item *s = args->deepen_not->items + i;
 412                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 413                 }
 414         }
 415         if (server_supports_filtering && args->filter_options.choice)
 416                 packet_buf_write(&req_buf, "filter %s",
 417                                  args->filter_options.filter_spec);
 418         packet_buf_flush(&req_buf);
 419         state_len = req_buf.len;
 420
 421         if (args->deepen) {
 422                 char *line;
 423                 const char *arg;
 424                 struct object_id oid;
 425
 426                 send_request(args, fd[1], &req_buf);
 427                 while ((line = packet_read_line(fd[0], NULL))) {
 428                         if (skip_prefix(line, "shallow ", &arg)) {
 429                                 if (get_oid_hex(arg, &oid))
 430                                         die(_("invalid shallow line: %s"), line);
 431                                 register_shallow(the_repository, &oid);
 432                                 continue;
 433                         }
 434                         if (skip_prefix(line, "unshallow ", &arg)) {
 435                                 if (get_oid_hex(arg, &oid))
 436                                         die(_("invalid unshallow line: %s"), line);
 437                                 if (!lookup_object(oid.hash))
 438                                         die(_("object not found: %s"), line);
 439                                 /* make sure that it is parsed as shallow */
 440                                 if (!parse_object(&oid))
 441                                         die(_("error in object: %s"), line);
 442                                 if (unregister_shallow(&oid))
 443                                         die(_("no shallow found: %s"), line);
 444                                 continue;
 445                         }
 446                         die(_("expected shallow/unshallow, got %s"), line);
 447                 }
 448         } else if (!args->stateless_rpc)
 449                 send_request(args, fd[1], &req_buf);
 450
 451         if (!args->stateless_rpc) {
 452                 /* If we aren't using the stateless-rpc interface
 453                  * we don't need to retain the headers.
 454                  */
 455                 strbuf_setlen(&req_buf, 0);
 456                 state_len = 0;
 457         }
 458
 459         flushes = 0;
 460         retval = -1;
 461         if (args->no_dependents)
 462                 goto done;
 463         while ((oid = get_rev())) {
 464                 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 465                 print_verbose(args, "have %s", oid_to_hex(oid));
 466                 in_vain++;
 467                 if (flush_at <= ++count) {
 468                         int ack;
 469
 470                         packet_buf_flush(&req_buf);
 471                         send_request(args, fd[1], &req_buf);
 472                         strbuf_setlen(&req_buf, state_len);
 473                         flushes++;
 474                         flush_at = next_flush(args->stateless_rpc, count);
 475
 476                         /*
 477                          * We keep one window "ahead" of the other side, and
 478                          * will wait for an ACK only on the next one
 479                          */
 480                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 481                                 continue;
 482
 483                         consume_shallow_list(args, fd[0]);
 484                         do {
 485                                 ack = get_ack(fd[0], result_oid);
 486                                 if (ack)
 487                                         print_verbose(args, _("got %s %d %s"), "ack",
 488                                                       ack, oid_to_hex(result_oid));
 489                                 switch (ack) {
 490                                 case ACK:
 491                                         flushes = 0;
 492                                         multi_ack = 0;
 493                                         retval = 0;
 494                                         goto done;
 495                                 case ACK_common:
 496                                 case ACK_ready:
 497                                 case ACK_continue: {
 498                                         struct commit *commit =
 499                                                 lookup_commit(result_oid);
 500                                         if (!commit)
 501                                                 die(_("invalid commit %s"), oid_to_hex(result_oid));
 502                                         if (args->stateless_rpc
 503                                          && ack == ACK_common
 504                                          && !(commit->object.flags & COMMON)) {
 505                                                 /* We need to replay the have for this object
 506                                                  * on the next RPC request so the peer knows
 507                                                  * it is in common with us.
 508                                                  */
 509                                                 const char *hex = oid_to_hex(result_oid);
 510                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 511                                                 state_len = req_buf.len;
 512                                                 /*
 513                                                  * Reset in_vain because an ack
 514                                                  * for this commit has not been
 515                                                  * seen.
 516                                                  */
 517                                                 in_vain = 0;
 518                                         } else if (!args->stateless_rpc
 519                                                    || ack != ACK_common)
 520                                                 in_vain = 0;
 521                                         mark_common(commit, 0, 1);
 522                                         retval = 0;
 523                                         got_continue = 1;
 524                                         if (ack == ACK_ready) {
 525                                                 clear_prio_queue(&rev_list);
 526                                                 got_ready = 1;
 527                                         }
 528                                         break;
 529                                         }
 530                                 }
 531                         } while (ack);
 532                         flushes--;
 533                         if (got_continue && MAX_IN_VAIN < in_vain) {
 534                                 print_verbose(args, _("giving up"));
 535                                 break; /* give up */
 536                         }
 537                 }
 538         }
 539 done:
 540         if (!got_ready || !no_done) {
 541                 packet_buf_write(&req_buf, "done\n");
 542                 send_request(args, fd[1], &req_buf);
 543         }
 544         print_verbose(args, _("done"));
 545         if (retval != 0) {
 546                 multi_ack = 0;
 547                 flushes++;
 548         }
 549         strbuf_release(&req_buf);
 550
 551         if (!got_ready || !no_done)
 552                 consume_shallow_list(args, fd[0]);
 553         while (flushes || multi_ack) {
 554                 int ack = get_ack(fd[0], result_oid);
 555                 if (ack) {
 556                         print_verbose(args, _("got %s (%d) %s"), "ack",
 557                                       ack, oid_to_hex(result_oid));
 558                         if (ack == ACK)
 559                                 return 0;
 560                         multi_ack = 1;
 561                         continue;
 562                 }
 563                 flushes--;
 564         }
 565         /* it is no error to fetch into a completely empty repo */
 566         return count ? retval : 0;
 567 }
 568
 569 static struct commit_list *complete;
 570
 571 static int mark_complete(const struct object_id *oid)
 572 {
 573         struct object *o = parse_object(oid);
 574
 575         while (o && o->type == OBJ_TAG) {
 576                 struct tag *t = (struct tag *) o;
 577                 if (!t->tagged)
 578                         break; /* broken repository */
 579                 o->flags |= COMPLETE;
 580                 o = parse_object(&t->tagged->oid);
 581         }
 582         if (o && o->type == OBJ_COMMIT) {
 583                 struct commit *commit = (struct commit *)o;
 584                 if (!(commit->object.flags & COMPLETE)) {
 585                         commit->object.flags |= COMPLETE;
 586                         commit_list_insert(commit, &complete);
 587                 }
 588         }
 589         return 0;
 590 }
 591
 592 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 593                              int flag, void *cb_data)
 594 {
 595         return mark_complete(oid);
 596 }
 597
 598 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 599                                          timestamp_t cutoff)
 600 {
 601         while (complete && cutoff <= complete->item->date) {
 602                 print_verbose(args, _("Marking %s as complete"),
 603                               oid_to_hex(&complete->item->object.oid));
 604                 pop_most_recent_commit(&complete, COMPLETE);
 605         }
 606 }
 607
 608 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 609 {
 610         for (; refs; refs = refs->next)
 611                 oidset_insert(oids, &refs->old_oid);
 612 }
 613
 614 static int tip_oids_contain(struct oidset *tip_oids,
 615                             struct ref *unmatched, struct ref *newlist,
 616                             const struct object_id *id)
 617 {
 618         /*
 619          * Note that this only looks at the ref lists the first time it's
 620          * called. This works out in filter_refs() because even though it may
 621          * add to "newlist" between calls, the additions will always be for
 622          * oids that are already in the set.
 623          */
 624         if (!tip_oids->map.map.tablesize) {
 625                 add_refs_to_oidset(tip_oids, unmatched);
 626                 add_refs_to_oidset(tip_oids, newlist);
 627         }
 628         return oidset_contains(tip_oids, id);
 629 }
 630
 631 static void filter_refs(struct fetch_pack_args *args,
 632                         struct ref **refs,
 633                         struct ref **sought, int nr_sought)
 634 {
 635         struct ref *newlist = NULL;
 636         struct ref **newtail = &newlist;
 637         struct ref *unmatched = NULL;
 638         struct ref *ref, *next;
 639         struct oidset tip_oids = OIDSET_INIT;
 640         int i;
 641
 642         i = 0;
 643         for (ref = *refs; ref; ref = next) {
 644                 int keep = 0;
 645                 next = ref->next;
 646
 647                 if (starts_with(ref->name, "refs/") &&
 648                     check_refname_format(ref->name, 0))
 649                         ; /* trash */
 650                 else {
 651                         while (i < nr_sought) {
 652                                 int cmp = strcmp(ref->name, sought[i]->name);
 653                                 if (cmp < 0)
 654                                         break; /* definitely do not have it */
 655                                 else if (cmp == 0) {
 656                                         keep = 1; /* definitely have it */
 657                                         sought[i]->match_status = REF_MATCHED;
 658                                 }
 659                                 i++;
 660                         }
 661
 662                         if (!keep && args->fetch_all &&
 663                             (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 664                                 keep = 1;
 665                 }
 666
 667                 if (keep) {
 668                         *newtail = ref;
 669                         ref->next = NULL;
 670                         newtail = &ref->next;
 671                 } else {
 672                         ref->next = unmatched;
 673                         unmatched = ref;
 674                 }
 675         }
 676
 677         /* Append unmatched requests to the list */
 678         for (i = 0; i < nr_sought; i++) {
 679                 struct object_id oid;
 680                 const char *p;
 681
 682                 ref = sought[i];
 683                 if (ref->match_status != REF_NOT_MATCHED)
 684                         continue;
 685                 if (parse_oid_hex(ref->name, &oid, &p) ||
 686                     *p != '\0' ||
 687                     oidcmp(&oid, &ref->old_oid))
 688                         continue;
 689
 690                 if ((allow_unadvertised_object_request &
 691                      (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 692                     tip_oids_contain(&tip_oids, unmatched, newlist,
 693                                      &ref->old_oid)) {
 694                         ref->match_status = REF_MATCHED;
 695                         *newtail = copy_ref(ref);
 696                         newtail = &(*newtail)->next;
 697                 } else {
 698                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 699                 }
 700         }
 701
 702         oidset_clear(&tip_oids);
 703         for (ref = unmatched; ref; ref = next) {
 704                 next = ref->next;
 705                 free(ref);
 706         }
 707
 708         *refs = newlist;
 709 }
 710
 711 static void mark_alternate_complete(struct object *obj)
 712 {
 713         mark_complete(&obj->oid);
 714 }
 715
 716 struct loose_object_iter {
 717         struct oidset *loose_object_set;
 718         struct ref *refs;
 719 };
 720
 721 /*
 722  *  If the number of refs is not larger than the number of loose objects,
 723  *  this function stops inserting.
 724  */
 725 static int add_loose_objects_to_set(const struct object_id *oid,
 726                                     const char *path,
 727                                     void *data)
 728 {
 729         struct loose_object_iter *iter = data;
 730         oidset_insert(iter->loose_object_set, oid);
 731         if (iter->refs == NULL)
 732                 return 1;
 733
 734         iter->refs = iter->refs->next;
 735         return 0;
 736 }
 737
 738 static int everything_local(struct fetch_pack_args *args,
 739                             struct ref **refs,
 740                             struct ref **sought, int nr_sought)
 741 {
 742         struct ref *ref;
 743         int retval;
 744         int old_save_commit_buffer = save_commit_buffer;
 745         timestamp_t cutoff = 0;
 746         struct oidset loose_oid_set = OIDSET_INIT;
 747         int use_oidset = 0;
 748         struct loose_object_iter iter = {&loose_oid_set, *refs};
 749
 750         /* Enumerate all loose objects or know refs are not so many. */
 751         use_oidset = !for_each_loose_object(add_loose_objects_to_set,
 752                                             &iter, 0);
 753
 754         save_commit_buffer = 0;
 755
 756         for (ref = *refs; ref; ref = ref->next) {
 757                 struct object *o;
 758                 unsigned int flags = OBJECT_INFO_QUICK;
 759
 760                 if (use_oidset &&
 761                     !oidset_contains(&loose_oid_set, &ref->old_oid)) {
 762                         /*
 763                          * I know this does not exist in the loose form,
 764                          * so check if it exists in a non-loose form.
 765                          */
 766                         flags |= OBJECT_INFO_IGNORE_LOOSE;
 767                 }
 768
 769                 if (!has_object_file_with_flags(&ref->old_oid, flags))
 770                         continue;
 771                 o = parse_object(&ref->old_oid);
 772                 if (!o)
 773                         continue;
 774
 775                 /* We already have it -- which may mean that we were
 776                  * in sync with the other side at some time after
 777                  * that (it is OK if we guess wrong here).
 778                  */
 779                 if (o->type == OBJ_COMMIT) {
 780                         struct commit *commit = (struct commit *)o;
 781                         if (!cutoff || cutoff < commit->date)
 782                                 cutoff = commit->date;
 783                 }
 784         }
 785
 786         oidset_clear(&loose_oid_set);
 787
 788         if (!args->no_dependents) {
 789                 if (!args->deepen) {
 790                         for_each_ref(mark_complete_oid, NULL);
 791                         for_each_cached_alternate(mark_alternate_complete);
 792                         commit_list_sort_by_date(&complete);
 793                         if (cutoff)
 794                                 mark_recent_complete_commits(args, cutoff);
 795                 }
 796
 797                 /*
 798                  * Mark all complete remote refs as common refs.
 799                  * Don't mark them common yet; the server has to be told so first.
 800                  */
 801                 for (ref = *refs; ref; ref = ref->next) {
 802                         struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 803                                                      NULL, 0);
 804
 805                         if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 806                                 continue;
 807
 808                         if (!(o->flags & SEEN)) {
 809                                 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 810
 811                                 mark_common((struct commit *)o, 1, 1);
 812                         }
 813                 }
 814         }
 815
 816         filter_refs(args, refs, sought, nr_sought);
 817
 818         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 819                 const struct object_id *remote = &ref->old_oid;
 820                 struct object *o;
 821
 822                 o = lookup_object(remote->hash);
 823                 if (!o || !(o->flags & COMPLETE)) {
 824                         retval = 0;
 825                         print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 826                                       ref->name);
 827                         continue;
 828                 }
 829                 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 830                               ref->name);
 831         }
 832
 833         save_commit_buffer = old_save_commit_buffer;
 834
 835         return retval;
 836 }
 837
 838 static int sideband_demux(int in, int out, void *data)
 839 {
 840         int *xd = data;
 841         int ret;
 842
 843         ret = recv_sideband("fetch-pack", xd[0], out);
 844         close(out);
 845         return ret;
 846 }
 847
 848 static int get_pack(struct fetch_pack_args *args,
 849                     int xd[2], char **pack_lockfile)
 850 {
 851         struct async demux;
 852         int do_keep = args->keep_pack;
 853         const char *cmd_name;
 854         struct pack_header header;
 855         int pass_header = 0;
 856         struct child_process cmd = CHILD_PROCESS_INIT;
 857         int ret;
 858
 859         memset(&demux, 0, sizeof(demux));
 860         if (use_sideband) {
 861                 /* xd[] is talking with upload-pack; subprocess reads from
 862                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 863                  * through demux->out.
 864                  */
 865                 demux.proc = sideband_demux;
 866                 demux.data = xd;
 867                 demux.out = -1;
 868                 demux.isolate_sigpipe = 1;
 869                 if (start_async(&demux))
 870                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 871         }
 872         else
 873                 demux.out = xd[0];
 874
 875         if (!args->keep_pack && unpack_limit) {
 876
 877                 if (read_pack_header(demux.out, &header))
 878                         die(_("protocol error: bad pack header"));
 879                 pass_header = 1;
 880                 if (ntohl(header.hdr_entries) < unpack_limit)
 881                         do_keep = 0;
 882                 else
 883                         do_keep = 1;
 884         }
 885
 886         if (alternate_shallow_file) {
 887                 argv_array_push(&cmd.args, "--shallow-file");
 888                 argv_array_push(&cmd.args, alternate_shallow_file);
 889         }
 890
 891         if (do_keep || args->from_promisor) {
 892                 if (pack_lockfile)
 893                         cmd.out = -1;
 894                 cmd_name = "index-pack";
 895                 argv_array_push(&cmd.args, cmd_name);
 896                 argv_array_push(&cmd.args, "--stdin");
 897                 if (!args->quiet && !args->no_progress)
 898                         argv_array_push(&cmd.args, "-v");
 899                 if (args->use_thin_pack)
 900                         argv_array_push(&cmd.args, "--fix-thin");
 901                 if (do_keep && (args->lock_pack || unpack_limit)) {
 902                         char hostname[HOST_NAME_MAX + 1];
 903                         if (xgethostname(hostname, sizeof(hostname)))
 904                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 905                         argv_array_pushf(&cmd.args,
 906                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 907                                         (uintmax_t)getpid(), hostname);
 908                 }
 909                 if (args->check_self_contained_and_connected)
 910                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 911                 if (args->from_promisor)
 912                         argv_array_push(&cmd.args, "--promisor");
 913         }
 914         else {
 915                 cmd_name = "unpack-objects";
 916                 argv_array_push(&cmd.args, cmd_name);
 917                 if (args->quiet || args->no_progress)
 918                         argv_array_push(&cmd.args, "-q");
 919                 args->check_self_contained_and_connected = 0;
 920         }
 921
 922         if (pass_header)
 923                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 924                                  ntohl(header.hdr_version),
 925                                  ntohl(header.hdr_entries));
 926         if (fetch_fsck_objects >= 0
 927             ? fetch_fsck_objects
 928             : transfer_fsck_objects >= 0
 929             ? transfer_fsck_objects
 930             : 0) {
 931                 if (args->from_promisor)
 932                         /*
 933                          * We cannot use --strict in index-pack because it
 934                          * checks both broken objects and links, but we only
 935                          * want to check for broken objects.
 936                          */
 937                         argv_array_push(&cmd.args, "--fsck-objects");
 938                 else
 939                         argv_array_push(&cmd.args, "--strict");
 940         }
 941
 942         cmd.in = demux.out;
 943         cmd.git_cmd = 1;
 944         if (start_command(&cmd))
 945                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 946         if (do_keep && pack_lockfile) {
 947                 *pack_lockfile = index_pack_lockfile(cmd.out);
 948                 close(cmd.out);
 949         }
 950
 951         if (!use_sideband)
 952                 /* Closed by start_command() */
 953                 xd[0] = -1;
 954
 955         ret = finish_command(&cmd);
 956         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 957                 args->self_contained_and_connected =
 958                         args->check_self_contained_and_connected &&
 959                         ret == 0;
 960         else
 961                 die(_("%s failed"), cmd_name);
 962         if (use_sideband && finish_async(&demux))
 963                 die(_("error in sideband demultiplexer"));
 964         return 0;
 965 }
 966
 967 static int cmp_ref_by_name(const void *a_, const void *b_)
 968 {
 969         const struct ref *a = *((const struct ref **)a_);
 970         const struct ref *b = *((const struct ref **)b_);
 971         return strcmp(a->name, b->name);
 972 }
 973
 974 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 975                                  int fd[2],
 976                                  const struct ref *orig_ref,
 977                                  struct ref **sought, int nr_sought,
 978                                  struct shallow_info *si,
 979                                  char **pack_lockfile)
 980 {
 981         struct ref *ref = copy_ref_list(orig_ref);
 982         struct object_id oid;
 983         const char *agent_feature;
 984         int agent_len;
 985
 986         sort_ref_list(&ref, ref_compare_name);
 987         QSORT(sought, nr_sought, cmp_ref_by_name);
 988
 989         if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
 990                 die(_("Server does not support shallow clients"));
 991         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 992                 args->deepen = 1;
 993         if (server_supports("multi_ack_detailed")) {
 994                 print_verbose(args, _("Server supports multi_ack_detailed"));
 995                 multi_ack = 2;
 996                 if (server_supports("no-done")) {
 997                         print_verbose(args, _("Server supports no-done"));
 998                         if (args->stateless_rpc)
 999                                 no_done = 1;
1000                 }
1001         }
1002         else if (server_supports("multi_ack")) {
1003                 print_verbose(args, _("Server supports multi_ack"));
1004                 multi_ack = 1;
1005         }
1006         if (server_supports("side-band-64k")) {
1007                 print_verbose(args, _("Server supports side-band-64k"));
1008                 use_sideband = 2;
1009         }
1010         else if (server_supports("side-band")) {
1011                 print_verbose(args, _("Server supports side-band"));
1012                 use_sideband = 1;
1013         }
1014         if (server_supports("allow-tip-sha1-in-want")) {
1015                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
1016                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1017         }
1018         if (server_supports("allow-reachable-sha1-in-want")) {
1019                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
1020                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1021         }
1022         if (!server_supports("thin-pack"))
1023                 args->use_thin_pack = 0;
1024         if (!server_supports("no-progress"))
1025                 args->no_progress = 0;
1026         if (!server_supports("include-tag"))
1027                 args->include_tag = 0;
1028         if (server_supports("ofs-delta"))
1029                 print_verbose(args, _("Server supports ofs-delta"));
1030         else
1031                 prefer_ofs_delta = 0;
1032
1033         if (server_supports("filter")) {
1034                 server_supports_filtering = 1;
1035                 print_verbose(args, _("Server supports filter"));
1036         } else if (args->filter_options.choice) {
1037                 warning("filtering not recognized by server, ignoring");
1038         }
1039
1040         if ((agent_feature = server_feature_value("agent", &agent_len))) {
1041                 agent_supported = 1;
1042                 if (agent_len)
1043                         print_verbose(args, _("Server version is %.*s"),
1044                                       agent_len, agent_feature);
1045         }
1046         if (server_supports("deepen-since"))
1047                 deepen_since_ok = 1;
1048         else if (args->deepen_since)
1049                 die(_("Server does not support --shallow-since"));
1050         if (server_supports("deepen-not"))
1051                 deepen_not_ok = 1;
1052         else if (args->deepen_not)
1053                 die(_("Server does not support --shallow-exclude"));
1054         if (!server_supports("deepen-relative") && args->deepen_relative)
1055                 die(_("Server does not support --deepen"));
1056
1057         if (everything_local(args, &ref, sought, nr_sought)) {
1058                 packet_flush(fd[1]);
1059                 goto all_done;
1060         }
1061         if (find_common(args, fd, &oid, ref) < 0)
1062                 if (!args->keep_pack)
1063                         /* When cloning, it is not unusual to have
1064                          * no common commit.
1065                          */
1066                         warning(_("no common commits"));
1067
1068         if (args->stateless_rpc)
1069                 packet_flush(fd[1]);
1070         if (args->deepen)
1071                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1072                                         NULL);
1073         else if (si->nr_ours || si->nr_theirs)
1074                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1075         else
1076                 alternate_shallow_file = NULL;
1077         if (get_pack(args, fd, pack_lockfile))
1078                 die(_("git fetch-pack: fetch failed."));
1079
1080  all_done:
1081         return ref;
1082 }
1083
1084 static void add_shallow_requests(struct strbuf *req_buf,
1085                                  const struct fetch_pack_args *args)
1086 {
1087         if (is_repository_shallow(the_repository))
1088                 write_shallow_commits(req_buf, 1, NULL);
1089         if (args->depth > 0)
1090                 packet_buf_write(req_buf, "deepen %d", args->depth);
1091         if (args->deepen_since) {
1092                 timestamp_t max_age = approxidate(args->deepen_since);
1093                 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1094         }
1095         if (args->deepen_not) {
1096                 int i;
1097                 for (i = 0; i < args->deepen_not->nr; i++) {
1098                         struct string_list_item *s = args->deepen_not->items + i;
1099                         packet_buf_write(req_buf, "deepen-not %s", s->string);
1100                 }
1101         }
1102 }
1103
1104 static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1105 {
1106         for ( ; wants ; wants = wants->next) {
1107                 const struct object_id *remote = &wants->old_oid;
1108                 const char *remote_hex;
1109                 struct object *o;
1110
1111                 /*
1112                  * If that object is complete (i.e. it is an ancestor of a
1113                  * local ref), we tell them we have it but do not have to
1114                  * tell them about its ancestors, which they already know
1115                  * about.
1116                  *
1117                  * We use lookup_object here because we are only
1118                  * interested in the case we *know* the object is
1119                  * reachable and we have already scanned it.
1120                  */
1121                 if (((o = lookup_object(remote->hash)) != NULL) &&
1122                     (o->flags & COMPLETE)) {
1123                         continue;
1124                 }
1125
1126                 remote_hex = oid_to_hex(remote);
1127                 packet_buf_write(req_buf, "want %s\n", remote_hex);
1128         }
1129 }
1130
1131 static void add_common(struct strbuf *req_buf, struct oidset *common)
1132 {
1133         struct oidset_iter iter;
1134         const struct object_id *oid;
1135         oidset_iter_init(common, &iter);
1136
1137         while ((oid = oidset_iter_next(&iter))) {
1138                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1139         }
1140 }
1141
1142 static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
1143 {
1144         int ret = 0;
1145         int haves_added = 0;
1146         const struct object_id *oid;
1147
1148         while ((oid = get_rev())) {
1149                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1150                 if (++haves_added >= *haves_to_send)
1151                         break;
1152         }
1153
1154         *in_vain += haves_added;
1155         if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1156                 /* Send Done */
1157                 packet_buf_write(req_buf, "done\n");
1158                 ret = 1;
1159         }
1160
1161         /* Increase haves to send on next round */
1162         *haves_to_send = next_flush(1, *haves_to_send);
1163
1164         return ret;
1165 }
1166
1167 static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
1168                               const struct ref *wants, struct oidset *common,
1169                               int *haves_to_send, int *in_vain)
1170 {
1171         int ret = 0;
1172         struct strbuf req_buf = STRBUF_INIT;
1173
1174         if (server_supports_v2("fetch", 1))
1175                 packet_buf_write(&req_buf, "command=fetch");
1176         if (server_supports_v2("agent", 0))
1177                 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1178         if (args->server_options && args->server_options->nr &&
1179             server_supports_v2("server-option", 1)) {
1180                 int i;
1181                 for (i = 0; i < args->server_options->nr; i++)
1182                         packet_write_fmt(fd_out, "server-option=%s",
1183                                          args->server_options->items[i].string);
1184         }
1185
1186         packet_buf_delim(&req_buf);
1187         if (args->use_thin_pack)
1188                 packet_buf_write(&req_buf, "thin-pack");
1189         if (args->no_progress)
1190                 packet_buf_write(&req_buf, "no-progress");
1191         if (args->include_tag)
1192                 packet_buf_write(&req_buf, "include-tag");
1193         if (prefer_ofs_delta)
1194                 packet_buf_write(&req_buf, "ofs-delta");
1195
1196         /* Add shallow-info and deepen request */
1197         if (server_supports_feature("fetch", "shallow", 0))
1198                 add_shallow_requests(&req_buf, args);
1199         else if (is_repository_shallow(the_repository) || args->deepen)
1200                 die(_("Server does not support shallow requests"));
1201
1202         /* Add filter */
1203         if (server_supports_feature("fetch", "filter", 0) &&
1204             args->filter_options.choice) {
1205                 print_verbose(args, _("Server supports filter"));
1206                 packet_buf_write(&req_buf, "filter %s",
1207                                  args->filter_options.filter_spec);
1208         } else if (args->filter_options.choice) {
1209                 warning("filtering not recognized by server, ignoring");
1210         }
1211
1212         /* add wants */
1213         add_wants(wants, &req_buf);
1214
1215         if (args->no_dependents) {
1216                 packet_buf_write(&req_buf, "done");
1217                 ret = 1;
1218         } else {
1219                 /* Add all of the common commits we've found in previous rounds */
1220                 add_common(&req_buf, common);
1221
1222                 /* Add initial haves */
1223                 ret = add_haves(&req_buf, haves_to_send, in_vain);
1224         }
1225
1226         /* Send request */
1227         packet_buf_flush(&req_buf);
1228         write_or_die(fd_out, req_buf.buf, req_buf.len);
1229
1230         strbuf_release(&req_buf);
1231         return ret;
1232 }
1233
1234 /*
1235  * Processes a section header in a server's response and checks if it matches
1236  * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1237  * not consumed); if 0, the line will be consumed and the function will die if
1238  * the section header doesn't match what was expected.
1239  */
1240 static int process_section_header(struct packet_reader *reader,
1241                                   const char *section, int peek)
1242 {
1243         int ret;
1244
1245         if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1246                 die("error reading section header '%s'", section);
1247
1248         ret = !strcmp(reader->line, section);
1249
1250         if (!peek) {
1251                 if (!ret)
1252                         die("expected '%s', received '%s'",
1253                             section, reader->line);
1254                 packet_reader_read(reader);
1255         }
1256
1257         return ret;
1258 }
1259
1260 static int process_acks(struct packet_reader *reader, struct oidset *common)
1261 {
1262         /* received */
1263         int received_ready = 0;
1264         int received_ack = 0;
1265
1266         process_section_header(reader, "acknowledgments", 0);
1267         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1268                 const char *arg;
1269
1270                 if (!strcmp(reader->line, "NAK"))
1271                         continue;
1272
1273                 if (skip_prefix(reader->line, "ACK ", &arg)) {
1274                         struct object_id oid;
1275                         if (!get_oid_hex(arg, &oid)) {
1276                                 struct commit *commit;
1277                                 oidset_insert(common, &oid);
1278                                 commit = lookup_commit(&oid);
1279                                 mark_common(commit, 0, 1);
1280                         }
1281                         continue;
1282                 }
1283
1284                 if (!strcmp(reader->line, "ready")) {
1285                         clear_prio_queue(&rev_list);
1286                         received_ready = 1;
1287                         continue;
1288                 }
1289
1290                 die("unexpected acknowledgment line: '%s'", reader->line);
1291         }
1292
1293         if (reader->status != PACKET_READ_FLUSH &&
1294             reader->status != PACKET_READ_DELIM)
1295                 die("error processing acks: %d", reader->status);
1296
1297         /* return 0 if no common, 1 if there are common, or 2 if ready */
1298         return received_ready ? 2 : (received_ack ? 1 : 0);
1299 }
1300
1301 static void receive_shallow_info(struct fetch_pack_args *args,
1302                                  struct packet_reader *reader)
1303 {
1304         process_section_header(reader, "shallow-info", 0);
1305         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1306                 const char *arg;
1307                 struct object_id oid;
1308
1309                 if (skip_prefix(reader->line, "shallow ", &arg)) {
1310                         if (get_oid_hex(arg, &oid))
1311                                 die(_("invalid shallow line: %s"), reader->line);
1312                         register_shallow(the_repository, &oid);
1313                         continue;
1314                 }
1315                 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1316                         if (get_oid_hex(arg, &oid))
1317                                 die(_("invalid unshallow line: %s"), reader->line);
1318                         if (!lookup_object(oid.hash))
1319                                 die(_("object not found: %s"), reader->line);
1320                         /* make sure that it is parsed as shallow */
1321                         if (!parse_object(&oid))
1322                                 die(_("error in object: %s"), reader->line);
1323                         if (unregister_shallow(&oid))
1324                                 die(_("no shallow found: %s"), reader->line);
1325                         continue;
1326                 }
1327                 die(_("expected shallow/unshallow, got %s"), reader->line);
1328         }
1329
1330         if (reader->status != PACKET_READ_FLUSH &&
1331             reader->status != PACKET_READ_DELIM)
1332                 die("error processing shallow info: %d", reader->status);
1333
1334         setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1335         args->deepen = 1;
1336 }
1337
1338 enum fetch_state {
1339         FETCH_CHECK_LOCAL = 0,
1340         FETCH_SEND_REQUEST,
1341         FETCH_PROCESS_ACKS,
1342         FETCH_GET_PACK,
1343         FETCH_DONE,
1344 };
1345
1346 static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1347                                     int fd[2],
1348                                     const struct ref *orig_ref,
1349                                     struct ref **sought, int nr_sought,
1350                                     char **pack_lockfile)
1351 {
1352         struct ref *ref = copy_ref_list(orig_ref);
1353         enum fetch_state state = FETCH_CHECK_LOCAL;
1354         struct oidset common = OIDSET_INIT;
1355         struct packet_reader reader;
1356         int in_vain = 0;
1357         int haves_to_send = INITIAL_FLUSH;
1358         packet_reader_init(&reader, fd[0], NULL, 0,
1359                            PACKET_READ_CHOMP_NEWLINE);
1360
1361         while (state != FETCH_DONE) {
1362                 switch (state) {
1363                 case FETCH_CHECK_LOCAL:
1364                         sort_ref_list(&ref, ref_compare_name);
1365                         QSORT(sought, nr_sought, cmp_ref_by_name);
1366
1367                         /* v2 supports these by default */
1368                         allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1369                         use_sideband = 2;
1370                         if (args->depth > 0 || args->deepen_since || args->deepen_not)
1371                                 args->deepen = 1;
1372
1373                         if (marked)
1374                                 for_each_ref(clear_marks, NULL);
1375                         marked = 1;
1376
1377                         for_each_ref(rev_list_insert_ref_oid, NULL);
1378                         for_each_cached_alternate(insert_one_alternate_object);
1379
1380                         /* Filter 'ref' by 'sought' and those that aren't local */
1381                         if (everything_local(args, &ref, sought, nr_sought))
1382                                 state = FETCH_DONE;
1383                         else
1384                                 state = FETCH_SEND_REQUEST;
1385                         break;
1386                 case FETCH_SEND_REQUEST:
1387                         if (send_fetch_request(fd[1], args, ref, &common,
1388                                                &haves_to_send, &in_vain))
1389                                 state = FETCH_GET_PACK;
1390                         else
1391                                 state = FETCH_PROCESS_ACKS;
1392                         break;
1393                 case FETCH_PROCESS_ACKS:
1394                         /* Process ACKs/NAKs */
1395                         switch (process_acks(&reader, &common)) {
1396                         case 2:
1397                                 state = FETCH_GET_PACK;
1398                                 break;
1399                         case 1:
1400                                 in_vain = 0;
1401                                 /* fallthrough */
1402                         default:
1403                                 state = FETCH_SEND_REQUEST;
1404                                 break;
1405                         }
1406                         break;
1407                 case FETCH_GET_PACK:
1408                         /* Check for shallow-info section */
1409                         if (process_section_header(&reader, "shallow-info", 1))
1410                                 receive_shallow_info(args, &reader);
1411
1412                         /* get the pack */
1413                         process_section_header(&reader, "packfile", 0);
1414                         if (get_pack(args, fd, pack_lockfile))
1415                                 die(_("git fetch-pack: fetch failed."));
1416
1417                         state = FETCH_DONE;
1418                         break;
1419                 case FETCH_DONE:
1420                         continue;
1421                 }
1422         }
1423
1424         oidset_clear(&common);
1425         return ref;
1426 }
1427
1428 static void fetch_pack_config(void)
1429 {
1430         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1431         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1432         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1433         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1434         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1435
1436         git_config(git_default_config, NULL);
1437 }
1438
1439 static void fetch_pack_setup(void)
1440 {
1441         static int did_setup;
1442         if (did_setup)
1443                 return;
1444         fetch_pack_config();
1445         if (0 <= transfer_unpack_limit)
1446                 unpack_limit = transfer_unpack_limit;
1447         else if (0 <= fetch_unpack_limit)
1448                 unpack_limit = fetch_unpack_limit;
1449         did_setup = 1;
1450 }
1451
1452 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1453 {
1454         struct string_list names = STRING_LIST_INIT_NODUP;
1455         int src, dst;
1456
1457         for (src = dst = 0; src < nr; src++) {
1458                 struct string_list_item *item;
1459                 item = string_list_insert(&names, ref[src]->name);
1460                 if (item->util)
1461                         continue; /* already have it */
1462                 item->util = ref[src];
1463                 if (src != dst)
1464                         ref[dst] = ref[src];
1465                 dst++;
1466         }
1467         for (src = dst; src < nr; src++)
1468                 ref[src] = NULL;
1469         string_list_clear(&names, 0);
1470         return dst;
1471 }
1472
1473 static void update_shallow(struct fetch_pack_args *args,
1474                            struct ref **sought, int nr_sought,
1475                            struct shallow_info *si)
1476 {
1477         struct oid_array ref = OID_ARRAY_INIT;
1478         int *status;
1479         int i;
1480
1481         if (args->deepen && alternate_shallow_file) {
1482                 if (*alternate_shallow_file == '\0') { /* --unshallow */
1483                         unlink_or_warn(git_path_shallow(the_repository));
1484                         rollback_lock_file(&shallow_lock);
1485                 } else
1486                         commit_lock_file(&shallow_lock);
1487                 return;
1488         }
1489
1490         if (!si->shallow || !si->shallow->nr)
1491                 return;
1492
1493         if (args->cloning) {
1494                 /*
1495                  * remote is shallow, but this is a clone, there are
1496                  * no objects in repo to worry about. Accept any
1497                  * shallow points that exist in the pack (iow in repo
1498                  * after get_pack() and reprepare_packed_git())
1499                  */
1500                 struct oid_array extra = OID_ARRAY_INIT;
1501                 struct object_id *oid = si->shallow->oid;
1502                 for (i = 0; i < si->shallow->nr; i++)
1503                         if (has_object_file(&oid[i]))
1504                                 oid_array_append(&extra, &oid[i]);
1505                 if (extra.nr) {
1506                         setup_alternate_shallow(&shallow_lock,
1507                                                 &alternate_shallow_file,
1508                                                 &extra);
1509                         commit_lock_file(&shallow_lock);
1510                 }
1511                 oid_array_clear(&extra);
1512                 return;
1513         }
1514
1515         if (!si->nr_ours && !si->nr_theirs)
1516                 return;
1517
1518         remove_nonexistent_theirs_shallow(si);
1519         if (!si->nr_ours && !si->nr_theirs)
1520                 return;
1521         for (i = 0; i < nr_sought; i++)
1522                 oid_array_append(&ref, &sought[i]->old_oid);
1523         si->ref = &ref;
1524
1525         if (args->update_shallow) {
1526                 /*
1527                  * remote is also shallow, .git/shallow may be updated
1528                  * so all refs can be accepted. Make sure we only add
1529                  * shallow roots that are actually reachable from new
1530                  * refs.
1531                  */
1532                 struct oid_array extra = OID_ARRAY_INIT;
1533                 struct object_id *oid = si->shallow->oid;
1534                 assign_shallow_commits_to_refs(si, NULL, NULL);
1535                 if (!si->nr_ours && !si->nr_theirs) {
1536                         oid_array_clear(&ref);
1537                         return;
1538                 }
1539                 for (i = 0; i < si->nr_ours; i++)
1540                         oid_array_append(&extra, &oid[si->ours[i]]);
1541                 for (i = 0; i < si->nr_theirs; i++)
1542                         oid_array_append(&extra, &oid[si->theirs[i]]);
1543                 setup_alternate_shallow(&shallow_lock,
1544                                         &alternate_shallow_file,
1545                                         &extra);
1546                 commit_lock_file(&shallow_lock);
1547                 oid_array_clear(&extra);
1548                 oid_array_clear(&ref);
1549                 return;
1550         }
1551
1552         /*
1553          * remote is also shallow, check what ref is safe to update
1554          * without updating .git/shallow
1555          */
1556         status = xcalloc(nr_sought, sizeof(*status));
1557         assign_shallow_commits_to_refs(si, NULL, status);
1558         if (si->nr_ours || si->nr_theirs) {
1559                 for (i = 0; i < nr_sought; i++)
1560                         if (status[i])
1561                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1562         }
1563         free(status);
1564         oid_array_clear(&ref);
1565 }
1566
1567 struct ref *fetch_pack(struct fetch_pack_args *args,
1568                        int fd[], struct child_process *conn,
1569                        const struct ref *ref,
1570                        const char *dest,
1571                        struct ref **sought, int nr_sought,
1572                        struct oid_array *shallow,
1573                        char **pack_lockfile,
1574                        enum protocol_version version)
1575 {
1576         struct ref *ref_cpy;
1577         struct shallow_info si;
1578
1579         fetch_pack_setup();
1580         if (nr_sought)
1581                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1582
1583         if (!ref) {
1584                 packet_flush(fd[1]);
1585                 die(_("no matching remote head"));
1586         }
1587         prepare_shallow_info(&si, shallow);
1588         if (version == protocol_v2)
1589                 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1590                                            pack_lockfile);
1591         else
1592                 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1593                                         &si, pack_lockfile);
1594         reprepare_packed_git(the_repository);
1595         update_shallow(args, sought, nr_sought, &si);
1596         clear_shallow_info(&si);
1597         return ref_cpy;
1598 }
1599
1600 int report_unmatched_refs(struct ref **sought, int nr_sought)
1601 {
1602         int i, ret = 0;
1603
1604         for (i = 0; i < nr_sought; i++) {
1605                 if (!sought[i])
1606                         continue;
1607                 switch (sought[i]->match_status) {
1608                 case REF_MATCHED:
1609                         continue;
1610                 case REF_NOT_MATCHED:
1611                         error(_("no such remote ref %s"), sought[i]->name);
1612                         break;
1613                 case REF_UNADVERTISED_NOT_ALLOWED:
1614                         error(_("Server does not allow request for unadvertised object %s"),
1615                               sought[i]->name);
1616                         break;
1617                 }
1618                 ret = 1;
1619         }
1620         return ret;
1621 }