git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "repository.h"
   3 #include "config.h"
   4 #include "lockfile.h"
   5 #include "refs.h"
   6 #include "pkt-line.h"
   7 #include "commit.h"
   8 #include "tag.h"
   9 #include "exec-cmd.h"
  10 #include "pack.h"
  11 #include "sideband.h"
  12 #include "fetch-pack.h"
  13 #include "remote.h"
  14 #include "run-command.h"
  15 #include "connect.h"
  16 #include "transport.h"
  17 #include "version.h"
  18 #include "prio-queue.h"
  19 #include "sha1-array.h"
  20 #include "oidset.h"
  21 #include "packfile.h"
  22
  23 static int transfer_unpack_limit = -1;
  24 static int fetch_unpack_limit = -1;
  25 static int unpack_limit = 100;
  26 static int prefer_ofs_delta = 1;
  27 static int no_done;
  28 static int deepen_since_ok;
  29 static int deepen_not_ok;
  30 static int fetch_fsck_objects = -1;
  31 static int transfer_fsck_objects = -1;
  32 static int agent_supported;
  33 static int server_supports_filtering;
  34 static struct lock_file shallow_lock;
  35 static const char *alternate_shallow_file;
  36
  37 /* Remember to update object flag allocation in object.h */
  38 #define COMPLETE        (1U << 0)
  39 #define COMMON          (1U << 1)
  40 #define COMMON_REF      (1U << 2)
  41 #define SEEN            (1U << 3)
  42 #define POPPED          (1U << 4)
  43 #define ALTERNATE       (1U << 5)
  44
  45 static int marked;
  46
  47 /*
  48  * After sending this many "have"s if we do not get any new ACK , we
  49  * give up traversing our history.
  50  */
  51 #define MAX_IN_VAIN 256
  52
  53 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  54 static int non_common_revs, multi_ack, use_sideband;
  55 /* Allow specifying sha1 if it is a ref tip. */
  56 #define ALLOW_TIP_SHA1  01
  57 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  58 #define ALLOW_REACHABLE_SHA1    02
  59 static unsigned int allow_unadvertised_object_request;
  60
  61 __attribute__((format (printf, 2, 3)))
  62 static inline void print_verbose(const struct fetch_pack_args *args,
  63                                  const char *fmt, ...)
  64 {
  65         va_list params;
  66
  67         if (!args->verbose)
  68                 return;
  69
  70         va_start(params, fmt);
  71         vfprintf(stderr, fmt, params);
  72         va_end(params);
  73         fputc('\n', stderr);
  74 }
  75
  76 struct alternate_object_cache {
  77         struct object **items;
  78         size_t nr, alloc;
  79 };
  80
  81 static void cache_one_alternate(const char *refname,
  82                                 const struct object_id *oid,
  83                                 void *vcache)
  84 {
  85         struct alternate_object_cache *cache = vcache;
  86         struct object *obj = parse_object(oid);
  87
  88         if (!obj || (obj->flags & ALTERNATE))
  89                 return;
  90
  91         obj->flags |= ALTERNATE;
  92         ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  93         cache->items[cache->nr++] = obj;
  94 }
  95
  96 static void for_each_cached_alternate(void (*cb)(struct object *))
  97 {
  98         static int initialized;
  99         static struct alternate_object_cache cache;
 100         size_t i;
 101
 102         if (!initialized) {
 103                 for_each_alternate_ref(cache_one_alternate, &cache);
 104                 initialized = 1;
 105         }
 106
 107         for (i = 0; i < cache.nr; i++)
 108                 cb(cache.items[i]);
 109 }
 110
 111 static void rev_list_push(struct commit *commit, int mark)
 112 {
 113         if (!(commit->object.flags & mark)) {
 114                 commit->object.flags |= mark;
 115
 116                 if (parse_commit(commit))
 117                         return;
 118
 119                 prio_queue_put(&rev_list, commit);
 120
 121                 if (!(commit->object.flags & COMMON))
 122                         non_common_revs++;
 123         }
 124 }
 125
 126 static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 127 {
 128         struct object *o = deref_tag(parse_object(oid), refname, 0);
 129
 130         if (o && o->type == OBJ_COMMIT)
 131                 rev_list_push((struct commit *)o, SEEN);
 132
 133         return 0;
 134 }
 135
 136 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 137                                    int flag, void *cb_data)
 138 {
 139         return rev_list_insert_ref(refname, oid);
 140 }
 141
 142 static int clear_marks(const char *refname, const struct object_id *oid,
 143                        int flag, void *cb_data)
 144 {
 145         struct object *o = deref_tag(parse_object(oid), refname, 0);
 146
 147         if (o && o->type == OBJ_COMMIT)
 148                 clear_commit_marks((struct commit *)o,
 149                                    COMMON | COMMON_REF | SEEN | POPPED);
 150         return 0;
 151 }
 152
 153 /*
 154    This function marks a rev and its ancestors as common.
 155    In some cases, it is desirable to mark only the ancestors (for example
 156    when only the server does not yet know that they are common).
 157 */
 158
 159 static void mark_common(struct commit *commit,
 160                 int ancestors_only, int dont_parse)
 161 {
 162         if (commit != NULL && !(commit->object.flags & COMMON)) {
 163                 struct object *o = (struct object *)commit;
 164
 165                 if (!ancestors_only)
 166                         o->flags |= COMMON;
 167
 168                 if (!(o->flags & SEEN))
 169                         rev_list_push(commit, SEEN);
 170                 else {
 171                         struct commit_list *parents;
 172
 173                         if (!ancestors_only && !(o->flags & POPPED))
 174                                 non_common_revs--;
 175                         if (!o->parsed && !dont_parse)
 176                                 if (parse_commit(commit))
 177                                         return;
 178
 179                         for (parents = commit->parents;
 180                                         parents;
 181                                         parents = parents->next)
 182                                 mark_common(parents->item, 0, dont_parse);
 183                 }
 184         }
 185 }
 186
 187 /*
 188   Get the next rev to send, ignoring the common.
 189 */
 190
 191 static const struct object_id *get_rev(void)
 192 {
 193         struct commit *commit = NULL;
 194
 195         while (commit == NULL) {
 196                 unsigned int mark;
 197                 struct commit_list *parents;
 198
 199                 if (rev_list.nr == 0 || non_common_revs == 0)
 200                         return NULL;
 201
 202                 commit = prio_queue_get(&rev_list);
 203                 parse_commit(commit);
 204                 parents = commit->parents;
 205
 206                 commit->object.flags |= POPPED;
 207                 if (!(commit->object.flags & COMMON))
 208                         non_common_revs--;
 209
 210                 if (commit->object.flags & COMMON) {
 211                         /* do not send "have", and ignore ancestors */
 212                         commit = NULL;
 213                         mark = COMMON | SEEN;
 214                 } else if (commit->object.flags & COMMON_REF)
 215                         /* send "have", and ignore ancestors */
 216                         mark = COMMON | SEEN;
 217                 else
 218                         /* send "have", also for its ancestors */
 219                         mark = SEEN;
 220
 221                 while (parents) {
 222                         if (!(parents->item->object.flags & SEEN))
 223                                 rev_list_push(parents->item, mark);
 224                         if (mark & COMMON)
 225                                 mark_common(parents->item, 1, 0);
 226                         parents = parents->next;
 227                 }
 228         }
 229
 230         return &commit->object.oid;
 231 }
 232
 233 enum ack_type {
 234         NAK = 0,
 235         ACK,
 236         ACK_continue,
 237         ACK_common,
 238         ACK_ready
 239 };
 240
 241 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 242 {
 243         if (args->stateless_rpc && args->deepen) {
 244                 /* If we sent a depth we will get back "duplicate"
 245                  * shallow and unshallow commands every time there
 246                  * is a block of have lines exchanged.
 247                  */
 248                 char *line;
 249                 while ((line = packet_read_line(fd, NULL))) {
 250                         if (starts_with(line, "shallow "))
 251                                 continue;
 252                         if (starts_with(line, "unshallow "))
 253                                 continue;
 254                         die(_("git fetch-pack: expected shallow list"));
 255                 }
 256         }
 257 }
 258
 259 static enum ack_type get_ack(int fd, struct object_id *result_oid)
 260 {
 261         int len;
 262         char *line = packet_read_line(fd, &len);
 263         const char *arg;
 264
 265         if (!line)
 266                 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
 267         if (!strcmp(line, "NAK"))
 268                 return NAK;
 269         if (skip_prefix(line, "ACK ", &arg)) {
 270                 if (!get_oid_hex(arg, result_oid)) {
 271                         arg += 40;
 272                         len -= arg - line;
 273                         if (len < 1)
 274                                 return ACK;
 275                         if (strstr(arg, "continue"))
 276                                 return ACK_continue;
 277                         if (strstr(arg, "common"))
 278                                 return ACK_common;
 279                         if (strstr(arg, "ready"))
 280                                 return ACK_ready;
 281                         return ACK;
 282                 }
 283         }
 284         if (skip_prefix(line, "ERR ", &arg))
 285                 die(_("remote error: %s"), arg);
 286         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 287 }
 288
 289 static void send_request(struct fetch_pack_args *args,
 290                          int fd, struct strbuf *buf)
 291 {
 292         if (args->stateless_rpc) {
 293                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 294                 packet_flush(fd);
 295         } else
 296                 write_or_die(fd, buf->buf, buf->len);
 297 }
 298
 299 static void insert_one_alternate_object(struct object *obj)
 300 {
 301         rev_list_insert_ref(NULL, &obj->oid);
 302 }
 303
 304 #define INITIAL_FLUSH 16
 305 #define PIPESAFE_FLUSH 32
 306 #define LARGE_FLUSH 16384
 307
 308 static int next_flush(int stateless_rpc, int count)
 309 {
 310         if (stateless_rpc) {
 311                 if (count < LARGE_FLUSH)
 312                         count <<= 1;
 313                 else
 314                         count = count * 11 / 10;
 315         } else {
 316                 if (count < PIPESAFE_FLUSH)
 317                         count <<= 1;
 318                 else
 319                         count += PIPESAFE_FLUSH;
 320         }
 321         return count;
 322 }
 323
 324 static int find_common(struct fetch_pack_args *args,
 325                        int fd[2], struct object_id *result_oid,
 326                        struct ref *refs)
 327 {
 328         int fetching;
 329         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 330         const struct object_id *oid;
 331         unsigned in_vain = 0;
 332         int got_continue = 0;
 333         int got_ready = 0;
 334         struct strbuf req_buf = STRBUF_INIT;
 335         size_t state_len = 0;
 336
 337         if (args->stateless_rpc && multi_ack == 1)
 338                 die(_("--stateless-rpc requires multi_ack_detailed"));
 339         if (marked)
 340                 for_each_ref(clear_marks, NULL);
 341         marked = 1;
 342
 343         for_each_ref(rev_list_insert_ref_oid, NULL);
 344         for_each_cached_alternate(insert_one_alternate_object);
 345
 346         fetching = 0;
 347         for ( ; refs ; refs = refs->next) {
 348                 struct object_id *remote = &refs->old_oid;
 349                 const char *remote_hex;
 350                 struct object *o;
 351
 352                 /*
 353                  * If that object is complete (i.e. it is an ancestor of a
 354                  * local ref), we tell them we have it but do not have to
 355                  * tell them about its ancestors, which they already know
 356                  * about.
 357                  *
 358                  * We use lookup_object here because we are only
 359                  * interested in the case we *know* the object is
 360                  * reachable and we have already scanned it.
 361                  */
 362                 if (((o = lookup_object(remote->hash)) != NULL) &&
 363                                 (o->flags & COMPLETE)) {
 364                         continue;
 365                 }
 366
 367                 remote_hex = oid_to_hex(remote);
 368                 if (!fetching) {
 369                         struct strbuf c = STRBUF_INIT;
 370                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 371                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 372                         if (no_done)            strbuf_addstr(&c, " no-done");
 373                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 374                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 375                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 376                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 377                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 378                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 379                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 380                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 381                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 382                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 383                                                             git_user_agent_sanitized());
 384                         if (args->filter_options.choice)
 385                                 strbuf_addstr(&c, " filter");
 386                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 387                         strbuf_release(&c);
 388                 } else
 389                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 390                 fetching++;
 391         }
 392
 393         if (!fetching) {
 394                 strbuf_release(&req_buf);
 395                 packet_flush(fd[1]);
 396                 return 1;
 397         }
 398
 399         if (is_repository_shallow())
 400                 write_shallow_commits(&req_buf, 1, NULL);
 401         if (args->depth > 0)
 402                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 403         if (args->deepen_since) {
 404                 timestamp_t max_age = approxidate(args->deepen_since);
 405                 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 406         }
 407         if (args->deepen_not) {
 408                 int i;
 409                 for (i = 0; i < args->deepen_not->nr; i++) {
 410                         struct string_list_item *s = args->deepen_not->items + i;
 411                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 412                 }
 413         }
 414         if (server_supports_filtering && args->filter_options.choice)
 415                 packet_buf_write(&req_buf, "filter %s",
 416                                  args->filter_options.filter_spec);
 417         packet_buf_flush(&req_buf);
 418         state_len = req_buf.len;
 419
 420         if (args->deepen) {
 421                 char *line;
 422                 const char *arg;
 423                 struct object_id oid;
 424
 425                 send_request(args, fd[1], &req_buf);
 426                 while ((line = packet_read_line(fd[0], NULL))) {
 427                         if (skip_prefix(line, "shallow ", &arg)) {
 428                                 if (get_oid_hex(arg, &oid))
 429                                         die(_("invalid shallow line: %s"), line);
 430                                 register_shallow(&oid);
 431                                 continue;
 432                         }
 433                         if (skip_prefix(line, "unshallow ", &arg)) {
 434                                 if (get_oid_hex(arg, &oid))
 435                                         die(_("invalid unshallow line: %s"), line);
 436                                 if (!lookup_object(oid.hash))
 437                                         die(_("object not found: %s"), line);
 438                                 /* make sure that it is parsed as shallow */
 439                                 if (!parse_object(&oid))
 440                                         die(_("error in object: %s"), line);
 441                                 if (unregister_shallow(&oid))
 442                                         die(_("no shallow found: %s"), line);
 443                                 continue;
 444                         }
 445                         die(_("expected shallow/unshallow, got %s"), line);
 446                 }
 447         } else if (!args->stateless_rpc)
 448                 send_request(args, fd[1], &req_buf);
 449
 450         if (!args->stateless_rpc) {
 451                 /* If we aren't using the stateless-rpc interface
 452                  * we don't need to retain the headers.
 453                  */
 454                 strbuf_setlen(&req_buf, 0);
 455                 state_len = 0;
 456         }
 457
 458         flushes = 0;
 459         retval = -1;
 460         if (args->no_dependents)
 461                 goto done;
 462         while ((oid = get_rev())) {
 463                 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 464                 print_verbose(args, "have %s", oid_to_hex(oid));
 465                 in_vain++;
 466                 if (flush_at <= ++count) {
 467                         int ack;
 468
 469                         packet_buf_flush(&req_buf);
 470                         send_request(args, fd[1], &req_buf);
 471                         strbuf_setlen(&req_buf, state_len);
 472                         flushes++;
 473                         flush_at = next_flush(args->stateless_rpc, count);
 474
 475                         /*
 476                          * We keep one window "ahead" of the other side, and
 477                          * will wait for an ACK only on the next one
 478                          */
 479                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 480                                 continue;
 481
 482                         consume_shallow_list(args, fd[0]);
 483                         do {
 484                                 ack = get_ack(fd[0], result_oid);
 485                                 if (ack)
 486                                         print_verbose(args, _("got %s %d %s"), "ack",
 487                                                       ack, oid_to_hex(result_oid));
 488                                 switch (ack) {
 489                                 case ACK:
 490                                         flushes = 0;
 491                                         multi_ack = 0;
 492                                         retval = 0;
 493                                         goto done;
 494                                 case ACK_common:
 495                                 case ACK_ready:
 496                                 case ACK_continue: {
 497                                         struct commit *commit =
 498                                                 lookup_commit(result_oid);
 499                                         if (!commit)
 500                                                 die(_("invalid commit %s"), oid_to_hex(result_oid));
 501                                         if (args->stateless_rpc
 502                                          && ack == ACK_common
 503                                          && !(commit->object.flags & COMMON)) {
 504                                                 /* We need to replay the have for this object
 505                                                  * on the next RPC request so the peer knows
 506                                                  * it is in common with us.
 507                                                  */
 508                                                 const char *hex = oid_to_hex(result_oid);
 509                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 510                                                 state_len = req_buf.len;
 511                                                 /*
 512                                                  * Reset in_vain because an ack
 513                                                  * for this commit has not been
 514                                                  * seen.
 515                                                  */
 516                                                 in_vain = 0;
 517                                         } else if (!args->stateless_rpc
 518                                                    || ack != ACK_common)
 519                                                 in_vain = 0;
 520                                         mark_common(commit, 0, 1);
 521                                         retval = 0;
 522                                         got_continue = 1;
 523                                         if (ack == ACK_ready) {
 524                                                 clear_prio_queue(&rev_list);
 525                                                 got_ready = 1;
 526                                         }
 527                                         break;
 528                                         }
 529                                 }
 530                         } while (ack);
 531                         flushes--;
 532                         if (got_continue && MAX_IN_VAIN < in_vain) {
 533                                 print_verbose(args, _("giving up"));
 534                                 break; /* give up */
 535                         }
 536                 }
 537         }
 538 done:
 539         if (!got_ready || !no_done) {
 540                 packet_buf_write(&req_buf, "done\n");
 541                 send_request(args, fd[1], &req_buf);
 542         }
 543         print_verbose(args, _("done"));
 544         if (retval != 0) {
 545                 multi_ack = 0;
 546                 flushes++;
 547         }
 548         strbuf_release(&req_buf);
 549
 550         if (!got_ready || !no_done)
 551                 consume_shallow_list(args, fd[0]);
 552         while (flushes || multi_ack) {
 553                 int ack = get_ack(fd[0], result_oid);
 554                 if (ack) {
 555                         print_verbose(args, _("got %s (%d) %s"), "ack",
 556                                       ack, oid_to_hex(result_oid));
 557                         if (ack == ACK)
 558                                 return 0;
 559                         multi_ack = 1;
 560                         continue;
 561                 }
 562                 flushes--;
 563         }
 564         /* it is no error to fetch into a completely empty repo */
 565         return count ? retval : 0;
 566 }
 567
 568 static struct commit_list *complete;
 569
 570 static int mark_complete(const struct object_id *oid)
 571 {
 572         struct object *o = parse_object(oid);
 573
 574         while (o && o->type == OBJ_TAG) {
 575                 struct tag *t = (struct tag *) o;
 576                 if (!t->tagged)
 577                         break; /* broken repository */
 578                 o->flags |= COMPLETE;
 579                 o = parse_object(&t->tagged->oid);
 580         }
 581         if (o && o->type == OBJ_COMMIT) {
 582                 struct commit *commit = (struct commit *)o;
 583                 if (!(commit->object.flags & COMPLETE)) {
 584                         commit->object.flags |= COMPLETE;
 585                         commit_list_insert(commit, &complete);
 586                 }
 587         }
 588         return 0;
 589 }
 590
 591 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 592                              int flag, void *cb_data)
 593 {
 594         return mark_complete(oid);
 595 }
 596
 597 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 598                                          timestamp_t cutoff)
 599 {
 600         while (complete && cutoff <= complete->item->date) {
 601                 print_verbose(args, _("Marking %s as complete"),
 602                               oid_to_hex(&complete->item->object.oid));
 603                 pop_most_recent_commit(&complete, COMPLETE);
 604         }
 605 }
 606
 607 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 608 {
 609         for (; refs; refs = refs->next)
 610                 oidset_insert(oids, &refs->old_oid);
 611 }
 612
 613 static int tip_oids_contain(struct oidset *tip_oids,
 614                             struct ref *unmatched, struct ref *newlist,
 615                             const struct object_id *id)
 616 {
 617         /*
 618          * Note that this only looks at the ref lists the first time it's
 619          * called. This works out in filter_refs() because even though it may
 620          * add to "newlist" between calls, the additions will always be for
 621          * oids that are already in the set.
 622          */
 623         if (!tip_oids->map.map.tablesize) {
 624                 add_refs_to_oidset(tip_oids, unmatched);
 625                 add_refs_to_oidset(tip_oids, newlist);
 626         }
 627         return oidset_contains(tip_oids, id);
 628 }
 629
 630 static void filter_refs(struct fetch_pack_args *args,
 631                         struct ref **refs,
 632                         struct ref **sought, int nr_sought)
 633 {
 634         struct ref *newlist = NULL;
 635         struct ref **newtail = &newlist;
 636         struct ref *unmatched = NULL;
 637         struct ref *ref, *next;
 638         struct oidset tip_oids = OIDSET_INIT;
 639         int i;
 640
 641         i = 0;
 642         for (ref = *refs; ref; ref = next) {
 643                 int keep = 0;
 644                 next = ref->next;
 645
 646                 if (starts_with(ref->name, "refs/") &&
 647                     check_refname_format(ref->name, 0))
 648                         ; /* trash */
 649                 else {
 650                         while (i < nr_sought) {
 651                                 int cmp = strcmp(ref->name, sought[i]->name);
 652                                 if (cmp < 0)
 653                                         break; /* definitely do not have it */
 654                                 else if (cmp == 0) {
 655                                         keep = 1; /* definitely have it */
 656                                         sought[i]->match_status = REF_MATCHED;
 657                                 }
 658                                 i++;
 659                         }
 660
 661                         if (!keep && args->fetch_all &&
 662                             (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 663                                 keep = 1;
 664                 }
 665
 666                 if (keep) {
 667                         *newtail = ref;
 668                         ref->next = NULL;
 669                         newtail = &ref->next;
 670                 } else {
 671                         ref->next = unmatched;
 672                         unmatched = ref;
 673                 }
 674         }
 675
 676         /* Append unmatched requests to the list */
 677         for (i = 0; i < nr_sought; i++) {
 678                 struct object_id oid;
 679                 const char *p;
 680
 681                 ref = sought[i];
 682                 if (ref->match_status != REF_NOT_MATCHED)
 683                         continue;
 684                 if (parse_oid_hex(ref->name, &oid, &p) ||
 685                     *p != '\0' ||
 686                     oidcmp(&oid, &ref->old_oid))
 687                         continue;
 688
 689                 if ((allow_unadvertised_object_request &
 690                      (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 691                     tip_oids_contain(&tip_oids, unmatched, newlist,
 692                                      &ref->old_oid)) {
 693                         ref->match_status = REF_MATCHED;
 694                         *newtail = copy_ref(ref);
 695                         newtail = &(*newtail)->next;
 696                 } else {
 697                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 698                 }
 699         }
 700
 701         oidset_clear(&tip_oids);
 702         for (ref = unmatched; ref; ref = next) {
 703                 next = ref->next;
 704                 free(ref);
 705         }
 706
 707         *refs = newlist;
 708 }
 709
 710 static void mark_alternate_complete(struct object *obj)
 711 {
 712         mark_complete(&obj->oid);
 713 }
 714
 715 struct loose_object_iter {
 716         struct oidset *loose_object_set;
 717         struct ref *refs;
 718 };
 719
 720 /*
 721  *  If the number of refs is not larger than the number of loose objects,
 722  *  this function stops inserting.
 723  */
 724 static int add_loose_objects_to_set(const struct object_id *oid,
 725                                     const char *path,
 726                                     void *data)
 727 {
 728         struct loose_object_iter *iter = data;
 729         oidset_insert(iter->loose_object_set, oid);
 730         if (iter->refs == NULL)
 731                 return 1;
 732
 733         iter->refs = iter->refs->next;
 734         return 0;
 735 }
 736
 737 static int everything_local(struct fetch_pack_args *args,
 738                             struct ref **refs,
 739                             struct ref **sought, int nr_sought)
 740 {
 741         struct ref *ref;
 742         int retval;
 743         int old_save_commit_buffer = save_commit_buffer;
 744         timestamp_t cutoff = 0;
 745         struct oidset loose_oid_set = OIDSET_INIT;
 746         int use_oidset = 0;
 747         struct loose_object_iter iter = {&loose_oid_set, *refs};
 748
 749         /* Enumerate all loose objects or know refs are not so many. */
 750         use_oidset = !for_each_loose_object(add_loose_objects_to_set,
 751                                             &iter, 0);
 752
 753         save_commit_buffer = 0;
 754
 755         for (ref = *refs; ref; ref = ref->next) {
 756                 struct object *o;
 757                 unsigned int flags = OBJECT_INFO_QUICK;
 758
 759                 if (use_oidset &&
 760                     !oidset_contains(&loose_oid_set, &ref->old_oid)) {
 761                         /*
 762                          * I know this does not exist in the loose form,
 763                          * so check if it exists in a non-loose form.
 764                          */
 765                         flags |= OBJECT_INFO_IGNORE_LOOSE;
 766                 }
 767
 768                 if (!has_object_file_with_flags(&ref->old_oid, flags))
 769                         continue;
 770                 o = parse_object(&ref->old_oid);
 771                 if (!o)
 772                         continue;
 773
 774                 /* We already have it -- which may mean that we were
 775                  * in sync with the other side at some time after
 776                  * that (it is OK if we guess wrong here).
 777                  */
 778                 if (o->type == OBJ_COMMIT) {
 779                         struct commit *commit = (struct commit *)o;
 780                         if (!cutoff || cutoff < commit->date)
 781                                 cutoff = commit->date;
 782                 }
 783         }
 784
 785         oidset_clear(&loose_oid_set);
 786
 787         if (!args->no_dependents) {
 788                 if (!args->deepen) {
 789                         for_each_ref(mark_complete_oid, NULL);
 790                         for_each_cached_alternate(mark_alternate_complete);
 791                         commit_list_sort_by_date(&complete);
 792                         if (cutoff)
 793                                 mark_recent_complete_commits(args, cutoff);
 794                 }
 795
 796                 /*
 797                  * Mark all complete remote refs as common refs.
 798                  * Don't mark them common yet; the server has to be told so first.
 799                  */
 800                 for (ref = *refs; ref; ref = ref->next) {
 801                         struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 802                                                      NULL, 0);
 803
 804                         if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 805                                 continue;
 806
 807                         if (!(o->flags & SEEN)) {
 808                                 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 809
 810                                 mark_common((struct commit *)o, 1, 1);
 811                         }
 812                 }
 813         }
 814
 815         filter_refs(args, refs, sought, nr_sought);
 816
 817         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 818                 const struct object_id *remote = &ref->old_oid;
 819                 struct object *o;
 820
 821                 o = lookup_object(remote->hash);
 822                 if (!o || !(o->flags & COMPLETE)) {
 823                         retval = 0;
 824                         print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 825                                       ref->name);
 826                         continue;
 827                 }
 828                 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 829                               ref->name);
 830         }
 831
 832         save_commit_buffer = old_save_commit_buffer;
 833
 834         return retval;
 835 }
 836
 837 static int sideband_demux(int in, int out, void *data)
 838 {
 839         int *xd = data;
 840         int ret;
 841
 842         ret = recv_sideband("fetch-pack", xd[0], out);
 843         close(out);
 844         return ret;
 845 }
 846
 847 static int get_pack(struct fetch_pack_args *args,
 848                     int xd[2], char **pack_lockfile)
 849 {
 850         struct async demux;
 851         int do_keep = args->keep_pack;
 852         const char *cmd_name;
 853         struct pack_header header;
 854         int pass_header = 0;
 855         struct child_process cmd = CHILD_PROCESS_INIT;
 856         int ret;
 857
 858         memset(&demux, 0, sizeof(demux));
 859         if (use_sideband) {
 860                 /* xd[] is talking with upload-pack; subprocess reads from
 861                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 862                  * through demux->out.
 863                  */
 864                 demux.proc = sideband_demux;
 865                 demux.data = xd;
 866                 demux.out = -1;
 867                 demux.isolate_sigpipe = 1;
 868                 if (start_async(&demux))
 869                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 870         }
 871         else
 872                 demux.out = xd[0];
 873
 874         if (!args->keep_pack && unpack_limit) {
 875
 876                 if (read_pack_header(demux.out, &header))
 877                         die(_("protocol error: bad pack header"));
 878                 pass_header = 1;
 879                 if (ntohl(header.hdr_entries) < unpack_limit)
 880                         do_keep = 0;
 881                 else
 882                         do_keep = 1;
 883         }
 884
 885         if (alternate_shallow_file) {
 886                 argv_array_push(&cmd.args, "--shallow-file");
 887                 argv_array_push(&cmd.args, alternate_shallow_file);
 888         }
 889
 890         if (do_keep || args->from_promisor) {
 891                 if (pack_lockfile)
 892                         cmd.out = -1;
 893                 cmd_name = "index-pack";
 894                 argv_array_push(&cmd.args, cmd_name);
 895                 argv_array_push(&cmd.args, "--stdin");
 896                 if (!args->quiet && !args->no_progress)
 897                         argv_array_push(&cmd.args, "-v");
 898                 if (args->use_thin_pack)
 899                         argv_array_push(&cmd.args, "--fix-thin");
 900                 if (do_keep && (args->lock_pack || unpack_limit)) {
 901                         char hostname[HOST_NAME_MAX + 1];
 902                         if (xgethostname(hostname, sizeof(hostname)))
 903                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 904                         argv_array_pushf(&cmd.args,
 905                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 906                                         (uintmax_t)getpid(), hostname);
 907                 }
 908                 if (args->check_self_contained_and_connected)
 909                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 910                 if (args->from_promisor)
 911                         argv_array_push(&cmd.args, "--promisor");
 912         }
 913         else {
 914                 cmd_name = "unpack-objects";
 915                 argv_array_push(&cmd.args, cmd_name);
 916                 if (args->quiet || args->no_progress)
 917                         argv_array_push(&cmd.args, "-q");
 918                 args->check_self_contained_and_connected = 0;
 919         }
 920
 921         if (pass_header)
 922                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 923                                  ntohl(header.hdr_version),
 924                                  ntohl(header.hdr_entries));
 925         if (fetch_fsck_objects >= 0
 926             ? fetch_fsck_objects
 927             : transfer_fsck_objects >= 0
 928             ? transfer_fsck_objects
 929             : 0) {
 930                 if (args->from_promisor)
 931                         /*
 932                          * We cannot use --strict in index-pack because it
 933                          * checks both broken objects and links, but we only
 934                          * want to check for broken objects.
 935                          */
 936                         argv_array_push(&cmd.args, "--fsck-objects");
 937                 else
 938                         argv_array_push(&cmd.args, "--strict");
 939         }
 940
 941         cmd.in = demux.out;
 942         cmd.git_cmd = 1;
 943         if (start_command(&cmd))
 944                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 945         if (do_keep && pack_lockfile) {
 946                 *pack_lockfile = index_pack_lockfile(cmd.out);
 947                 close(cmd.out);
 948         }
 949
 950         if (!use_sideband)
 951                 /* Closed by start_command() */
 952                 xd[0] = -1;
 953
 954         ret = finish_command(&cmd);
 955         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 956                 args->self_contained_and_connected =
 957                         args->check_self_contained_and_connected &&
 958                         ret == 0;
 959         else
 960                 die(_("%s failed"), cmd_name);
 961         if (use_sideband && finish_async(&demux))
 962                 die(_("error in sideband demultiplexer"));
 963         return 0;
 964 }
 965
 966 static int cmp_ref_by_name(const void *a_, const void *b_)
 967 {
 968         const struct ref *a = *((const struct ref **)a_);
 969         const struct ref *b = *((const struct ref **)b_);
 970         return strcmp(a->name, b->name);
 971 }
 972
 973 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 974                                  int fd[2],
 975                                  const struct ref *orig_ref,
 976                                  struct ref **sought, int nr_sought,
 977                                  struct shallow_info *si,
 978                                  char **pack_lockfile)
 979 {
 980         struct ref *ref = copy_ref_list(orig_ref);
 981         struct object_id oid;
 982         const char *agent_feature;
 983         int agent_len;
 984
 985         sort_ref_list(&ref, ref_compare_name);
 986         QSORT(sought, nr_sought, cmp_ref_by_name);
 987
 988         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 989                 die(_("Server does not support shallow clients"));
 990         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 991                 args->deepen = 1;
 992         if (server_supports("multi_ack_detailed")) {
 993                 print_verbose(args, _("Server supports multi_ack_detailed"));
 994                 multi_ack = 2;
 995                 if (server_supports("no-done")) {
 996                         print_verbose(args, _("Server supports no-done"));
 997                         if (args->stateless_rpc)
 998                                 no_done = 1;
 999                 }
1000         }
1001         else if (server_supports("multi_ack")) {
1002                 print_verbose(args, _("Server supports multi_ack"));
1003                 multi_ack = 1;
1004         }
1005         if (server_supports("side-band-64k")) {
1006                 print_verbose(args, _("Server supports side-band-64k"));
1007                 use_sideband = 2;
1008         }
1009         else if (server_supports("side-band")) {
1010                 print_verbose(args, _("Server supports side-band"));
1011                 use_sideband = 1;
1012         }
1013         if (server_supports("allow-tip-sha1-in-want")) {
1014                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
1015                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1016         }
1017         if (server_supports("allow-reachable-sha1-in-want")) {
1018                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
1019                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1020         }
1021         if (!server_supports("thin-pack"))
1022                 args->use_thin_pack = 0;
1023         if (!server_supports("no-progress"))
1024                 args->no_progress = 0;
1025         if (!server_supports("include-tag"))
1026                 args->include_tag = 0;
1027         if (server_supports("ofs-delta"))
1028                 print_verbose(args, _("Server supports ofs-delta"));
1029         else
1030                 prefer_ofs_delta = 0;
1031
1032         if (server_supports("filter")) {
1033                 server_supports_filtering = 1;
1034                 print_verbose(args, _("Server supports filter"));
1035         } else if (args->filter_options.choice) {
1036                 warning("filtering not recognized by server, ignoring");
1037         }
1038
1039         if ((agent_feature = server_feature_value("agent", &agent_len))) {
1040                 agent_supported = 1;
1041                 if (agent_len)
1042                         print_verbose(args, _("Server version is %.*s"),
1043                                       agent_len, agent_feature);
1044         }
1045         if (server_supports("deepen-since"))
1046                 deepen_since_ok = 1;
1047         else if (args->deepen_since)
1048                 die(_("Server does not support --shallow-since"));
1049         if (server_supports("deepen-not"))
1050                 deepen_not_ok = 1;
1051         else if (args->deepen_not)
1052                 die(_("Server does not support --shallow-exclude"));
1053         if (!server_supports("deepen-relative") && args->deepen_relative)
1054                 die(_("Server does not support --deepen"));
1055
1056         if (everything_local(args, &ref, sought, nr_sought)) {
1057                 packet_flush(fd[1]);
1058                 goto all_done;
1059         }
1060         if (find_common(args, fd, &oid, ref) < 0)
1061                 if (!args->keep_pack)
1062                         /* When cloning, it is not unusual to have
1063                          * no common commit.
1064                          */
1065                         warning(_("no common commits"));
1066
1067         if (args->stateless_rpc)
1068                 packet_flush(fd[1]);
1069         if (args->deepen)
1070                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1071                                         NULL);
1072         else if (si->nr_ours || si->nr_theirs)
1073                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1074         else
1075                 alternate_shallow_file = NULL;
1076         if (get_pack(args, fd, pack_lockfile))
1077                 die(_("git fetch-pack: fetch failed."));
1078
1079  all_done:
1080         return ref;
1081 }
1082
1083 static void add_shallow_requests(struct strbuf *req_buf,
1084                                  const struct fetch_pack_args *args)
1085 {
1086         if (is_repository_shallow())
1087                 write_shallow_commits(req_buf, 1, NULL);
1088         if (args->depth > 0)
1089                 packet_buf_write(req_buf, "deepen %d", args->depth);
1090         if (args->deepen_since) {
1091                 timestamp_t max_age = approxidate(args->deepen_since);
1092                 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1093         }
1094         if (args->deepen_not) {
1095                 int i;
1096                 for (i = 0; i < args->deepen_not->nr; i++) {
1097                         struct string_list_item *s = args->deepen_not->items + i;
1098                         packet_buf_write(req_buf, "deepen-not %s", s->string);
1099                 }
1100         }
1101 }
1102
1103 static void add_wants(const struct ref *wants, struct strbuf *req_buf)
1104 {
1105         for ( ; wants ; wants = wants->next) {
1106                 const struct object_id *remote = &wants->old_oid;
1107                 const char *remote_hex;
1108                 struct object *o;
1109
1110                 /*
1111                  * If that object is complete (i.e. it is an ancestor of a
1112                  * local ref), we tell them we have it but do not have to
1113                  * tell them about its ancestors, which they already know
1114                  * about.
1115                  *
1116                  * We use lookup_object here because we are only
1117                  * interested in the case we *know* the object is
1118                  * reachable and we have already scanned it.
1119                  */
1120                 if (((o = lookup_object(remote->hash)) != NULL) &&
1121                     (o->flags & COMPLETE)) {
1122                         continue;
1123                 }
1124
1125                 remote_hex = oid_to_hex(remote);
1126                 packet_buf_write(req_buf, "want %s\n", remote_hex);
1127         }
1128 }
1129
1130 static void add_common(struct strbuf *req_buf, struct oidset *common)
1131 {
1132         struct oidset_iter iter;
1133         const struct object_id *oid;
1134         oidset_iter_init(common, &iter);
1135
1136         while ((oid = oidset_iter_next(&iter))) {
1137                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1138         }
1139 }
1140
1141 static int add_haves(struct strbuf *req_buf, int *haves_to_send, int *in_vain)
1142 {
1143         int ret = 0;
1144         int haves_added = 0;
1145         const struct object_id *oid;
1146
1147         while ((oid = get_rev())) {
1148                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1149                 if (++haves_added >= *haves_to_send)
1150                         break;
1151         }
1152
1153         *in_vain += haves_added;
1154         if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1155                 /* Send Done */
1156                 packet_buf_write(req_buf, "done\n");
1157                 ret = 1;
1158         }
1159
1160         /* Increase haves to send on next round */
1161         *haves_to_send = next_flush(1, *haves_to_send);
1162
1163         return ret;
1164 }
1165
1166 static int send_fetch_request(int fd_out, const struct fetch_pack_args *args,
1167                               const struct ref *wants, struct oidset *common,
1168                               int *haves_to_send, int *in_vain)
1169 {
1170         int ret = 0;
1171         struct strbuf req_buf = STRBUF_INIT;
1172
1173         if (server_supports_v2("fetch", 1))
1174                 packet_buf_write(&req_buf, "command=fetch");
1175         if (server_supports_v2("agent", 0))
1176                 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1177         if (args->server_options && args->server_options->nr &&
1178             server_supports_v2("server-option", 1)) {
1179                 int i;
1180                 for (i = 0; i < args->server_options->nr; i++)
1181                         packet_write_fmt(fd_out, "server-option=%s",
1182                                          args->server_options->items[i].string);
1183         }
1184
1185         packet_buf_delim(&req_buf);
1186         if (args->use_thin_pack)
1187                 packet_buf_write(&req_buf, "thin-pack");
1188         if (args->no_progress)
1189                 packet_buf_write(&req_buf, "no-progress");
1190         if (args->include_tag)
1191                 packet_buf_write(&req_buf, "include-tag");
1192         if (prefer_ofs_delta)
1193                 packet_buf_write(&req_buf, "ofs-delta");
1194
1195         /* Add shallow-info and deepen request */
1196         if (server_supports_feature("fetch", "shallow", 0))
1197                 add_shallow_requests(&req_buf, args);
1198         else if (is_repository_shallow() || args->deepen)
1199                 die(_("Server does not support shallow requests"));
1200
1201         /* Add filter */
1202         if (server_supports_feature("fetch", "filter", 0) &&
1203             args->filter_options.choice) {
1204                 print_verbose(args, _("Server supports filter"));
1205                 packet_buf_write(&req_buf, "filter %s",
1206                                  args->filter_options.filter_spec);
1207         } else if (args->filter_options.choice) {
1208                 warning("filtering not recognized by server, ignoring");
1209         }
1210
1211         /* add wants */
1212         add_wants(wants, &req_buf);
1213
1214         if (args->no_dependents) {
1215                 packet_buf_write(&req_buf, "done");
1216                 ret = 1;
1217         } else {
1218                 /* Add all of the common commits we've found in previous rounds */
1219                 add_common(&req_buf, common);
1220
1221                 /* Add initial haves */
1222                 ret = add_haves(&req_buf, haves_to_send, in_vain);
1223         }
1224
1225         /* Send request */
1226         packet_buf_flush(&req_buf);
1227         write_or_die(fd_out, req_buf.buf, req_buf.len);
1228
1229         strbuf_release(&req_buf);
1230         return ret;
1231 }
1232
1233 /*
1234  * Processes a section header in a server's response and checks if it matches
1235  * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1236  * not consumed); if 0, the line will be consumed and the function will die if
1237  * the section header doesn't match what was expected.
1238  */
1239 static int process_section_header(struct packet_reader *reader,
1240                                   const char *section, int peek)
1241 {
1242         int ret;
1243
1244         if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1245                 die("error reading section header '%s'", section);
1246
1247         ret = !strcmp(reader->line, section);
1248
1249         if (!peek) {
1250                 if (!ret)
1251                         die("expected '%s', received '%s'",
1252                             section, reader->line);
1253                 packet_reader_read(reader);
1254         }
1255
1256         return ret;
1257 }
1258
1259 static int process_acks(struct packet_reader *reader, struct oidset *common)
1260 {
1261         /* received */
1262         int received_ready = 0;
1263         int received_ack = 0;
1264
1265         process_section_header(reader, "acknowledgments", 0);
1266         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1267                 const char *arg;
1268
1269                 if (!strcmp(reader->line, "NAK"))
1270                         continue;
1271
1272                 if (skip_prefix(reader->line, "ACK ", &arg)) {
1273                         struct object_id oid;
1274                         if (!get_oid_hex(arg, &oid)) {
1275                                 struct commit *commit;
1276                                 oidset_insert(common, &oid);
1277                                 commit = lookup_commit(&oid);
1278                                 mark_common(commit, 0, 1);
1279                         }
1280                         continue;
1281                 }
1282
1283                 if (!strcmp(reader->line, "ready")) {
1284                         clear_prio_queue(&rev_list);
1285                         received_ready = 1;
1286                         continue;
1287                 }
1288
1289                 die("unexpected acknowledgment line: '%s'", reader->line);
1290         }
1291
1292         if (reader->status != PACKET_READ_FLUSH &&
1293             reader->status != PACKET_READ_DELIM)
1294                 die("error processing acks: %d", reader->status);
1295
1296         /* return 0 if no common, 1 if there are common, or 2 if ready */
1297         return received_ready ? 2 : (received_ack ? 1 : 0);
1298 }
1299
1300 static void receive_shallow_info(struct fetch_pack_args *args,
1301                                  struct packet_reader *reader)
1302 {
1303         process_section_header(reader, "shallow-info", 0);
1304         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1305                 const char *arg;
1306                 struct object_id oid;
1307
1308                 if (skip_prefix(reader->line, "shallow ", &arg)) {
1309                         if (get_oid_hex(arg, &oid))
1310                                 die(_("invalid shallow line: %s"), reader->line);
1311                         register_shallow(&oid);
1312                         continue;
1313                 }
1314                 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1315                         if (get_oid_hex(arg, &oid))
1316                                 die(_("invalid unshallow line: %s"), reader->line);
1317                         if (!lookup_object(oid.hash))
1318                                 die(_("object not found: %s"), reader->line);
1319                         /* make sure that it is parsed as shallow */
1320                         if (!parse_object(&oid))
1321                                 die(_("error in object: %s"), reader->line);
1322                         if (unregister_shallow(&oid))
1323                                 die(_("no shallow found: %s"), reader->line);
1324                         continue;
1325                 }
1326                 die(_("expected shallow/unshallow, got %s"), reader->line);
1327         }
1328
1329         if (reader->status != PACKET_READ_FLUSH &&
1330             reader->status != PACKET_READ_DELIM)
1331                 die("error processing shallow info: %d", reader->status);
1332
1333         setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1334         args->deepen = 1;
1335 }
1336
1337 enum fetch_state {
1338         FETCH_CHECK_LOCAL = 0,
1339         FETCH_SEND_REQUEST,
1340         FETCH_PROCESS_ACKS,
1341         FETCH_GET_PACK,
1342         FETCH_DONE,
1343 };
1344
1345 static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1346                                     int fd[2],
1347                                     const struct ref *orig_ref,
1348                                     struct ref **sought, int nr_sought,
1349                                     char **pack_lockfile)
1350 {
1351         struct ref *ref = copy_ref_list(orig_ref);
1352         enum fetch_state state = FETCH_CHECK_LOCAL;
1353         struct oidset common = OIDSET_INIT;
1354         struct packet_reader reader;
1355         int in_vain = 0;
1356         int haves_to_send = INITIAL_FLUSH;
1357         packet_reader_init(&reader, fd[0], NULL, 0,
1358                            PACKET_READ_CHOMP_NEWLINE);
1359
1360         while (state != FETCH_DONE) {
1361                 switch (state) {
1362                 case FETCH_CHECK_LOCAL:
1363                         sort_ref_list(&ref, ref_compare_name);
1364                         QSORT(sought, nr_sought, cmp_ref_by_name);
1365
1366                         /* v2 supports these by default */
1367                         allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1368                         use_sideband = 2;
1369                         if (args->depth > 0 || args->deepen_since || args->deepen_not)
1370                                 args->deepen = 1;
1371
1372                         if (marked)
1373                                 for_each_ref(clear_marks, NULL);
1374                         marked = 1;
1375
1376                         for_each_ref(rev_list_insert_ref_oid, NULL);
1377                         for_each_cached_alternate(insert_one_alternate_object);
1378
1379                         /* Filter 'ref' by 'sought' and those that aren't local */
1380                         if (everything_local(args, &ref, sought, nr_sought))
1381                                 state = FETCH_DONE;
1382                         else
1383                                 state = FETCH_SEND_REQUEST;
1384                         break;
1385                 case FETCH_SEND_REQUEST:
1386                         if (send_fetch_request(fd[1], args, ref, &common,
1387                                                &haves_to_send, &in_vain))
1388                                 state = FETCH_GET_PACK;
1389                         else
1390                                 state = FETCH_PROCESS_ACKS;
1391                         break;
1392                 case FETCH_PROCESS_ACKS:
1393                         /* Process ACKs/NAKs */
1394                         switch (process_acks(&reader, &common)) {
1395                         case 2:
1396                                 state = FETCH_GET_PACK;
1397                                 break;
1398                         case 1:
1399                                 in_vain = 0;
1400                                 /* fallthrough */
1401                         default:
1402                                 state = FETCH_SEND_REQUEST;
1403                                 break;
1404                         }
1405                         break;
1406                 case FETCH_GET_PACK:
1407                         /* Check for shallow-info section */
1408                         if (process_section_header(&reader, "shallow-info", 1))
1409                                 receive_shallow_info(args, &reader);
1410
1411                         /* get the pack */
1412                         process_section_header(&reader, "packfile", 0);
1413                         if (get_pack(args, fd, pack_lockfile))
1414                                 die(_("git fetch-pack: fetch failed."));
1415
1416                         state = FETCH_DONE;
1417                         break;
1418                 case FETCH_DONE:
1419                         continue;
1420                 }
1421         }
1422
1423         oidset_clear(&common);
1424         return ref;
1425 }
1426
1427 static void fetch_pack_config(void)
1428 {
1429         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1430         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1431         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1432         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1433         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1434
1435         git_config(git_default_config, NULL);
1436 }
1437
1438 static void fetch_pack_setup(void)
1439 {
1440         static int did_setup;
1441         if (did_setup)
1442                 return;
1443         fetch_pack_config();
1444         if (0 <= transfer_unpack_limit)
1445                 unpack_limit = transfer_unpack_limit;
1446         else if (0 <= fetch_unpack_limit)
1447                 unpack_limit = fetch_unpack_limit;
1448         did_setup = 1;
1449 }
1450
1451 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1452 {
1453         struct string_list names = STRING_LIST_INIT_NODUP;
1454         int src, dst;
1455
1456         for (src = dst = 0; src < nr; src++) {
1457                 struct string_list_item *item;
1458                 item = string_list_insert(&names, ref[src]->name);
1459                 if (item->util)
1460                         continue; /* already have it */
1461                 item->util = ref[src];
1462                 if (src != dst)
1463                         ref[dst] = ref[src];
1464                 dst++;
1465         }
1466         for (src = dst; src < nr; src++)
1467                 ref[src] = NULL;
1468         string_list_clear(&names, 0);
1469         return dst;
1470 }
1471
1472 static void update_shallow(struct fetch_pack_args *args,
1473                            struct ref **sought, int nr_sought,
1474                            struct shallow_info *si)
1475 {
1476         struct oid_array ref = OID_ARRAY_INIT;
1477         int *status;
1478         int i;
1479
1480         if (args->deepen && alternate_shallow_file) {
1481                 if (*alternate_shallow_file == '\0') { /* --unshallow */
1482                         unlink_or_warn(git_path_shallow());
1483                         rollback_lock_file(&shallow_lock);
1484                 } else
1485                         commit_lock_file(&shallow_lock);
1486                 return;
1487         }
1488
1489         if (!si->shallow || !si->shallow->nr)
1490                 return;
1491
1492         if (args->cloning) {
1493                 /*
1494                  * remote is shallow, but this is a clone, there are
1495                  * no objects in repo to worry about. Accept any
1496                  * shallow points that exist in the pack (iow in repo
1497                  * after get_pack() and reprepare_packed_git())
1498                  */
1499                 struct oid_array extra = OID_ARRAY_INIT;
1500                 struct object_id *oid = si->shallow->oid;
1501                 for (i = 0; i < si->shallow->nr; i++)
1502                         if (has_object_file(&oid[i]))
1503                                 oid_array_append(&extra, &oid[i]);
1504                 if (extra.nr) {
1505                         setup_alternate_shallow(&shallow_lock,
1506                                                 &alternate_shallow_file,
1507                                                 &extra);
1508                         commit_lock_file(&shallow_lock);
1509                 }
1510                 oid_array_clear(&extra);
1511                 return;
1512         }
1513
1514         if (!si->nr_ours && !si->nr_theirs)
1515                 return;
1516
1517         remove_nonexistent_theirs_shallow(si);
1518         if (!si->nr_ours && !si->nr_theirs)
1519                 return;
1520         for (i = 0; i < nr_sought; i++)
1521                 oid_array_append(&ref, &sought[i]->old_oid);
1522         si->ref = &ref;
1523
1524         if (args->update_shallow) {
1525                 /*
1526                  * remote is also shallow, .git/shallow may be updated
1527                  * so all refs can be accepted. Make sure we only add
1528                  * shallow roots that are actually reachable from new
1529                  * refs.
1530                  */
1531                 struct oid_array extra = OID_ARRAY_INIT;
1532                 struct object_id *oid = si->shallow->oid;
1533                 assign_shallow_commits_to_refs(si, NULL, NULL);
1534                 if (!si->nr_ours && !si->nr_theirs) {
1535                         oid_array_clear(&ref);
1536                         return;
1537                 }
1538                 for (i = 0; i < si->nr_ours; i++)
1539                         oid_array_append(&extra, &oid[si->ours[i]]);
1540                 for (i = 0; i < si->nr_theirs; i++)
1541                         oid_array_append(&extra, &oid[si->theirs[i]]);
1542                 setup_alternate_shallow(&shallow_lock,
1543                                         &alternate_shallow_file,
1544                                         &extra);
1545                 commit_lock_file(&shallow_lock);
1546                 oid_array_clear(&extra);
1547                 oid_array_clear(&ref);
1548                 return;
1549         }
1550
1551         /*
1552          * remote is also shallow, check what ref is safe to update
1553          * without updating .git/shallow
1554          */
1555         status = xcalloc(nr_sought, sizeof(*status));
1556         assign_shallow_commits_to_refs(si, NULL, status);
1557         if (si->nr_ours || si->nr_theirs) {
1558                 for (i = 0; i < nr_sought; i++)
1559                         if (status[i])
1560                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1561         }
1562         free(status);
1563         oid_array_clear(&ref);
1564 }
1565
1566 struct ref *fetch_pack(struct fetch_pack_args *args,
1567                        int fd[], struct child_process *conn,
1568                        const struct ref *ref,
1569                        const char *dest,
1570                        struct ref **sought, int nr_sought,
1571                        struct oid_array *shallow,
1572                        char **pack_lockfile,
1573                        enum protocol_version version)
1574 {
1575         struct ref *ref_cpy;
1576         struct shallow_info si;
1577
1578         fetch_pack_setup();
1579         if (nr_sought)
1580                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1581
1582         if (!ref) {
1583                 packet_flush(fd[1]);
1584                 die(_("no matching remote head"));
1585         }
1586         prepare_shallow_info(&si, shallow);
1587         if (version == protocol_v2)
1588                 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1589                                            pack_lockfile);
1590         else
1591                 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1592                                         &si, pack_lockfile);
1593         reprepare_packed_git(the_repository);
1594         update_shallow(args, sought, nr_sought, &si);
1595         clear_shallow_info(&si);
1596         return ref_cpy;
1597 }
1598
1599 int report_unmatched_refs(struct ref **sought, int nr_sought)
1600 {
1601         int i, ret = 0;
1602
1603         for (i = 0; i < nr_sought; i++) {
1604                 if (!sought[i])
1605                         continue;
1606                 switch (sought[i]->match_status) {
1607                 case REF_MATCHED:
1608                         continue;
1609                 case REF_NOT_MATCHED:
1610                         error(_("no such remote ref %s"), sought[i]->name);
1611                         break;
1612                 case REF_UNADVERTISED_NOT_ALLOWED:
1613                         error(_("Server does not allow request for unadvertised object %s"),
1614                               sought[i]->name);
1615                         break;
1616                 }
1617                 ret = 1;
1618         }
1619         return ret;
1620 }