git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "repository.h"
   3 #include "config.h"
   4 #include "lockfile.h"
   5 #include "refs.h"
   6 #include "pkt-line.h"
   7 #include "commit.h"
   8 #include "tag.h"
   9 #include "exec_cmd.h"
  10 #include "pack.h"
  11 #include "sideband.h"
  12 #include "fetch-pack.h"
  13 #include "remote.h"
  14 #include "run-command.h"
  15 #include "connect.h"
  16 #include "transport.h"
  17 #include "version.h"
  18 #include "prio-queue.h"
  19 #include "sha1-array.h"
  20 #include "oidset.h"
  21 #include "packfile.h"
  22 #include "object-store.h"
  23
  24 static int transfer_unpack_limit = -1;
  25 static int fetch_unpack_limit = -1;
  26 static int unpack_limit = 100;
  27 static int prefer_ofs_delta = 1;
  28 static int no_done;
  29 static int deepen_since_ok;
  30 static int deepen_not_ok;
  31 static int fetch_fsck_objects = -1;
  32 static int transfer_fsck_objects = -1;
  33 static int agent_supported;
  34 static int server_supports_filtering;
  35 static struct lock_file shallow_lock;
  36 static const char *alternate_shallow_file;
  37
  38 /* Remember to update object flag allocation in object.h */
  39 #define COMPLETE        (1U << 0)
  40 #define COMMON          (1U << 1)
  41 #define COMMON_REF      (1U << 2)
  42 #define SEEN            (1U << 3)
  43 #define POPPED          (1U << 4)
  44 #define ALTERNATE       (1U << 5)
  45
  46 static int marked;
  47
  48 /*
  49  * After sending this many "have"s if we do not get any new ACK , we
  50  * give up traversing our history.
  51  */
  52 #define MAX_IN_VAIN 256
  53
  54 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  55 static int non_common_revs, multi_ack, use_sideband;
  56 /* Allow specifying sha1 if it is a ref tip. */
  57 #define ALLOW_TIP_SHA1  01
  58 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  59 #define ALLOW_REACHABLE_SHA1    02
  60 static unsigned int allow_unadvertised_object_request;
  61
  62 __attribute__((format (printf, 2, 3)))
  63 static inline void print_verbose(const struct fetch_pack_args *args,
  64                                  const char *fmt, ...)
  65 {
  66         va_list params;
  67
  68         if (!args->verbose)
  69                 return;
  70
  71         va_start(params, fmt);
  72         vfprintf(stderr, fmt, params);
  73         va_end(params);
  74         fputc('\n', stderr);
  75 }
  76
  77 struct alternate_object_cache {
  78         struct object **items;
  79         size_t nr, alloc;
  80 };
  81
  82 static void cache_one_alternate(const char *refname,
  83                                 const struct object_id *oid,
  84                                 void *vcache)
  85 {
  86         struct alternate_object_cache *cache = vcache;
  87         struct object *obj = parse_object(oid);
  88
  89         if (!obj || (obj->flags & ALTERNATE))
  90                 return;
  91
  92         obj->flags |= ALTERNATE;
  93         ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  94         cache->items[cache->nr++] = obj;
  95 }
  96
  97 static void for_each_cached_alternate(void (*cb)(struct object *))
  98 {
  99         static int initialized;
 100         static struct alternate_object_cache cache;
 101         size_t i;
 102
 103         if (!initialized) {
 104                 for_each_alternate_ref(cache_one_alternate, &cache);
 105                 initialized = 1;
 106         }
 107
 108         for (i = 0; i < cache.nr; i++)
 109                 cb(cache.items[i]);
 110 }
 111
 112 static void rev_list_push(struct commit *commit, int mark)
 113 {
 114         if (!(commit->object.flags & mark)) {
 115                 commit->object.flags |= mark;
 116
 117                 if (parse_commit(commit))
 118                         return;
 119
 120                 prio_queue_put(&rev_list, commit);
 121
 122                 if (!(commit->object.flags & COMMON))
 123                         non_common_revs++;
 124         }
 125 }
 126
 127 static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 128 {
 129         struct object *o = deref_tag(parse_object(oid), refname, 0);
 130
 131         if (o && o->type == OBJ_COMMIT)
 132                 rev_list_push((struct commit *)o, SEEN);
 133
 134         return 0;
 135 }
 136
 137 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 138                                    int flag, void *cb_data)
 139 {
 140         return rev_list_insert_ref(refname, oid);
 141 }
 142
 143 static int clear_marks(const char *refname, const struct object_id *oid,
 144                        int flag, void *cb_data)
 145 {
 146         struct object *o = deref_tag(parse_object(oid), refname, 0);
 147
 148         if (o && o->type == OBJ_COMMIT)
 149                 clear_commit_marks((struct commit *)o,
 150                                    COMMON | COMMON_REF | SEEN | POPPED);
 151         return 0;
 152 }
 153
 154 /*
 155    This function marks a rev and its ancestors as common.
 156    In some cases, it is desirable to mark only the ancestors (for example
 157    when only the server does not yet know that they are common).
 158 */
 159
 160 static void mark_common(struct commit *commit,
 161                 int ancestors_only, int dont_parse)
 162 {
 163         if (commit != NULL && !(commit->object.flags & COMMON)) {
 164                 struct object *o = (struct object *)commit;
 165
 166                 if (!ancestors_only)
 167                         o->flags |= COMMON;
 168
 169                 if (!(o->flags & SEEN))
 170                         rev_list_push(commit, SEEN);
 171                 else {
 172                         struct commit_list *parents;
 173
 174                         if (!ancestors_only && !(o->flags & POPPED))
 175                                 non_common_revs--;
 176                         if (!o->parsed && !dont_parse)
 177                                 if (parse_commit(commit))
 178                                         return;
 179
 180                         for (parents = commit->parents;
 181                                         parents;
 182                                         parents = parents->next)
 183                                 mark_common(parents->item, 0, dont_parse);
 184                 }
 185         }
 186 }
 187
 188 /*
 189   Get the next rev to send, ignoring the common.
 190 */
 191
 192 static const struct object_id *get_rev(void)
 193 {
 194         struct commit *commit = NULL;
 195
 196         while (commit == NULL) {
 197                 unsigned int mark;
 198                 struct commit_list *parents;
 199
 200                 if (rev_list.nr == 0 || non_common_revs == 0)
 201                         return NULL;
 202
 203                 commit = prio_queue_get(&rev_list);
 204                 parse_commit(commit);
 205                 parents = commit->parents;
 206
 207                 commit->object.flags |= POPPED;
 208                 if (!(commit->object.flags & COMMON))
 209                         non_common_revs--;
 210
 211                 if (commit->object.flags & COMMON) {
 212                         /* do not send "have", and ignore ancestors */
 213                         commit = NULL;
 214                         mark = COMMON | SEEN;
 215                 } else if (commit->object.flags & COMMON_REF)
 216                         /* send "have", and ignore ancestors */
 217                         mark = COMMON | SEEN;
 218                 else
 219                         /* send "have", also for its ancestors */
 220                         mark = SEEN;
 221
 222                 while (parents) {
 223                         if (!(parents->item->object.flags & SEEN))
 224                                 rev_list_push(parents->item, mark);
 225                         if (mark & COMMON)
 226                                 mark_common(parents->item, 1, 0);
 227                         parents = parents->next;
 228                 }
 229         }
 230
 231         return &commit->object.oid;
 232 }
 233
 234 enum ack_type {
 235         NAK = 0,
 236         ACK,
 237         ACK_continue,
 238         ACK_common,
 239         ACK_ready
 240 };
 241
 242 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 243 {
 244         if (args->stateless_rpc && args->deepen) {
 245                 /* If we sent a depth we will get back "duplicate"
 246                  * shallow and unshallow commands every time there
 247                  * is a block of have lines exchanged.
 248                  */
 249                 char *line;
 250                 while ((line = packet_read_line(fd, NULL))) {
 251                         if (starts_with(line, "shallow "))
 252                                 continue;
 253                         if (starts_with(line, "unshallow "))
 254                                 continue;
 255                         die(_("git fetch-pack: expected shallow list"));
 256                 }
 257         }
 258 }
 259
 260 static enum ack_type get_ack(int fd, struct object_id *result_oid)
 261 {
 262         int len;
 263         char *line = packet_read_line(fd, &len);
 264         const char *arg;
 265
 266         if (!line)
 267                 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
 268         if (!strcmp(line, "NAK"))
 269                 return NAK;
 270         if (skip_prefix(line, "ACK ", &arg)) {
 271                 if (!get_oid_hex(arg, result_oid)) {
 272                         arg += 40;
 273                         len -= arg - line;
 274                         if (len < 1)
 275                                 return ACK;
 276                         if (strstr(arg, "continue"))
 277                                 return ACK_continue;
 278                         if (strstr(arg, "common"))
 279                                 return ACK_common;
 280                         if (strstr(arg, "ready"))
 281                                 return ACK_ready;
 282                         return ACK;
 283                 }
 284         }
 285         if (skip_prefix(line, "ERR ", &arg))
 286                 die(_("remote error: %s"), arg);
 287         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 288 }
 289
 290 static void send_request(struct fetch_pack_args *args,
 291                          int fd, struct strbuf *buf)
 292 {
 293         if (args->stateless_rpc) {
 294                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 295                 packet_flush(fd);
 296         } else
 297                 write_or_die(fd, buf->buf, buf->len);
 298 }
 299
 300 static void insert_one_alternate_object(struct object *obj)
 301 {
 302         rev_list_insert_ref(NULL, &obj->oid);
 303 }
 304
 305 #define INITIAL_FLUSH 16
 306 #define PIPESAFE_FLUSH 32
 307 #define LARGE_FLUSH 16384
 308
 309 static int next_flush(struct fetch_pack_args *args, int count)
 310 {
 311         if (args->stateless_rpc) {
 312                 if (count < LARGE_FLUSH)
 313                         count <<= 1;
 314                 else
 315                         count = count * 11 / 10;
 316         } else {
 317                 if (count < PIPESAFE_FLUSH)
 318                         count <<= 1;
 319                 else
 320                         count += PIPESAFE_FLUSH;
 321         }
 322         return count;
 323 }
 324
 325 static int find_common(struct fetch_pack_args *args,
 326                        int fd[2], struct object_id *result_oid,
 327                        struct ref *refs)
 328 {
 329         int fetching;
 330         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 331         const struct object_id *oid;
 332         unsigned in_vain = 0;
 333         int got_continue = 0;
 334         int got_ready = 0;
 335         struct strbuf req_buf = STRBUF_INIT;
 336         size_t state_len = 0;
 337
 338         if (args->stateless_rpc && multi_ack == 1)
 339                 die(_("--stateless-rpc requires multi_ack_detailed"));
 340         if (marked)
 341                 for_each_ref(clear_marks, NULL);
 342         marked = 1;
 343
 344         for_each_ref(rev_list_insert_ref_oid, NULL);
 345         for_each_cached_alternate(insert_one_alternate_object);
 346
 347         fetching = 0;
 348         for ( ; refs ; refs = refs->next) {
 349                 struct object_id *remote = &refs->old_oid;
 350                 const char *remote_hex;
 351                 struct object *o;
 352
 353                 /*
 354                  * If that object is complete (i.e. it is an ancestor of a
 355                  * local ref), we tell them we have it but do not have to
 356                  * tell them about its ancestors, which they already know
 357                  * about.
 358                  *
 359                  * We use lookup_object here because we are only
 360                  * interested in the case we *know* the object is
 361                  * reachable and we have already scanned it.
 362                  */
 363                 if (((o = lookup_object(remote->hash)) != NULL) &&
 364                                 (o->flags & COMPLETE)) {
 365                         continue;
 366                 }
 367
 368                 remote_hex = oid_to_hex(remote);
 369                 if (!fetching) {
 370                         struct strbuf c = STRBUF_INIT;
 371                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 372                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 373                         if (no_done)            strbuf_addstr(&c, " no-done");
 374                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 375                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 376                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 377                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 378                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 379                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 380                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 381                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 382                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 383                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 384                                                             git_user_agent_sanitized());
 385                         if (args->filter_options.choice)
 386                                 strbuf_addstr(&c, " filter");
 387                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 388                         strbuf_release(&c);
 389                 } else
 390                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 391                 fetching++;
 392         }
 393
 394         if (!fetching) {
 395                 strbuf_release(&req_buf);
 396                 packet_flush(fd[1]);
 397                 return 1;
 398         }
 399
 400         if (is_repository_shallow(the_repository))
 401                 write_shallow_commits(&req_buf, 1, NULL);
 402         if (args->depth > 0)
 403                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 404         if (args->deepen_since) {
 405                 timestamp_t max_age = approxidate(args->deepen_since);
 406                 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 407         }
 408         if (args->deepen_not) {
 409                 int i;
 410                 for (i = 0; i < args->deepen_not->nr; i++) {
 411                         struct string_list_item *s = args->deepen_not->items + i;
 412                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 413                 }
 414         }
 415         if (server_supports_filtering && args->filter_options.choice)
 416                 packet_buf_write(&req_buf, "filter %s",
 417                                  args->filter_options.filter_spec);
 418         packet_buf_flush(&req_buf);
 419         state_len = req_buf.len;
 420
 421         if (args->deepen) {
 422                 char *line;
 423                 const char *arg;
 424                 struct object_id oid;
 425
 426                 send_request(args, fd[1], &req_buf);
 427                 while ((line = packet_read_line(fd[0], NULL))) {
 428                         if (skip_prefix(line, "shallow ", &arg)) {
 429                                 if (get_oid_hex(arg, &oid))
 430                                         die(_("invalid shallow line: %s"), line);
 431                                 register_shallow(the_repository, &oid);
 432                                 continue;
 433                         }
 434                         if (skip_prefix(line, "unshallow ", &arg)) {
 435                                 if (get_oid_hex(arg, &oid))
 436                                         die(_("invalid unshallow line: %s"), line);
 437                                 if (!lookup_object(oid.hash))
 438                                         die(_("object not found: %s"), line);
 439                                 /* make sure that it is parsed as shallow */
 440                                 if (!parse_object(&oid))
 441                                         die(_("error in object: %s"), line);
 442                                 if (unregister_shallow(&oid))
 443                                         die(_("no shallow found: %s"), line);
 444                                 continue;
 445                         }
 446                         die(_("expected shallow/unshallow, got %s"), line);
 447                 }
 448         } else if (!args->stateless_rpc)
 449                 send_request(args, fd[1], &req_buf);
 450
 451         if (!args->stateless_rpc) {
 452                 /* If we aren't using the stateless-rpc interface
 453                  * we don't need to retain the headers.
 454                  */
 455                 strbuf_setlen(&req_buf, 0);
 456                 state_len = 0;
 457         }
 458
 459         flushes = 0;
 460         retval = -1;
 461         if (args->no_dependents)
 462                 goto done;
 463         while ((oid = get_rev())) {
 464                 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 465                 print_verbose(args, "have %s", oid_to_hex(oid));
 466                 in_vain++;
 467                 if (flush_at <= ++count) {
 468                         int ack;
 469
 470                         packet_buf_flush(&req_buf);
 471                         send_request(args, fd[1], &req_buf);
 472                         strbuf_setlen(&req_buf, state_len);
 473                         flushes++;
 474                         flush_at = next_flush(args, count);
 475
 476                         /*
 477                          * We keep one window "ahead" of the other side, and
 478                          * will wait for an ACK only on the next one
 479                          */
 480                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 481                                 continue;
 482
 483                         consume_shallow_list(args, fd[0]);
 484                         do {
 485                                 ack = get_ack(fd[0], result_oid);
 486                                 if (ack)
 487                                         print_verbose(args, _("got %s %d %s"), "ack",
 488                                                       ack, oid_to_hex(result_oid));
 489                                 switch (ack) {
 490                                 case ACK:
 491                                         flushes = 0;
 492                                         multi_ack = 0;
 493                                         retval = 0;
 494                                         goto done;
 495                                 case ACK_common:
 496                                 case ACK_ready:
 497                                 case ACK_continue: {
 498                                         struct commit *commit =
 499                                                 lookup_commit(result_oid);
 500                                         if (!commit)
 501                                                 die(_("invalid commit %s"), oid_to_hex(result_oid));
 502                                         if (args->stateless_rpc
 503                                          && ack == ACK_common
 504                                          && !(commit->object.flags & COMMON)) {
 505                                                 /* We need to replay the have for this object
 506                                                  * on the next RPC request so the peer knows
 507                                                  * it is in common with us.
 508                                                  */
 509                                                 const char *hex = oid_to_hex(result_oid);
 510                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 511                                                 state_len = req_buf.len;
 512                                                 /*
 513                                                  * Reset in_vain because an ack
 514                                                  * for this commit has not been
 515                                                  * seen.
 516                                                  */
 517                                                 in_vain = 0;
 518                                         } else if (!args->stateless_rpc
 519                                                    || ack != ACK_common)
 520                                                 in_vain = 0;
 521                                         mark_common(commit, 0, 1);
 522                                         retval = 0;
 523                                         got_continue = 1;
 524                                         if (ack == ACK_ready) {
 525                                                 clear_prio_queue(&rev_list);
 526                                                 got_ready = 1;
 527                                         }
 528                                         break;
 529                                         }
 530                                 }
 531                         } while (ack);
 532                         flushes--;
 533                         if (got_continue && MAX_IN_VAIN < in_vain) {
 534                                 print_verbose(args, _("giving up"));
 535                                 break; /* give up */
 536                         }
 537                 }
 538         }
 539 done:
 540         if (!got_ready || !no_done) {
 541                 packet_buf_write(&req_buf, "done\n");
 542                 send_request(args, fd[1], &req_buf);
 543         }
 544         print_verbose(args, _("done"));
 545         if (retval != 0) {
 546                 multi_ack = 0;
 547                 flushes++;
 548         }
 549         strbuf_release(&req_buf);
 550
 551         if (!got_ready || !no_done)
 552                 consume_shallow_list(args, fd[0]);
 553         while (flushes || multi_ack) {
 554                 int ack = get_ack(fd[0], result_oid);
 555                 if (ack) {
 556                         print_verbose(args, _("got %s (%d) %s"), "ack",
 557                                       ack, oid_to_hex(result_oid));
 558                         if (ack == ACK)
 559                                 return 0;
 560                         multi_ack = 1;
 561                         continue;
 562                 }
 563                 flushes--;
 564         }
 565         /* it is no error to fetch into a completely empty repo */
 566         return count ? retval : 0;
 567 }
 568
 569 static struct commit_list *complete;
 570
 571 static int mark_complete(const struct object_id *oid)
 572 {
 573         struct object *o = parse_object(oid);
 574
 575         while (o && o->type == OBJ_TAG) {
 576                 struct tag *t = (struct tag *) o;
 577                 if (!t->tagged)
 578                         break; /* broken repository */
 579                 o->flags |= COMPLETE;
 580                 o = parse_object(&t->tagged->oid);
 581         }
 582         if (o && o->type == OBJ_COMMIT) {
 583                 struct commit *commit = (struct commit *)o;
 584                 if (!(commit->object.flags & COMPLETE)) {
 585                         commit->object.flags |= COMPLETE;
 586                         commit_list_insert(commit, &complete);
 587                 }
 588         }
 589         return 0;
 590 }
 591
 592 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 593                              int flag, void *cb_data)
 594 {
 595         return mark_complete(oid);
 596 }
 597
 598 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 599                                          timestamp_t cutoff)
 600 {
 601         while (complete && cutoff <= complete->item->date) {
 602                 print_verbose(args, _("Marking %s as complete"),
 603                               oid_to_hex(&complete->item->object.oid));
 604                 pop_most_recent_commit(&complete, COMPLETE);
 605         }
 606 }
 607
 608 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 609 {
 610         for (; refs; refs = refs->next)
 611                 oidset_insert(oids, &refs->old_oid);
 612 }
 613
 614 static int tip_oids_contain(struct oidset *tip_oids,
 615                             struct ref *unmatched, struct ref *newlist,
 616                             const struct object_id *id)
 617 {
 618         /*
 619          * Note that this only looks at the ref lists the first time it's
 620          * called. This works out in filter_refs() because even though it may
 621          * add to "newlist" between calls, the additions will always be for
 622          * oids that are already in the set.
 623          */
 624         if (!tip_oids->map.map.tablesize) {
 625                 add_refs_to_oidset(tip_oids, unmatched);
 626                 add_refs_to_oidset(tip_oids, newlist);
 627         }
 628         return oidset_contains(tip_oids, id);
 629 }
 630
 631 static void filter_refs(struct fetch_pack_args *args,
 632                         struct ref **refs,
 633                         struct ref **sought, int nr_sought)
 634 {
 635         struct ref *newlist = NULL;
 636         struct ref **newtail = &newlist;
 637         struct ref *unmatched = NULL;
 638         struct ref *ref, *next;
 639         struct oidset tip_oids = OIDSET_INIT;
 640         int i;
 641
 642         i = 0;
 643         for (ref = *refs; ref; ref = next) {
 644                 int keep = 0;
 645                 next = ref->next;
 646
 647                 if (starts_with(ref->name, "refs/") &&
 648                     check_refname_format(ref->name, 0))
 649                         ; /* trash */
 650                 else {
 651                         while (i < nr_sought) {
 652                                 int cmp = strcmp(ref->name, sought[i]->name);
 653                                 if (cmp < 0)
 654                                         break; /* definitely do not have it */
 655                                 else if (cmp == 0) {
 656                                         keep = 1; /* definitely have it */
 657                                         sought[i]->match_status = REF_MATCHED;
 658                                 }
 659                                 i++;
 660                         }
 661                 }
 662
 663                 if (!keep && args->fetch_all &&
 664                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 665                         keep = 1;
 666
 667                 if (keep) {
 668                         *newtail = ref;
 669                         ref->next = NULL;
 670                         newtail = &ref->next;
 671                 } else {
 672                         ref->next = unmatched;
 673                         unmatched = ref;
 674                 }
 675         }
 676
 677         /* Append unmatched requests to the list */
 678         for (i = 0; i < nr_sought; i++) {
 679                 struct object_id oid;
 680                 const char *p;
 681
 682                 ref = sought[i];
 683                 if (ref->match_status != REF_NOT_MATCHED)
 684                         continue;
 685                 if (parse_oid_hex(ref->name, &oid, &p) ||
 686                     *p != '\0' ||
 687                     oidcmp(&oid, &ref->old_oid))
 688                         continue;
 689
 690                 if ((allow_unadvertised_object_request &
 691                      (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 692                     tip_oids_contain(&tip_oids, unmatched, newlist,
 693                                      &ref->old_oid)) {
 694                         ref->match_status = REF_MATCHED;
 695                         *newtail = copy_ref(ref);
 696                         newtail = &(*newtail)->next;
 697                 } else {
 698                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 699                 }
 700         }
 701
 702         oidset_clear(&tip_oids);
 703         for (ref = unmatched; ref; ref = next) {
 704                 next = ref->next;
 705                 free(ref);
 706         }
 707
 708         *refs = newlist;
 709 }
 710
 711 static void mark_alternate_complete(struct object *obj)
 712 {
 713         mark_complete(&obj->oid);
 714 }
 715
 716 struct loose_object_iter {
 717         struct oidset *loose_object_set;
 718         struct ref *refs;
 719 };
 720
 721 /*
 722  *  If the number of refs is not larger than the number of loose objects,
 723  *  this function stops inserting.
 724  */
 725 static int add_loose_objects_to_set(const struct object_id *oid,
 726                                     const char *path,
 727                                     void *data)
 728 {
 729         struct loose_object_iter *iter = data;
 730         oidset_insert(iter->loose_object_set, oid);
 731         if (iter->refs == NULL)
 732                 return 1;
 733
 734         iter->refs = iter->refs->next;
 735         return 0;
 736 }
 737
 738 static int everything_local(struct fetch_pack_args *args,
 739                             struct ref **refs,
 740                             struct ref **sought, int nr_sought)
 741 {
 742         struct ref *ref;
 743         int retval;
 744         int old_save_commit_buffer = save_commit_buffer;
 745         timestamp_t cutoff = 0;
 746         struct oidset loose_oid_set = OIDSET_INIT;
 747         int use_oidset = 0;
 748         struct loose_object_iter iter = {&loose_oid_set, *refs};
 749
 750         /* Enumerate all loose objects or know refs are not so many. */
 751         use_oidset = !for_each_loose_object(add_loose_objects_to_set,
 752                                             &iter, 0);
 753
 754         save_commit_buffer = 0;
 755
 756         for (ref = *refs; ref; ref = ref->next) {
 757                 struct object *o;
 758                 unsigned int flags = OBJECT_INFO_QUICK;
 759
 760                 if (use_oidset &&
 761                     !oidset_contains(&loose_oid_set, &ref->old_oid)) {
 762                         /*
 763                          * I know this does not exist in the loose form,
 764                          * so check if it exists in a non-loose form.
 765                          */
 766                         flags |= OBJECT_INFO_IGNORE_LOOSE;
 767                 }
 768
 769                 if (!has_object_file_with_flags(&ref->old_oid, flags))
 770                         continue;
 771                 o = parse_object(&ref->old_oid);
 772                 if (!o)
 773                         continue;
 774
 775                 /* We already have it -- which may mean that we were
 776                  * in sync with the other side at some time after
 777                  * that (it is OK if we guess wrong here).
 778                  */
 779                 if (o->type == OBJ_COMMIT) {
 780                         struct commit *commit = (struct commit *)o;
 781                         if (!cutoff || cutoff < commit->date)
 782                                 cutoff = commit->date;
 783                 }
 784         }
 785
 786         oidset_clear(&loose_oid_set);
 787
 788         if (!args->no_dependents) {
 789                 if (!args->deepen) {
 790                         for_each_ref(mark_complete_oid, NULL);
 791                         for_each_cached_alternate(mark_alternate_complete);
 792                         commit_list_sort_by_date(&complete);
 793                         if (cutoff)
 794                                 mark_recent_complete_commits(args, cutoff);
 795                 }
 796
 797                 /*
 798                  * Mark all complete remote refs as common refs.
 799                  * Don't mark them common yet; the server has to be told so first.
 800                  */
 801                 for (ref = *refs; ref; ref = ref->next) {
 802                         struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 803                                                      NULL, 0);
 804
 805                         if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 806                                 continue;
 807
 808                         if (!(o->flags & SEEN)) {
 809                                 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 810
 811                                 mark_common((struct commit *)o, 1, 1);
 812                         }
 813                 }
 814         }
 815
 816         filter_refs(args, refs, sought, nr_sought);
 817
 818         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 819                 const struct object_id *remote = &ref->old_oid;
 820                 struct object *o;
 821
 822                 o = lookup_object(remote->hash);
 823                 if (!o || !(o->flags & COMPLETE)) {
 824                         retval = 0;
 825                         print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 826                                       ref->name);
 827                         continue;
 828                 }
 829                 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 830                               ref->name);
 831         }
 832
 833         save_commit_buffer = old_save_commit_buffer;
 834
 835         return retval;
 836 }
 837
 838 static int sideband_demux(int in, int out, void *data)
 839 {
 840         int *xd = data;
 841         int ret;
 842
 843         ret = recv_sideband("fetch-pack", xd[0], out);
 844         close(out);
 845         return ret;
 846 }
 847
 848 static int get_pack(struct fetch_pack_args *args,
 849                     int xd[2], char **pack_lockfile)
 850 {
 851         struct async demux;
 852         int do_keep = args->keep_pack;
 853         const char *cmd_name;
 854         struct pack_header header;
 855         int pass_header = 0;
 856         struct child_process cmd = CHILD_PROCESS_INIT;
 857         int ret;
 858
 859         memset(&demux, 0, sizeof(demux));
 860         if (use_sideband) {
 861                 /* xd[] is talking with upload-pack; subprocess reads from
 862                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 863                  * through demux->out.
 864                  */
 865                 demux.proc = sideband_demux;
 866                 demux.data = xd;
 867                 demux.out = -1;
 868                 demux.isolate_sigpipe = 1;
 869                 if (start_async(&demux))
 870                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 871         }
 872         else
 873                 demux.out = xd[0];
 874
 875         if (!args->keep_pack && unpack_limit) {
 876
 877                 if (read_pack_header(demux.out, &header))
 878                         die(_("protocol error: bad pack header"));
 879                 pass_header = 1;
 880                 if (ntohl(header.hdr_entries) < unpack_limit)
 881                         do_keep = 0;
 882                 else
 883                         do_keep = 1;
 884         }
 885
 886         if (alternate_shallow_file) {
 887                 argv_array_push(&cmd.args, "--shallow-file");
 888                 argv_array_push(&cmd.args, alternate_shallow_file);
 889         }
 890
 891         if (do_keep || args->from_promisor) {
 892                 if (pack_lockfile)
 893                         cmd.out = -1;
 894                 cmd_name = "index-pack";
 895                 argv_array_push(&cmd.args, cmd_name);
 896                 argv_array_push(&cmd.args, "--stdin");
 897                 if (!args->quiet && !args->no_progress)
 898                         argv_array_push(&cmd.args, "-v");
 899                 if (args->use_thin_pack)
 900                         argv_array_push(&cmd.args, "--fix-thin");
 901                 if (do_keep && (args->lock_pack || unpack_limit)) {
 902                         char hostname[HOST_NAME_MAX + 1];
 903                         if (xgethostname(hostname, sizeof(hostname)))
 904                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 905                         argv_array_pushf(&cmd.args,
 906                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 907                                         (uintmax_t)getpid(), hostname);
 908                 }
 909                 if (args->check_self_contained_and_connected)
 910                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 911                 if (args->from_promisor)
 912                         argv_array_push(&cmd.args, "--promisor");
 913         }
 914         else {
 915                 cmd_name = "unpack-objects";
 916                 argv_array_push(&cmd.args, cmd_name);
 917                 if (args->quiet || args->no_progress)
 918                         argv_array_push(&cmd.args, "-q");
 919                 args->check_self_contained_and_connected = 0;
 920         }
 921
 922         if (pass_header)
 923                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 924                                  ntohl(header.hdr_version),
 925                                  ntohl(header.hdr_entries));
 926         if (fetch_fsck_objects >= 0
 927             ? fetch_fsck_objects
 928             : transfer_fsck_objects >= 0
 929             ? transfer_fsck_objects
 930             : 0) {
 931                 if (args->from_promisor)
 932                         /*
 933                          * We cannot use --strict in index-pack because it
 934                          * checks both broken objects and links, but we only
 935                          * want to check for broken objects.
 936                          */
 937                         argv_array_push(&cmd.args, "--fsck-objects");
 938                 else
 939                         argv_array_push(&cmd.args, "--strict");
 940         }
 941
 942         cmd.in = demux.out;
 943         cmd.git_cmd = 1;
 944         if (start_command(&cmd))
 945                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 946         if (do_keep && pack_lockfile) {
 947                 *pack_lockfile = index_pack_lockfile(cmd.out);
 948                 close(cmd.out);
 949         }
 950
 951         if (!use_sideband)
 952                 /* Closed by start_command() */
 953                 xd[0] = -1;
 954
 955         ret = finish_command(&cmd);
 956         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 957                 args->self_contained_and_connected =
 958                         args->check_self_contained_and_connected &&
 959                         ret == 0;
 960         else
 961                 die(_("%s failed"), cmd_name);
 962         if (use_sideband && finish_async(&demux))
 963                 die(_("error in sideband demultiplexer"));
 964         return 0;
 965 }
 966
 967 static int cmp_ref_by_name(const void *a_, const void *b_)
 968 {
 969         const struct ref *a = *((const struct ref **)a_);
 970         const struct ref *b = *((const struct ref **)b_);
 971         return strcmp(a->name, b->name);
 972 }
 973
 974 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 975                                  int fd[2],
 976                                  const struct ref *orig_ref,
 977                                  struct ref **sought, int nr_sought,
 978                                  struct shallow_info *si,
 979                                  char **pack_lockfile)
 980 {
 981         struct ref *ref = copy_ref_list(orig_ref);
 982         struct object_id oid;
 983         const char *agent_feature;
 984         int agent_len;
 985
 986         sort_ref_list(&ref, ref_compare_name);
 987         QSORT(sought, nr_sought, cmp_ref_by_name);
 988
 989         if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
 990                 die(_("Server does not support shallow clients"));
 991         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 992                 args->deepen = 1;
 993         if (server_supports("multi_ack_detailed")) {
 994                 print_verbose(args, _("Server supports multi_ack_detailed"));
 995                 multi_ack = 2;
 996                 if (server_supports("no-done")) {
 997                         print_verbose(args, _("Server supports no-done"));
 998                         if (args->stateless_rpc)
 999                                 no_done = 1;
1000                 }
1001         }
1002         else if (server_supports("multi_ack")) {
1003                 print_verbose(args, _("Server supports multi_ack"));
1004                 multi_ack = 1;
1005         }
1006         if (server_supports("side-band-64k")) {
1007                 print_verbose(args, _("Server supports side-band-64k"));
1008                 use_sideband = 2;
1009         }
1010         else if (server_supports("side-band")) {
1011                 print_verbose(args, _("Server supports side-band"));
1012                 use_sideband = 1;
1013         }
1014         if (server_supports("allow-tip-sha1-in-want")) {
1015                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
1016                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1017         }
1018         if (server_supports("allow-reachable-sha1-in-want")) {
1019                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
1020                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1021         }
1022         if (!server_supports("thin-pack"))
1023                 args->use_thin_pack = 0;
1024         if (!server_supports("no-progress"))
1025                 args->no_progress = 0;
1026         if (!server_supports("include-tag"))
1027                 args->include_tag = 0;
1028         if (server_supports("ofs-delta"))
1029                 print_verbose(args, _("Server supports ofs-delta"));
1030         else
1031                 prefer_ofs_delta = 0;
1032
1033         if (server_supports("filter")) {
1034                 server_supports_filtering = 1;
1035                 print_verbose(args, _("Server supports filter"));
1036         } else if (args->filter_options.choice) {
1037                 warning("filtering not recognized by server, ignoring");
1038         }
1039
1040         if ((agent_feature = server_feature_value("agent", &agent_len))) {
1041                 agent_supported = 1;
1042                 if (agent_len)
1043                         print_verbose(args, _("Server version is %.*s"),
1044                                       agent_len, agent_feature);
1045         }
1046         if (server_supports("deepen-since"))
1047                 deepen_since_ok = 1;
1048         else if (args->deepen_since)
1049                 die(_("Server does not support --shallow-since"));
1050         if (server_supports("deepen-not"))
1051                 deepen_not_ok = 1;
1052         else if (args->deepen_not)
1053                 die(_("Server does not support --shallow-exclude"));
1054         if (!server_supports("deepen-relative") && args->deepen_relative)
1055                 die(_("Server does not support --deepen"));
1056
1057         if (everything_local(args, &ref, sought, nr_sought)) {
1058                 packet_flush(fd[1]);
1059                 goto all_done;
1060         }
1061         if (find_common(args, fd, &oid, ref) < 0)
1062                 if (!args->keep_pack)
1063                         /* When cloning, it is not unusual to have
1064                          * no common commit.
1065                          */
1066                         warning(_("no common commits"));
1067
1068         if (args->stateless_rpc)
1069                 packet_flush(fd[1]);
1070         if (args->deepen)
1071                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1072                                         NULL);
1073         else if (si->nr_ours || si->nr_theirs)
1074                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1075         else
1076                 alternate_shallow_file = NULL;
1077         if (get_pack(args, fd, pack_lockfile))
1078                 die(_("git fetch-pack: fetch failed."));
1079
1080  all_done:
1081         return ref;
1082 }
1083
1084 static void fetch_pack_config(void)
1085 {
1086         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1087         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1088         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1089         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1090         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1091
1092         git_config(git_default_config, NULL);
1093 }
1094
1095 static void fetch_pack_setup(void)
1096 {
1097         static int did_setup;
1098         if (did_setup)
1099                 return;
1100         fetch_pack_config();
1101         if (0 <= transfer_unpack_limit)
1102                 unpack_limit = transfer_unpack_limit;
1103         else if (0 <= fetch_unpack_limit)
1104                 unpack_limit = fetch_unpack_limit;
1105         did_setup = 1;
1106 }
1107
1108 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1109 {
1110         struct string_list names = STRING_LIST_INIT_NODUP;
1111         int src, dst;
1112
1113         for (src = dst = 0; src < nr; src++) {
1114                 struct string_list_item *item;
1115                 item = string_list_insert(&names, ref[src]->name);
1116                 if (item->util)
1117                         continue; /* already have it */
1118                 item->util = ref[src];
1119                 if (src != dst)
1120                         ref[dst] = ref[src];
1121                 dst++;
1122         }
1123         for (src = dst; src < nr; src++)
1124                 ref[src] = NULL;
1125         string_list_clear(&names, 0);
1126         return dst;
1127 }
1128
1129 static void update_shallow(struct fetch_pack_args *args,
1130                            struct ref **sought, int nr_sought,
1131                            struct shallow_info *si)
1132 {
1133         struct oid_array ref = OID_ARRAY_INIT;
1134         int *status;
1135         int i;
1136
1137         if (args->deepen && alternate_shallow_file) {
1138                 if (*alternate_shallow_file == '\0') { /* --unshallow */
1139                         unlink_or_warn(git_path_shallow(the_repository));
1140                         rollback_lock_file(&shallow_lock);
1141                 } else
1142                         commit_lock_file(&shallow_lock);
1143                 return;
1144         }
1145
1146         if (!si->shallow || !si->shallow->nr)
1147                 return;
1148
1149         if (args->cloning) {
1150                 /*
1151                  * remote is shallow, but this is a clone, there are
1152                  * no objects in repo to worry about. Accept any
1153                  * shallow points that exist in the pack (iow in repo
1154                  * after get_pack() and reprepare_packed_git())
1155                  */
1156                 struct oid_array extra = OID_ARRAY_INIT;
1157                 struct object_id *oid = si->shallow->oid;
1158                 for (i = 0; i < si->shallow->nr; i++)
1159                         if (has_object_file(&oid[i]))
1160                                 oid_array_append(&extra, &oid[i]);
1161                 if (extra.nr) {
1162                         setup_alternate_shallow(&shallow_lock,
1163                                                 &alternate_shallow_file,
1164                                                 &extra);
1165                         commit_lock_file(&shallow_lock);
1166                 }
1167                 oid_array_clear(&extra);
1168                 return;
1169         }
1170
1171         if (!si->nr_ours && !si->nr_theirs)
1172                 return;
1173
1174         remove_nonexistent_theirs_shallow(si);
1175         if (!si->nr_ours && !si->nr_theirs)
1176                 return;
1177         for (i = 0; i < nr_sought; i++)
1178                 oid_array_append(&ref, &sought[i]->old_oid);
1179         si->ref = &ref;
1180
1181         if (args->update_shallow) {
1182                 /*
1183                  * remote is also shallow, .git/shallow may be updated
1184                  * so all refs can be accepted. Make sure we only add
1185                  * shallow roots that are actually reachable from new
1186                  * refs.
1187                  */
1188                 struct oid_array extra = OID_ARRAY_INIT;
1189                 struct object_id *oid = si->shallow->oid;
1190                 assign_shallow_commits_to_refs(si, NULL, NULL);
1191                 if (!si->nr_ours && !si->nr_theirs) {
1192                         oid_array_clear(&ref);
1193                         return;
1194                 }
1195                 for (i = 0; i < si->nr_ours; i++)
1196                         oid_array_append(&extra, &oid[si->ours[i]]);
1197                 for (i = 0; i < si->nr_theirs; i++)
1198                         oid_array_append(&extra, &oid[si->theirs[i]]);
1199                 setup_alternate_shallow(&shallow_lock,
1200                                         &alternate_shallow_file,
1201                                         &extra);
1202                 commit_lock_file(&shallow_lock);
1203                 oid_array_clear(&extra);
1204                 oid_array_clear(&ref);
1205                 return;
1206         }
1207
1208         /*
1209          * remote is also shallow, check what ref is safe to update
1210          * without updating .git/shallow
1211          */
1212         status = xcalloc(nr_sought, sizeof(*status));
1213         assign_shallow_commits_to_refs(si, NULL, status);
1214         if (si->nr_ours || si->nr_theirs) {
1215                 for (i = 0; i < nr_sought; i++)
1216                         if (status[i])
1217                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1218         }
1219         free(status);
1220         oid_array_clear(&ref);
1221 }
1222
1223 struct ref *fetch_pack(struct fetch_pack_args *args,
1224                        int fd[], struct child_process *conn,
1225                        const struct ref *ref,
1226                        const char *dest,
1227                        struct ref **sought, int nr_sought,
1228                        struct oid_array *shallow,
1229                        char **pack_lockfile)
1230 {
1231         struct ref *ref_cpy;
1232         struct shallow_info si;
1233
1234         fetch_pack_setup();
1235         if (nr_sought)
1236                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1237
1238         if (!ref) {
1239                 packet_flush(fd[1]);
1240                 die(_("no matching remote head"));
1241         }
1242         prepare_shallow_info(&si, shallow);
1243         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1244                                 &si, pack_lockfile);
1245         reprepare_packed_git(the_repository);
1246         update_shallow(args, sought, nr_sought, &si);
1247         clear_shallow_info(&si);
1248         return ref_cpy;
1249 }
1250
1251 int report_unmatched_refs(struct ref **sought, int nr_sought)
1252 {
1253         int i, ret = 0;
1254
1255         for (i = 0; i < nr_sought; i++) {
1256                 if (!sought[i])
1257                         continue;
1258                 switch (sought[i]->match_status) {
1259                 case REF_MATCHED:
1260                         continue;
1261                 case REF_NOT_MATCHED:
1262                         error(_("no such remote ref %s"), sought[i]->name);
1263                         break;
1264                 case REF_UNADVERTISED_NOT_ALLOWED:
1265                         error(_("Server does not allow request for unadvertised object %s"),
1266                               sought[i]->name);
1267                         break;
1268                 }
1269                 ret = 1;
1270         }
1271         return ret;
1272 }