git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int deepen_since_ok;
  25 static int deepen_not_ok;
  26 static int fetch_fsck_objects = -1;
  27 static int transfer_fsck_objects = -1;
  28 static int agent_supported;
  29 static struct lock_file shallow_lock;
  30 static const char *alternate_shallow_file;
  31
  32 /* Remember to update object flag allocation in object.h */
  33 #define COMPLETE        (1U << 0)
  34 #define COMMON          (1U << 1)
  35 #define COMMON_REF      (1U << 2)
  36 #define SEEN            (1U << 3)
  37 #define POPPED          (1U << 4)
  38 #define ALTERNATE       (1U << 5)
  39
  40 static int marked;
  41
  42 /*
  43  * After sending this many "have"s if we do not get any new ACK , we
  44  * give up traversing our history.
  45  */
  46 #define MAX_IN_VAIN 256
  47
  48 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  49 static int non_common_revs, multi_ack, use_sideband;
  50 /* Allow specifying sha1 if it is a ref tip. */
  51 #define ALLOW_TIP_SHA1  01
  52 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  53 #define ALLOW_REACHABLE_SHA1    02
  54 static unsigned int allow_unadvertised_object_request;
  55
  56 __attribute__((format (printf, 2, 3)))
  57 static inline void print_verbose(const struct fetch_pack_args *args,
  58                                  const char *fmt, ...)
  59 {
  60         va_list params;
  61
  62         if (!args->verbose)
  63                 return;
  64
  65         va_start(params, fmt);
  66         vfprintf(stderr, fmt, params);
  67         va_end(params);
  68         fputc('\n', stderr);
  69 }
  70
  71 struct alternate_object_cache {
  72         struct object **items;
  73         size_t nr, alloc;
  74 };
  75
  76 static void cache_one_alternate(const char *refname,
  77                                 const struct object_id *oid,
  78                                 void *vcache)
  79 {
  80         struct alternate_object_cache *cache = vcache;
  81         struct object *obj = parse_object(oid->hash);
  82
  83         if (!obj || (obj->flags & ALTERNATE))
  84                 return;
  85
  86         obj->flags |= ALTERNATE;
  87         ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  88         cache->items[cache->nr++] = obj;
  89 }
  90
  91 static void for_each_cached_alternate(void (*cb)(struct object *))
  92 {
  93         static int initialized;
  94         static struct alternate_object_cache cache;
  95         size_t i;
  96
  97         if (!initialized) {
  98                 for_each_alternate_ref(cache_one_alternate, &cache);
  99                 initialized = 1;
 100         }
 101
 102         for (i = 0; i < cache.nr; i++)
 103                 cb(cache.items[i]);
 104 }
 105
 106 static void rev_list_push(struct commit *commit, int mark)
 107 {
 108         if (!(commit->object.flags & mark)) {
 109                 commit->object.flags |= mark;
 110
 111                 if (parse_commit(commit))
 112                         return;
 113
 114                 prio_queue_put(&rev_list, commit);
 115
 116                 if (!(commit->object.flags & COMMON))
 117                         non_common_revs++;
 118         }
 119 }
 120
 121 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
 122 {
 123         struct object *o = deref_tag(parse_object(sha1), refname, 0);
 124
 125         if (o && o->type == OBJ_COMMIT)
 126                 rev_list_push((struct commit *)o, SEEN);
 127
 128         return 0;
 129 }
 130
 131 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 132                                    int flag, void *cb_data)
 133 {
 134         return rev_list_insert_ref(refname, oid->hash);
 135 }
 136
 137 static int clear_marks(const char *refname, const struct object_id *oid,
 138                        int flag, void *cb_data)
 139 {
 140         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
 141
 142         if (o && o->type == OBJ_COMMIT)
 143                 clear_commit_marks((struct commit *)o,
 144                                    COMMON | COMMON_REF | SEEN | POPPED);
 145         return 0;
 146 }
 147
 148 /*
 149    This function marks a rev and its ancestors as common.
 150    In some cases, it is desirable to mark only the ancestors (for example
 151    when only the server does not yet know that they are common).
 152 */
 153
 154 static void mark_common(struct commit *commit,
 155                 int ancestors_only, int dont_parse)
 156 {
 157         if (commit != NULL && !(commit->object.flags & COMMON)) {
 158                 struct object *o = (struct object *)commit;
 159
 160                 if (!ancestors_only)
 161                         o->flags |= COMMON;
 162
 163                 if (!(o->flags & SEEN))
 164                         rev_list_push(commit, SEEN);
 165                 else {
 166                         struct commit_list *parents;
 167
 168                         if (!ancestors_only && !(o->flags & POPPED))
 169                                 non_common_revs--;
 170                         if (!o->parsed && !dont_parse)
 171                                 if (parse_commit(commit))
 172                                         return;
 173
 174                         for (parents = commit->parents;
 175                                         parents;
 176                                         parents = parents->next)
 177                                 mark_common(parents->item, 0, dont_parse);
 178                 }
 179         }
 180 }
 181
 182 /*
 183   Get the next rev to send, ignoring the common.
 184 */
 185
 186 static const unsigned char *get_rev(void)
 187 {
 188         struct commit *commit = NULL;
 189
 190         while (commit == NULL) {
 191                 unsigned int mark;
 192                 struct commit_list *parents;
 193
 194                 if (rev_list.nr == 0 || non_common_revs == 0)
 195                         return NULL;
 196
 197                 commit = prio_queue_get(&rev_list);
 198                 parse_commit(commit);
 199                 parents = commit->parents;
 200
 201                 commit->object.flags |= POPPED;
 202                 if (!(commit->object.flags & COMMON))
 203                         non_common_revs--;
 204
 205                 if (commit->object.flags & COMMON) {
 206                         /* do not send "have", and ignore ancestors */
 207                         commit = NULL;
 208                         mark = COMMON | SEEN;
 209                 } else if (commit->object.flags & COMMON_REF)
 210                         /* send "have", and ignore ancestors */
 211                         mark = COMMON | SEEN;
 212                 else
 213                         /* send "have", also for its ancestors */
 214                         mark = SEEN;
 215
 216                 while (parents) {
 217                         if (!(parents->item->object.flags & SEEN))
 218                                 rev_list_push(parents->item, mark);
 219                         if (mark & COMMON)
 220                                 mark_common(parents->item, 1, 0);
 221                         parents = parents->next;
 222                 }
 223         }
 224
 225         return commit->object.oid.hash;
 226 }
 227
 228 enum ack_type {
 229         NAK = 0,
 230         ACK,
 231         ACK_continue,
 232         ACK_common,
 233         ACK_ready
 234 };
 235
 236 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 237 {
 238         if (args->stateless_rpc && args->deepen) {
 239                 /* If we sent a depth we will get back "duplicate"
 240                  * shallow and unshallow commands every time there
 241                  * is a block of have lines exchanged.
 242                  */
 243                 char *line;
 244                 while ((line = packet_read_line(fd, NULL))) {
 245                         if (starts_with(line, "shallow "))
 246                                 continue;
 247                         if (starts_with(line, "unshallow "))
 248                                 continue;
 249                         die(_("git fetch-pack: expected shallow list"));
 250                 }
 251         }
 252 }
 253
 254 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 255 {
 256         int len;
 257         char *line = packet_read_line(fd, &len);
 258         const char *arg;
 259
 260         if (!len)
 261                 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
 262         if (!strcmp(line, "NAK"))
 263                 return NAK;
 264         if (skip_prefix(line, "ACK ", &arg)) {
 265                 if (!get_sha1_hex(arg, result_sha1)) {
 266                         arg += 40;
 267                         len -= arg - line;
 268                         if (len < 1)
 269                                 return ACK;
 270                         if (strstr(arg, "continue"))
 271                                 return ACK_continue;
 272                         if (strstr(arg, "common"))
 273                                 return ACK_common;
 274                         if (strstr(arg, "ready"))
 275                                 return ACK_ready;
 276                         return ACK;
 277                 }
 278         }
 279         if (skip_prefix(line, "ERR ", &arg))
 280                 die(_("remote error: %s"), arg);
 281         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 282 }
 283
 284 static void send_request(struct fetch_pack_args *args,
 285                          int fd, struct strbuf *buf)
 286 {
 287         if (args->stateless_rpc) {
 288                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 289                 packet_flush(fd);
 290         } else
 291                 write_or_die(fd, buf->buf, buf->len);
 292 }
 293
 294 static void insert_one_alternate_object(struct object *obj)
 295 {
 296         rev_list_insert_ref(NULL, obj->oid.hash);
 297 }
 298
 299 #define INITIAL_FLUSH 16
 300 #define PIPESAFE_FLUSH 32
 301 #define LARGE_FLUSH 16384
 302
 303 static int next_flush(struct fetch_pack_args *args, int count)
 304 {
 305         if (args->stateless_rpc) {
 306                 if (count < LARGE_FLUSH)
 307                         count <<= 1;
 308                 else
 309                         count = count * 11 / 10;
 310         } else {
 311                 if (count < PIPESAFE_FLUSH)
 312                         count <<= 1;
 313                 else
 314                         count += PIPESAFE_FLUSH;
 315         }
 316         return count;
 317 }
 318
 319 static int find_common(struct fetch_pack_args *args,
 320                        int fd[2], unsigned char *result_sha1,
 321                        struct ref *refs)
 322 {
 323         int fetching;
 324         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 325         const unsigned char *sha1;
 326         unsigned in_vain = 0;
 327         int got_continue = 0;
 328         int got_ready = 0;
 329         struct strbuf req_buf = STRBUF_INIT;
 330         size_t state_len = 0;
 331
 332         if (args->stateless_rpc && multi_ack == 1)
 333                 die(_("--stateless-rpc requires multi_ack_detailed"));
 334         if (marked)
 335                 for_each_ref(clear_marks, NULL);
 336         marked = 1;
 337
 338         for_each_ref(rev_list_insert_ref_oid, NULL);
 339         for_each_cached_alternate(insert_one_alternate_object);
 340
 341         fetching = 0;
 342         for ( ; refs ; refs = refs->next) {
 343                 unsigned char *remote = refs->old_oid.hash;
 344                 const char *remote_hex;
 345                 struct object *o;
 346
 347                 /*
 348                  * If that object is complete (i.e. it is an ancestor of a
 349                  * local ref), we tell them we have it but do not have to
 350                  * tell them about its ancestors, which they already know
 351                  * about.
 352                  *
 353                  * We use lookup_object here because we are only
 354                  * interested in the case we *know* the object is
 355                  * reachable and we have already scanned it.
 356                  */
 357                 if (((o = lookup_object(remote)) != NULL) &&
 358                                 (o->flags & COMPLETE)) {
 359                         continue;
 360                 }
 361
 362                 remote_hex = sha1_to_hex(remote);
 363                 if (!fetching) {
 364                         struct strbuf c = STRBUF_INIT;
 365                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 366                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 367                         if (no_done)            strbuf_addstr(&c, " no-done");
 368                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 369                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 370                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 371                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 372                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 373                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 374                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 375                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 376                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 377                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 378                                                             git_user_agent_sanitized());
 379                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 380                         strbuf_release(&c);
 381                 } else
 382                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 383                 fetching++;
 384         }
 385
 386         if (!fetching) {
 387                 strbuf_release(&req_buf);
 388                 packet_flush(fd[1]);
 389                 return 1;
 390         }
 391
 392         if (is_repository_shallow())
 393                 write_shallow_commits(&req_buf, 1, NULL);
 394         if (args->depth > 0)
 395                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 396         if (args->deepen_since) {
 397                 unsigned long max_age = approxidate(args->deepen_since);
 398                 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
 399         }
 400         if (args->deepen_not) {
 401                 int i;
 402                 for (i = 0; i < args->deepen_not->nr; i++) {
 403                         struct string_list_item *s = args->deepen_not->items + i;
 404                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 405                 }
 406         }
 407         packet_buf_flush(&req_buf);
 408         state_len = req_buf.len;
 409
 410         if (args->deepen) {
 411                 char *line;
 412                 const char *arg;
 413                 unsigned char sha1[20];
 414
 415                 send_request(args, fd[1], &req_buf);
 416                 while ((line = packet_read_line(fd[0], NULL))) {
 417                         if (skip_prefix(line, "shallow ", &arg)) {
 418                                 if (get_sha1_hex(arg, sha1))
 419                                         die(_("invalid shallow line: %s"), line);
 420                                 register_shallow(sha1);
 421                                 continue;
 422                         }
 423                         if (skip_prefix(line, "unshallow ", &arg)) {
 424                                 if (get_sha1_hex(arg, sha1))
 425                                         die(_("invalid unshallow line: %s"), line);
 426                                 if (!lookup_object(sha1))
 427                                         die(_("object not found: %s"), line);
 428                                 /* make sure that it is parsed as shallow */
 429                                 if (!parse_object(sha1))
 430                                         die(_("error in object: %s"), line);
 431                                 if (unregister_shallow(sha1))
 432                                         die(_("no shallow found: %s"), line);
 433                                 continue;
 434                         }
 435                         die(_("expected shallow/unshallow, got %s"), line);
 436                 }
 437         } else if (!args->stateless_rpc)
 438                 send_request(args, fd[1], &req_buf);
 439
 440         if (!args->stateless_rpc) {
 441                 /* If we aren't using the stateless-rpc interface
 442                  * we don't need to retain the headers.
 443                  */
 444                 strbuf_setlen(&req_buf, 0);
 445                 state_len = 0;
 446         }
 447
 448         flushes = 0;
 449         retval = -1;
 450         while ((sha1 = get_rev())) {
 451                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 452                 print_verbose(args, "have %s", sha1_to_hex(sha1));
 453                 in_vain++;
 454                 if (flush_at <= ++count) {
 455                         int ack;
 456
 457                         packet_buf_flush(&req_buf);
 458                         send_request(args, fd[1], &req_buf);
 459                         strbuf_setlen(&req_buf, state_len);
 460                         flushes++;
 461                         flush_at = next_flush(args, count);
 462
 463                         /*
 464                          * We keep one window "ahead" of the other side, and
 465                          * will wait for an ACK only on the next one
 466                          */
 467                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 468                                 continue;
 469
 470                         consume_shallow_list(args, fd[0]);
 471                         do {
 472                                 ack = get_ack(fd[0], result_sha1);
 473                                 if (ack)
 474                                         print_verbose(args, _("got %s %d %s"), "ack",
 475                                                       ack, sha1_to_hex(result_sha1));
 476                                 switch (ack) {
 477                                 case ACK:
 478                                         flushes = 0;
 479                                         multi_ack = 0;
 480                                         retval = 0;
 481                                         goto done;
 482                                 case ACK_common:
 483                                 case ACK_ready:
 484                                 case ACK_continue: {
 485                                         struct commit *commit =
 486                                                 lookup_commit(result_sha1);
 487                                         if (!commit)
 488                                                 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
 489                                         if (args->stateless_rpc
 490                                          && ack == ACK_common
 491                                          && !(commit->object.flags & COMMON)) {
 492                                                 /* We need to replay the have for this object
 493                                                  * on the next RPC request so the peer knows
 494                                                  * it is in common with us.
 495                                                  */
 496                                                 const char *hex = sha1_to_hex(result_sha1);
 497                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 498                                                 state_len = req_buf.len;
 499                                                 /*
 500                                                  * Reset in_vain because an ack
 501                                                  * for this commit has not been
 502                                                  * seen.
 503                                                  */
 504                                                 in_vain = 0;
 505                                         } else if (!args->stateless_rpc
 506                                                    || ack != ACK_common)
 507                                                 in_vain = 0;
 508                                         mark_common(commit, 0, 1);
 509                                         retval = 0;
 510                                         got_continue = 1;
 511                                         if (ack == ACK_ready) {
 512                                                 clear_prio_queue(&rev_list);
 513                                                 got_ready = 1;
 514                                         }
 515                                         break;
 516                                         }
 517                                 }
 518                         } while (ack);
 519                         flushes--;
 520                         if (got_continue && MAX_IN_VAIN < in_vain) {
 521                                 print_verbose(args, _("giving up"));
 522                                 break; /* give up */
 523                         }
 524                 }
 525         }
 526 done:
 527         if (!got_ready || !no_done) {
 528                 packet_buf_write(&req_buf, "done\n");
 529                 send_request(args, fd[1], &req_buf);
 530         }
 531         print_verbose(args, _("done"));
 532         if (retval != 0) {
 533                 multi_ack = 0;
 534                 flushes++;
 535         }
 536         strbuf_release(&req_buf);
 537
 538         if (!got_ready || !no_done)
 539                 consume_shallow_list(args, fd[0]);
 540         while (flushes || multi_ack) {
 541                 int ack = get_ack(fd[0], result_sha1);
 542                 if (ack) {
 543                         print_verbose(args, _("got %s (%d) %s"), "ack",
 544                                       ack, sha1_to_hex(result_sha1));
 545                         if (ack == ACK)
 546                                 return 0;
 547                         multi_ack = 1;
 548                         continue;
 549                 }
 550                 flushes--;
 551         }
 552         /* it is no error to fetch into a completely empty repo */
 553         return count ? retval : 0;
 554 }
 555
 556 static struct commit_list *complete;
 557
 558 static int mark_complete(const unsigned char *sha1)
 559 {
 560         struct object *o = parse_object(sha1);
 561
 562         while (o && o->type == OBJ_TAG) {
 563                 struct tag *t = (struct tag *) o;
 564                 if (!t->tagged)
 565                         break; /* broken repository */
 566                 o->flags |= COMPLETE;
 567                 o = parse_object(t->tagged->oid.hash);
 568         }
 569         if (o && o->type == OBJ_COMMIT) {
 570                 struct commit *commit = (struct commit *)o;
 571                 if (!(commit->object.flags & COMPLETE)) {
 572                         commit->object.flags |= COMPLETE;
 573                         commit_list_insert(commit, &complete);
 574                 }
 575         }
 576         return 0;
 577 }
 578
 579 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 580                              int flag, void *cb_data)
 581 {
 582         return mark_complete(oid->hash);
 583 }
 584
 585 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 586                                          unsigned long cutoff)
 587 {
 588         while (complete && cutoff <= complete->item->date) {
 589                 print_verbose(args, _("Marking %s as complete"),
 590                               oid_to_hex(&complete->item->object.oid));
 591                 pop_most_recent_commit(&complete, COMPLETE);
 592         }
 593 }
 594
 595 static void filter_refs(struct fetch_pack_args *args,
 596                         struct ref **refs,
 597                         struct ref **sought, int nr_sought)
 598 {
 599         struct ref *newlist = NULL;
 600         struct ref **newtail = &newlist;
 601         struct ref *ref, *next;
 602         int i;
 603
 604         i = 0;
 605         for (ref = *refs; ref; ref = next) {
 606                 int keep = 0;
 607                 next = ref->next;
 608
 609                 if (starts_with(ref->name, "refs/") &&
 610                     check_refname_format(ref->name, 0))
 611                         ; /* trash */
 612                 else {
 613                         while (i < nr_sought) {
 614                                 int cmp = strcmp(ref->name, sought[i]->name);
 615                                 if (cmp < 0)
 616                                         break; /* definitely do not have it */
 617                                 else if (cmp == 0) {
 618                                         keep = 1; /* definitely have it */
 619                                         sought[i]->match_status = REF_MATCHED;
 620                                 }
 621                                 i++;
 622                         }
 623                 }
 624
 625                 if (!keep && args->fetch_all &&
 626                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 627                         keep = 1;
 628
 629                 if (keep) {
 630                         *newtail = ref;
 631                         ref->next = NULL;
 632                         newtail = &ref->next;
 633                 } else {
 634                         free(ref);
 635                 }
 636         }
 637
 638         /* Append unmatched requests to the list */
 639         for (i = 0; i < nr_sought; i++) {
 640                 unsigned char sha1[20];
 641
 642                 ref = sought[i];
 643                 if (ref->match_status != REF_NOT_MATCHED)
 644                         continue;
 645                 if (get_sha1_hex(ref->name, sha1) ||
 646                     ref->name[40] != '\0' ||
 647                     hashcmp(sha1, ref->old_oid.hash))
 648                         continue;
 649
 650                 if ((allow_unadvertised_object_request &
 651                     (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 652                         ref->match_status = REF_MATCHED;
 653                         *newtail = copy_ref(ref);
 654                         newtail = &(*newtail)->next;
 655                 } else {
 656                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 657                 }
 658         }
 659         *refs = newlist;
 660 }
 661
 662 static void mark_alternate_complete(struct object *obj)
 663 {
 664         mark_complete(obj->oid.hash);
 665 }
 666
 667 static int everything_local(struct fetch_pack_args *args,
 668                             struct ref **refs,
 669                             struct ref **sought, int nr_sought)
 670 {
 671         struct ref *ref;
 672         int retval;
 673         unsigned long cutoff = 0;
 674
 675         save_commit_buffer = 0;
 676
 677         for (ref = *refs; ref; ref = ref->next) {
 678                 struct object *o;
 679
 680                 if (!has_object_file(&ref->old_oid))
 681                         continue;
 682
 683                 o = parse_object(ref->old_oid.hash);
 684                 if (!o)
 685                         continue;
 686
 687                 /* We already have it -- which may mean that we were
 688                  * in sync with the other side at some time after
 689                  * that (it is OK if we guess wrong here).
 690                  */
 691                 if (o->type == OBJ_COMMIT) {
 692                         struct commit *commit = (struct commit *)o;
 693                         if (!cutoff || cutoff < commit->date)
 694                                 cutoff = commit->date;
 695                 }
 696         }
 697
 698         if (!args->deepen) {
 699                 for_each_ref(mark_complete_oid, NULL);
 700                 for_each_cached_alternate(mark_alternate_complete);
 701                 commit_list_sort_by_date(&complete);
 702                 if (cutoff)
 703                         mark_recent_complete_commits(args, cutoff);
 704         }
 705
 706         /*
 707          * Mark all complete remote refs as common refs.
 708          * Don't mark them common yet; the server has to be told so first.
 709          */
 710         for (ref = *refs; ref; ref = ref->next) {
 711                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 712                                              NULL, 0);
 713
 714                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 715                         continue;
 716
 717                 if (!(o->flags & SEEN)) {
 718                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 719
 720                         mark_common((struct commit *)o, 1, 1);
 721                 }
 722         }
 723
 724         filter_refs(args, refs, sought, nr_sought);
 725
 726         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 727                 const unsigned char *remote = ref->old_oid.hash;
 728                 struct object *o;
 729
 730                 o = lookup_object(remote);
 731                 if (!o || !(o->flags & COMPLETE)) {
 732                         retval = 0;
 733                         print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
 734                                       ref->name);
 735                         continue;
 736                 }
 737                 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
 738                               ref->name);
 739         }
 740         return retval;
 741 }
 742
 743 static int sideband_demux(int in, int out, void *data)
 744 {
 745         int *xd = data;
 746         int ret;
 747
 748         ret = recv_sideband("fetch-pack", xd[0], out);
 749         close(out);
 750         return ret;
 751 }
 752
 753 static int get_pack(struct fetch_pack_args *args,
 754                     int xd[2], char **pack_lockfile)
 755 {
 756         struct async demux;
 757         int do_keep = args->keep_pack;
 758         const char *cmd_name;
 759         struct pack_header header;
 760         int pass_header = 0;
 761         struct child_process cmd = CHILD_PROCESS_INIT;
 762         int ret;
 763
 764         memset(&demux, 0, sizeof(demux));
 765         if (use_sideband) {
 766                 /* xd[] is talking with upload-pack; subprocess reads from
 767                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 768                  * through demux->out.
 769                  */
 770                 demux.proc = sideband_demux;
 771                 demux.data = xd;
 772                 demux.out = -1;
 773                 demux.isolate_sigpipe = 1;
 774                 if (start_async(&demux))
 775                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 776         }
 777         else
 778                 demux.out = xd[0];
 779
 780         if (!args->keep_pack && unpack_limit) {
 781
 782                 if (read_pack_header(demux.out, &header))
 783                         die(_("protocol error: bad pack header"));
 784                 pass_header = 1;
 785                 if (ntohl(header.hdr_entries) < unpack_limit)
 786                         do_keep = 0;
 787                 else
 788                         do_keep = 1;
 789         }
 790
 791         if (alternate_shallow_file) {
 792                 argv_array_push(&cmd.args, "--shallow-file");
 793                 argv_array_push(&cmd.args, alternate_shallow_file);
 794         }
 795
 796         if (do_keep) {
 797                 if (pack_lockfile)
 798                         cmd.out = -1;
 799                 cmd_name = "index-pack";
 800                 argv_array_push(&cmd.args, cmd_name);
 801                 argv_array_push(&cmd.args, "--stdin");
 802                 if (!args->quiet && !args->no_progress)
 803                         argv_array_push(&cmd.args, "-v");
 804                 if (args->use_thin_pack)
 805                         argv_array_push(&cmd.args, "--fix-thin");
 806                 if (args->lock_pack || unpack_limit) {
 807                         char hostname[HOST_NAME_MAX + 1];
 808                         if (xgethostname(hostname, sizeof(hostname)))
 809                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 810                         argv_array_pushf(&cmd.args,
 811                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 812                                         (uintmax_t)getpid(), hostname);
 813                 }
 814                 if (args->check_self_contained_and_connected)
 815                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 816         }
 817         else {
 818                 cmd_name = "unpack-objects";
 819                 argv_array_push(&cmd.args, cmd_name);
 820                 if (args->quiet || args->no_progress)
 821                         argv_array_push(&cmd.args, "-q");
 822                 args->check_self_contained_and_connected = 0;
 823         }
 824
 825         if (pass_header)
 826                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 827                                  ntohl(header.hdr_version),
 828                                  ntohl(header.hdr_entries));
 829         if (fetch_fsck_objects >= 0
 830             ? fetch_fsck_objects
 831             : transfer_fsck_objects >= 0
 832             ? transfer_fsck_objects
 833             : 0)
 834                 argv_array_push(&cmd.args, "--strict");
 835
 836         cmd.in = demux.out;
 837         cmd.git_cmd = 1;
 838         if (start_command(&cmd))
 839                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 840         if (do_keep && pack_lockfile) {
 841                 *pack_lockfile = index_pack_lockfile(cmd.out);
 842                 close(cmd.out);
 843         }
 844
 845         if (!use_sideband)
 846                 /* Closed by start_command() */
 847                 xd[0] = -1;
 848
 849         ret = finish_command(&cmd);
 850         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 851                 args->self_contained_and_connected =
 852                         args->check_self_contained_and_connected &&
 853                         ret == 0;
 854         else
 855                 die(_("%s failed"), cmd_name);
 856         if (use_sideband && finish_async(&demux))
 857                 die(_("error in sideband demultiplexer"));
 858         return 0;
 859 }
 860
 861 static int cmp_ref_by_name(const void *a_, const void *b_)
 862 {
 863         const struct ref *a = *((const struct ref **)a_);
 864         const struct ref *b = *((const struct ref **)b_);
 865         return strcmp(a->name, b->name);
 866 }
 867
 868 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 869                                  int fd[2],
 870                                  const struct ref *orig_ref,
 871                                  struct ref **sought, int nr_sought,
 872                                  struct shallow_info *si,
 873                                  char **pack_lockfile)
 874 {
 875         struct ref *ref = copy_ref_list(orig_ref);
 876         unsigned char sha1[20];
 877         const char *agent_feature;
 878         int agent_len;
 879
 880         sort_ref_list(&ref, ref_compare_name);
 881         QSORT(sought, nr_sought, cmp_ref_by_name);
 882
 883         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 884                 die(_("Server does not support shallow clients"));
 885         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 886                 args->deepen = 1;
 887         if (server_supports("multi_ack_detailed")) {
 888                 print_verbose(args, _("Server supports multi_ack_detailed"));
 889                 multi_ack = 2;
 890                 if (server_supports("no-done")) {
 891                         print_verbose(args, _("Server supports no-done"));
 892                         if (args->stateless_rpc)
 893                                 no_done = 1;
 894                 }
 895         }
 896         else if (server_supports("multi_ack")) {
 897                 print_verbose(args, _("Server supports multi_ack"));
 898                 multi_ack = 1;
 899         }
 900         if (server_supports("side-band-64k")) {
 901                 print_verbose(args, _("Server supports side-band-64k"));
 902                 use_sideband = 2;
 903         }
 904         else if (server_supports("side-band")) {
 905                 print_verbose(args, _("Server supports side-band"));
 906                 use_sideband = 1;
 907         }
 908         if (server_supports("allow-tip-sha1-in-want")) {
 909                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 910                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 911         }
 912         if (server_supports("allow-reachable-sha1-in-want")) {
 913                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 914                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 915         }
 916         if (!server_supports("thin-pack"))
 917                 args->use_thin_pack = 0;
 918         if (!server_supports("no-progress"))
 919                 args->no_progress = 0;
 920         if (!server_supports("include-tag"))
 921                 args->include_tag = 0;
 922         if (server_supports("ofs-delta"))
 923                 print_verbose(args, _("Server supports ofs-delta"));
 924         else
 925                 prefer_ofs_delta = 0;
 926
 927         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 928                 agent_supported = 1;
 929                 if (agent_len)
 930                         print_verbose(args, _("Server version is %.*s"),
 931                                       agent_len, agent_feature);
 932         }
 933         if (server_supports("deepen-since"))
 934                 deepen_since_ok = 1;
 935         else if (args->deepen_since)
 936                 die(_("Server does not support --shallow-since"));
 937         if (server_supports("deepen-not"))
 938                 deepen_not_ok = 1;
 939         else if (args->deepen_not)
 940                 die(_("Server does not support --shallow-exclude"));
 941         if (!server_supports("deepen-relative") && args->deepen_relative)
 942                 die(_("Server does not support --deepen"));
 943
 944         if (everything_local(args, &ref, sought, nr_sought)) {
 945                 packet_flush(fd[1]);
 946                 goto all_done;
 947         }
 948         if (find_common(args, fd, sha1, ref) < 0)
 949                 if (!args->keep_pack)
 950                         /* When cloning, it is not unusual to have
 951                          * no common commit.
 952                          */
 953                         warning(_("no common commits"));
 954
 955         if (args->stateless_rpc)
 956                 packet_flush(fd[1]);
 957         if (args->deepen)
 958                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 959                                         NULL);
 960         else if (si->nr_ours || si->nr_theirs)
 961                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 962         else
 963                 alternate_shallow_file = NULL;
 964         if (get_pack(args, fd, pack_lockfile))
 965                 die(_("git fetch-pack: fetch failed."));
 966
 967  all_done:
 968         return ref;
 969 }
 970
 971 static void fetch_pack_config(void)
 972 {
 973         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 974         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 975         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 976         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 977         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 978
 979         git_config(git_default_config, NULL);
 980 }
 981
 982 static void fetch_pack_setup(void)
 983 {
 984         static int did_setup;
 985         if (did_setup)
 986                 return;
 987         fetch_pack_config();
 988         if (0 <= transfer_unpack_limit)
 989                 unpack_limit = transfer_unpack_limit;
 990         else if (0 <= fetch_unpack_limit)
 991                 unpack_limit = fetch_unpack_limit;
 992         did_setup = 1;
 993 }
 994
 995 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 996 {
 997         struct string_list names = STRING_LIST_INIT_NODUP;
 998         int src, dst;
 999
1000         for (src = dst = 0; src < nr; src++) {
1001                 struct string_list_item *item;
1002                 item = string_list_insert(&names, ref[src]->name);
1003                 if (item->util)
1004                         continue; /* already have it */
1005                 item->util = ref[src];
1006                 if (src != dst)
1007                         ref[dst] = ref[src];
1008                 dst++;
1009         }
1010         for (src = dst; src < nr; src++)
1011                 ref[src] = NULL;
1012         string_list_clear(&names, 0);
1013         return dst;
1014 }
1015
1016 static void update_shallow(struct fetch_pack_args *args,
1017                            struct ref **sought, int nr_sought,
1018                            struct shallow_info *si)
1019 {
1020         struct oid_array ref = OID_ARRAY_INIT;
1021         int *status;
1022         int i;
1023
1024         if (args->deepen && alternate_shallow_file) {
1025                 if (*alternate_shallow_file == '\0') { /* --unshallow */
1026                         unlink_or_warn(git_path_shallow());
1027                         rollback_lock_file(&shallow_lock);
1028                 } else
1029                         commit_lock_file(&shallow_lock);
1030                 return;
1031         }
1032
1033         if (!si->shallow || !si->shallow->nr)
1034                 return;
1035
1036         if (args->cloning) {
1037                 /*
1038                  * remote is shallow, but this is a clone, there are
1039                  * no objects in repo to worry about. Accept any
1040                  * shallow points that exist in the pack (iow in repo
1041                  * after get_pack() and reprepare_packed_git())
1042                  */
1043                 struct oid_array extra = OID_ARRAY_INIT;
1044                 struct object_id *oid = si->shallow->oid;
1045                 for (i = 0; i < si->shallow->nr; i++)
1046                         if (has_object_file(&oid[i]))
1047                                 oid_array_append(&extra, &oid[i]);
1048                 if (extra.nr) {
1049                         setup_alternate_shallow(&shallow_lock,
1050                                                 &alternate_shallow_file,
1051                                                 &extra);
1052                         commit_lock_file(&shallow_lock);
1053                 }
1054                 oid_array_clear(&extra);
1055                 return;
1056         }
1057
1058         if (!si->nr_ours && !si->nr_theirs)
1059                 return;
1060
1061         remove_nonexistent_theirs_shallow(si);
1062         if (!si->nr_ours && !si->nr_theirs)
1063                 return;
1064         for (i = 0; i < nr_sought; i++)
1065                 oid_array_append(&ref, &sought[i]->old_oid);
1066         si->ref = &ref;
1067
1068         if (args->update_shallow) {
1069                 /*
1070                  * remote is also shallow, .git/shallow may be updated
1071                  * so all refs can be accepted. Make sure we only add
1072                  * shallow roots that are actually reachable from new
1073                  * refs.
1074                  */
1075                 struct oid_array extra = OID_ARRAY_INIT;
1076                 struct object_id *oid = si->shallow->oid;
1077                 assign_shallow_commits_to_refs(si, NULL, NULL);
1078                 if (!si->nr_ours && !si->nr_theirs) {
1079                         oid_array_clear(&ref);
1080                         return;
1081                 }
1082                 for (i = 0; i < si->nr_ours; i++)
1083                         oid_array_append(&extra, &oid[si->ours[i]]);
1084                 for (i = 0; i < si->nr_theirs; i++)
1085                         oid_array_append(&extra, &oid[si->theirs[i]]);
1086                 setup_alternate_shallow(&shallow_lock,
1087                                         &alternate_shallow_file,
1088                                         &extra);
1089                 commit_lock_file(&shallow_lock);
1090                 oid_array_clear(&extra);
1091                 oid_array_clear(&ref);
1092                 return;
1093         }
1094
1095         /*
1096          * remote is also shallow, check what ref is safe to update
1097          * without updating .git/shallow
1098          */
1099         status = xcalloc(nr_sought, sizeof(*status));
1100         assign_shallow_commits_to_refs(si, NULL, status);
1101         if (si->nr_ours || si->nr_theirs) {
1102                 for (i = 0; i < nr_sought; i++)
1103                         if (status[i])
1104                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1105         }
1106         free(status);
1107         oid_array_clear(&ref);
1108 }
1109
1110 struct ref *fetch_pack(struct fetch_pack_args *args,
1111                        int fd[], struct child_process *conn,
1112                        const struct ref *ref,
1113                        const char *dest,
1114                        struct ref **sought, int nr_sought,
1115                        struct oid_array *shallow,
1116                        char **pack_lockfile)
1117 {
1118         struct ref *ref_cpy;
1119         struct shallow_info si;
1120
1121         fetch_pack_setup();
1122         if (nr_sought)
1123                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1124
1125         if (!ref) {
1126                 packet_flush(fd[1]);
1127                 die(_("no matching remote head"));
1128         }
1129         prepare_shallow_info(&si, shallow);
1130         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1131                                 &si, pack_lockfile);
1132         reprepare_packed_git();
1133         update_shallow(args, sought, nr_sought, &si);
1134         clear_shallow_info(&si);
1135         return ref_cpy;
1136 }
1137
1138 int report_unmatched_refs(struct ref **sought, int nr_sought)
1139 {
1140         int i, ret = 0;
1141
1142         for (i = 0; i < nr_sought; i++) {
1143                 if (!sought[i])
1144                         continue;
1145                 switch (sought[i]->match_status) {
1146                 case REF_MATCHED:
1147                         continue;
1148                 case REF_NOT_MATCHED:
1149                         error(_("no such remote ref %s"), sought[i]->name);
1150                         break;
1151                 case REF_UNADVERTISED_NOT_ALLOWED:
1152                         error(_("Server does not allow request for unadvertised object %s"),
1153                               sought[i]->name);
1154                         break;
1155                 }
1156                 ret = 1;
1157         }
1158         return ret;
1159 }