git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband;
  47 /* Allow specifying sha1 if it is a ref tip. */
  48 #define ALLOW_TIP_SHA1  01
  49 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  50 #define ALLOW_REACHABLE_SHA1    02
  51 static unsigned int allow_unadvertised_object_request;
  52
  53 static void rev_list_push(struct commit *commit, int mark)
  54 {
  55         if (!(commit->object.flags & mark)) {
  56                 commit->object.flags |= mark;
  57
  58                 if (parse_commit(commit))
  59                         return;
  60
  61                 prio_queue_put(&rev_list, commit);
  62
  63                 if (!(commit->object.flags & COMMON))
  64                         non_common_revs++;
  65         }
  66 }
  67
  68 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  69 {
  70         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  71
  72         if (o && o->type == OBJ_COMMIT)
  73                 rev_list_push((struct commit *)o, SEEN);
  74
  75         return 0;
  76 }
  77
  78 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  79 {
  80         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  81
  82         if (o && o->type == OBJ_COMMIT)
  83                 clear_commit_marks((struct commit *)o,
  84                                    COMMON | COMMON_REF | SEEN | POPPED);
  85         return 0;
  86 }
  87
  88 /*
  89    This function marks a rev and its ancestors as common.
  90    In some cases, it is desirable to mark only the ancestors (for example
  91    when only the server does not yet know that they are common).
  92 */
  93
  94 static void mark_common(struct commit *commit,
  95                 int ancestors_only, int dont_parse)
  96 {
  97         if (commit != NULL && !(commit->object.flags & COMMON)) {
  98                 struct object *o = (struct object *)commit;
  99
 100                 if (!ancestors_only)
 101                         o->flags |= COMMON;
 102
 103                 if (!(o->flags & SEEN))
 104                         rev_list_push(commit, SEEN);
 105                 else {
 106                         struct commit_list *parents;
 107
 108                         if (!ancestors_only && !(o->flags & POPPED))
 109                                 non_common_revs--;
 110                         if (!o->parsed && !dont_parse)
 111                                 if (parse_commit(commit))
 112                                         return;
 113
 114                         for (parents = commit->parents;
 115                                         parents;
 116                                         parents = parents->next)
 117                                 mark_common(parents->item, 0, dont_parse);
 118                 }
 119         }
 120 }
 121
 122 /*
 123   Get the next rev to send, ignoring the common.
 124 */
 125
 126 static const unsigned char *get_rev(void)
 127 {
 128         struct commit *commit = NULL;
 129
 130         while (commit == NULL) {
 131                 unsigned int mark;
 132                 struct commit_list *parents;
 133
 134                 if (rev_list.nr == 0 || non_common_revs == 0)
 135                         return NULL;
 136
 137                 commit = prio_queue_get(&rev_list);
 138                 parse_commit(commit);
 139                 parents = commit->parents;
 140
 141                 commit->object.flags |= POPPED;
 142                 if (!(commit->object.flags & COMMON))
 143                         non_common_revs--;
 144
 145                 if (commit->object.flags & COMMON) {
 146                         /* do not send "have", and ignore ancestors */
 147                         commit = NULL;
 148                         mark = COMMON | SEEN;
 149                 } else if (commit->object.flags & COMMON_REF)
 150                         /* send "have", and ignore ancestors */
 151                         mark = COMMON | SEEN;
 152                 else
 153                         /* send "have", also for its ancestors */
 154                         mark = SEEN;
 155
 156                 while (parents) {
 157                         if (!(parents->item->object.flags & SEEN))
 158                                 rev_list_push(parents->item, mark);
 159                         if (mark & COMMON)
 160                                 mark_common(parents->item, 1, 0);
 161                         parents = parents->next;
 162                 }
 163         }
 164
 165         return commit->object.sha1;
 166 }
 167
 168 enum ack_type {
 169         NAK = 0,
 170         ACK,
 171         ACK_continue,
 172         ACK_common,
 173         ACK_ready
 174 };
 175
 176 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 177 {
 178         if (args->stateless_rpc && args->depth > 0) {
 179                 /* If we sent a depth we will get back "duplicate"
 180                  * shallow and unshallow commands every time there
 181                  * is a block of have lines exchanged.
 182                  */
 183                 char *line;
 184                 while ((line = packet_read_line(fd, NULL))) {
 185                         if (starts_with(line, "shallow "))
 186                                 continue;
 187                         if (starts_with(line, "unshallow "))
 188                                 continue;
 189                         die("git fetch-pack: expected shallow list");
 190                 }
 191         }
 192 }
 193
 194 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 195 {
 196         int len;
 197         char *line = packet_read_line(fd, &len);
 198         const char *arg;
 199
 200         if (!len)
 201                 die("git fetch-pack: expected ACK/NAK, got EOF");
 202         if (!strcmp(line, "NAK"))
 203                 return NAK;
 204         if (skip_prefix(line, "ACK ", &arg)) {
 205                 if (!get_sha1_hex(arg, result_sha1)) {
 206                         arg += 40;
 207                         len -= arg - line;
 208                         if (len < 1)
 209                                 return ACK;
 210                         if (strstr(arg, "continue"))
 211                                 return ACK_continue;
 212                         if (strstr(arg, "common"))
 213                                 return ACK_common;
 214                         if (strstr(arg, "ready"))
 215                                 return ACK_ready;
 216                         return ACK;
 217                 }
 218         }
 219         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 220 }
 221
 222 static void send_request(struct fetch_pack_args *args,
 223                          int fd, struct strbuf *buf)
 224 {
 225         if (args->stateless_rpc) {
 226                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 227                 packet_flush(fd);
 228         } else
 229                 write_or_die(fd, buf->buf, buf->len);
 230 }
 231
 232 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 233 {
 234         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 235 }
 236
 237 #define INITIAL_FLUSH 16
 238 #define PIPESAFE_FLUSH 32
 239 #define LARGE_FLUSH 1024
 240
 241 static int next_flush(struct fetch_pack_args *args, int count)
 242 {
 243         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 244
 245         if (count < flush_limit)
 246                 count <<= 1;
 247         else
 248                 count += flush_limit;
 249         return count;
 250 }
 251
 252 static int find_common(struct fetch_pack_args *args,
 253                        int fd[2], unsigned char *result_sha1,
 254                        struct ref *refs)
 255 {
 256         int fetching;
 257         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 258         const unsigned char *sha1;
 259         unsigned in_vain = 0;
 260         int got_continue = 0;
 261         int got_ready = 0;
 262         struct strbuf req_buf = STRBUF_INIT;
 263         size_t state_len = 0;
 264
 265         if (args->stateless_rpc && multi_ack == 1)
 266                 die("--stateless-rpc requires multi_ack_detailed");
 267         if (marked)
 268                 for_each_ref(clear_marks, NULL);
 269         marked = 1;
 270
 271         for_each_ref(rev_list_insert_ref, NULL);
 272         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 273
 274         fetching = 0;
 275         for ( ; refs ; refs = refs->next) {
 276                 unsigned char *remote = refs->old_sha1;
 277                 const char *remote_hex;
 278                 struct object *o;
 279
 280                 /*
 281                  * If that object is complete (i.e. it is an ancestor of a
 282                  * local ref), we tell them we have it but do not have to
 283                  * tell them about its ancestors, which they already know
 284                  * about.
 285                  *
 286                  * We use lookup_object here because we are only
 287                  * interested in the case we *know* the object is
 288                  * reachable and we have already scanned it.
 289                  */
 290                 if (((o = lookup_object(remote)) != NULL) &&
 291                                 (o->flags & COMPLETE)) {
 292                         continue;
 293                 }
 294
 295                 remote_hex = sha1_to_hex(remote);
 296                 if (!fetching) {
 297                         struct strbuf c = STRBUF_INIT;
 298                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 299                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 300                         if (no_done)            strbuf_addstr(&c, " no-done");
 301                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 302                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 303                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 304                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 305                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 306                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 307                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 308                                                             git_user_agent_sanitized());
 309                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 310                         strbuf_release(&c);
 311                 } else
 312                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 313                 fetching++;
 314         }
 315
 316         if (!fetching) {
 317                 strbuf_release(&req_buf);
 318                 packet_flush(fd[1]);
 319                 return 1;
 320         }
 321
 322         if (is_repository_shallow())
 323                 write_shallow_commits(&req_buf, 1, NULL);
 324         if (args->depth > 0)
 325                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 326         packet_buf_flush(&req_buf);
 327         state_len = req_buf.len;
 328
 329         if (args->depth > 0) {
 330                 char *line;
 331                 const char *arg;
 332                 unsigned char sha1[20];
 333
 334                 send_request(args, fd[1], &req_buf);
 335                 while ((line = packet_read_line(fd[0], NULL))) {
 336                         if (skip_prefix(line, "shallow ", &arg)) {
 337                                 if (get_sha1_hex(arg, sha1))
 338                                         die("invalid shallow line: %s", line);
 339                                 register_shallow(sha1);
 340                                 continue;
 341                         }
 342                         if (skip_prefix(line, "unshallow ", &arg)) {
 343                                 if (get_sha1_hex(arg, sha1))
 344                                         die("invalid unshallow line: %s", line);
 345                                 if (!lookup_object(sha1))
 346                                         die("object not found: %s", line);
 347                                 /* make sure that it is parsed as shallow */
 348                                 if (!parse_object(sha1))
 349                                         die("error in object: %s", line);
 350                                 if (unregister_shallow(sha1))
 351                                         die("no shallow found: %s", line);
 352                                 continue;
 353                         }
 354                         die("expected shallow/unshallow, got %s", line);
 355                 }
 356         } else if (!args->stateless_rpc)
 357                 send_request(args, fd[1], &req_buf);
 358
 359         if (!args->stateless_rpc) {
 360                 /* If we aren't using the stateless-rpc interface
 361                  * we don't need to retain the headers.
 362                  */
 363                 strbuf_setlen(&req_buf, 0);
 364                 state_len = 0;
 365         }
 366
 367         flushes = 0;
 368         retval = -1;
 369         while ((sha1 = get_rev())) {
 370                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 371                 if (args->verbose)
 372                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 373                 in_vain++;
 374                 if (flush_at <= ++count) {
 375                         int ack;
 376
 377                         packet_buf_flush(&req_buf);
 378                         send_request(args, fd[1], &req_buf);
 379                         strbuf_setlen(&req_buf, state_len);
 380                         flushes++;
 381                         flush_at = next_flush(args, count);
 382
 383                         /*
 384                          * We keep one window "ahead" of the other side, and
 385                          * will wait for an ACK only on the next one
 386                          */
 387                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 388                                 continue;
 389
 390                         consume_shallow_list(args, fd[0]);
 391                         do {
 392                                 ack = get_ack(fd[0], result_sha1);
 393                                 if (args->verbose && ack)
 394                                         fprintf(stderr, "got ack %d %s\n", ack,
 395                                                         sha1_to_hex(result_sha1));
 396                                 switch (ack) {
 397                                 case ACK:
 398                                         flushes = 0;
 399                                         multi_ack = 0;
 400                                         retval = 0;
 401                                         goto done;
 402                                 case ACK_common:
 403                                 case ACK_ready:
 404                                 case ACK_continue: {
 405                                         struct commit *commit =
 406                                                 lookup_commit(result_sha1);
 407                                         if (!commit)
 408                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 409                                         if (args->stateless_rpc
 410                                          && ack == ACK_common
 411                                          && !(commit->object.flags & COMMON)) {
 412                                                 /* We need to replay the have for this object
 413                                                  * on the next RPC request so the peer knows
 414                                                  * it is in common with us.
 415                                                  */
 416                                                 const char *hex = sha1_to_hex(result_sha1);
 417                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 418                                                 state_len = req_buf.len;
 419                                         }
 420                                         mark_common(commit, 0, 1);
 421                                         retval = 0;
 422                                         in_vain = 0;
 423                                         got_continue = 1;
 424                                         if (ack == ACK_ready) {
 425                                                 clear_prio_queue(&rev_list);
 426                                                 got_ready = 1;
 427                                         }
 428                                         break;
 429                                         }
 430                                 }
 431                         } while (ack);
 432                         flushes--;
 433                         if (got_continue && MAX_IN_VAIN < in_vain) {
 434                                 if (args->verbose)
 435                                         fprintf(stderr, "giving up\n");
 436                                 break; /* give up */
 437                         }
 438                 }
 439         }
 440 done:
 441         if (!got_ready || !no_done) {
 442                 packet_buf_write(&req_buf, "done\n");
 443                 send_request(args, fd[1], &req_buf);
 444         }
 445         if (args->verbose)
 446                 fprintf(stderr, "done\n");
 447         if (retval != 0) {
 448                 multi_ack = 0;
 449                 flushes++;
 450         }
 451         strbuf_release(&req_buf);
 452
 453         if (!got_ready || !no_done)
 454                 consume_shallow_list(args, fd[0]);
 455         while (flushes || multi_ack) {
 456                 int ack = get_ack(fd[0], result_sha1);
 457                 if (ack) {
 458                         if (args->verbose)
 459                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 460                                         sha1_to_hex(result_sha1));
 461                         if (ack == ACK)
 462                                 return 0;
 463                         multi_ack = 1;
 464                         continue;
 465                 }
 466                 flushes--;
 467         }
 468         /* it is no error to fetch into a completely empty repo */
 469         return count ? retval : 0;
 470 }
 471
 472 static struct commit_list *complete;
 473
 474 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 475 {
 476         struct object *o = parse_object(sha1);
 477
 478         while (o && o->type == OBJ_TAG) {
 479                 struct tag *t = (struct tag *) o;
 480                 if (!t->tagged)
 481                         break; /* broken repository */
 482                 o->flags |= COMPLETE;
 483                 o = parse_object(t->tagged->sha1);
 484         }
 485         if (o && o->type == OBJ_COMMIT) {
 486                 struct commit *commit = (struct commit *)o;
 487                 if (!(commit->object.flags & COMPLETE)) {
 488                         commit->object.flags |= COMPLETE;
 489                         commit_list_insert(commit, &complete);
 490                 }
 491         }
 492         return 0;
 493 }
 494
 495 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 496                                          unsigned long cutoff)
 497 {
 498         while (complete && cutoff <= complete->item->date) {
 499                 if (args->verbose)
 500                         fprintf(stderr, "Marking %s as complete\n",
 501                                 sha1_to_hex(complete->item->object.sha1));
 502                 pop_most_recent_commit(&complete, COMPLETE);
 503         }
 504 }
 505
 506 static void filter_refs(struct fetch_pack_args *args,
 507                         struct ref **refs,
 508                         struct ref **sought, int nr_sought)
 509 {
 510         struct ref *newlist = NULL;
 511         struct ref **newtail = &newlist;
 512         struct ref *ref, *next;
 513         int i;
 514
 515         i = 0;
 516         for (ref = *refs; ref; ref = next) {
 517                 int keep = 0;
 518                 next = ref->next;
 519
 520                 if (starts_with(ref->name, "refs/") &&
 521                     check_refname_format(ref->name, 0))
 522                         ; /* trash */
 523                 else {
 524                         while (i < nr_sought) {
 525                                 int cmp = strcmp(ref->name, sought[i]->name);
 526                                 if (cmp < 0)
 527                                         break; /* definitely do not have it */
 528                                 else if (cmp == 0) {
 529                                         keep = 1; /* definitely have it */
 530                                         sought[i]->matched = 1;
 531                                 }
 532                                 i++;
 533                         }
 534                 }
 535
 536                 if (!keep && args->fetch_all &&
 537                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 538                         keep = 1;
 539
 540                 if (keep) {
 541                         *newtail = ref;
 542                         ref->next = NULL;
 543                         newtail = &ref->next;
 544                 } else {
 545                         free(ref);
 546                 }
 547         }
 548
 549         /* Append unmatched requests to the list */
 550         if ((allow_unadvertised_object_request &
 551             (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 552                 for (i = 0; i < nr_sought; i++) {
 553                         unsigned char sha1[20];
 554
 555                         ref = sought[i];
 556                         if (ref->matched)
 557                                 continue;
 558                         if (get_sha1_hex(ref->name, sha1) ||
 559                             ref->name[40] != '\0' ||
 560                             hashcmp(sha1, ref->old_sha1))
 561                                 continue;
 562
 563                         ref->matched = 1;
 564                         *newtail = copy_ref(ref);
 565                         newtail = &(*newtail)->next;
 566                 }
 567         }
 568         *refs = newlist;
 569 }
 570
 571 static void mark_alternate_complete(const struct ref *ref, void *unused)
 572 {
 573         mark_complete(NULL, ref->old_sha1, 0, NULL);
 574 }
 575
 576 static int everything_local(struct fetch_pack_args *args,
 577                             struct ref **refs,
 578                             struct ref **sought, int nr_sought)
 579 {
 580         struct ref *ref;
 581         int retval;
 582         unsigned long cutoff = 0;
 583
 584         save_commit_buffer = 0;
 585
 586         for (ref = *refs; ref; ref = ref->next) {
 587                 struct object *o;
 588
 589                 if (!has_sha1_file(ref->old_sha1))
 590                         continue;
 591
 592                 o = parse_object(ref->old_sha1);
 593                 if (!o)
 594                         continue;
 595
 596                 /* We already have it -- which may mean that we were
 597                  * in sync with the other side at some time after
 598                  * that (it is OK if we guess wrong here).
 599                  */
 600                 if (o->type == OBJ_COMMIT) {
 601                         struct commit *commit = (struct commit *)o;
 602                         if (!cutoff || cutoff < commit->date)
 603                                 cutoff = commit->date;
 604                 }
 605         }
 606
 607         if (!args->depth) {
 608                 for_each_ref(mark_complete, NULL);
 609                 for_each_alternate_ref(mark_alternate_complete, NULL);
 610                 commit_list_sort_by_date(&complete);
 611                 if (cutoff)
 612                         mark_recent_complete_commits(args, cutoff);
 613         }
 614
 615         /*
 616          * Mark all complete remote refs as common refs.
 617          * Don't mark them common yet; the server has to be told so first.
 618          */
 619         for (ref = *refs; ref; ref = ref->next) {
 620                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 621                                              NULL, 0);
 622
 623                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 624                         continue;
 625
 626                 if (!(o->flags & SEEN)) {
 627                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 628
 629                         mark_common((struct commit *)o, 1, 1);
 630                 }
 631         }
 632
 633         filter_refs(args, refs, sought, nr_sought);
 634
 635         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 636                 const unsigned char *remote = ref->old_sha1;
 637                 struct object *o;
 638
 639                 o = lookup_object(remote);
 640                 if (!o || !(o->flags & COMPLETE)) {
 641                         retval = 0;
 642                         if (!args->verbose)
 643                                 continue;
 644                         fprintf(stderr,
 645                                 "want %s (%s)\n", sha1_to_hex(remote),
 646                                 ref->name);
 647                         continue;
 648                 }
 649                 if (!args->verbose)
 650                         continue;
 651                 fprintf(stderr,
 652                         "already have %s (%s)\n", sha1_to_hex(remote),
 653                         ref->name);
 654         }
 655         return retval;
 656 }
 657
 658 static int sideband_demux(int in, int out, void *data)
 659 {
 660         int *xd = data;
 661
 662         int ret = recv_sideband("fetch-pack", xd[0], out);
 663         close(out);
 664         return ret;
 665 }
 666
 667 static int get_pack(struct fetch_pack_args *args,
 668                     int xd[2], char **pack_lockfile)
 669 {
 670         struct async demux;
 671         const char *argv[22];
 672         char keep_arg[256];
 673         char hdr_arg[256];
 674         const char **av, *cmd_name;
 675         int do_keep = args->keep_pack;
 676         struct child_process cmd = CHILD_PROCESS_INIT;
 677         int ret;
 678
 679         memset(&demux, 0, sizeof(demux));
 680         if (use_sideband) {
 681                 /* xd[] is talking with upload-pack; subprocess reads from
 682                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 683                  * through demux->out.
 684                  */
 685                 demux.proc = sideband_demux;
 686                 demux.data = xd;
 687                 demux.out = -1;
 688                 if (start_async(&demux))
 689                         die("fetch-pack: unable to fork off sideband"
 690                             " demultiplexer");
 691         }
 692         else
 693                 demux.out = xd[0];
 694
 695         cmd.argv = argv;
 696         av = argv;
 697         *hdr_arg = 0;
 698         if (!args->keep_pack && unpack_limit) {
 699                 struct pack_header header;
 700
 701                 if (read_pack_header(demux.out, &header))
 702                         die("protocol error: bad pack header");
 703                 snprintf(hdr_arg, sizeof(hdr_arg),
 704                          "--pack_header=%"PRIu32",%"PRIu32,
 705                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 706                 if (ntohl(header.hdr_entries) < unpack_limit)
 707                         do_keep = 0;
 708                 else
 709                         do_keep = 1;
 710         }
 711
 712         if (alternate_shallow_file) {
 713                 *av++ = "--shallow-file";
 714                 *av++ = alternate_shallow_file;
 715         }
 716
 717         if (do_keep) {
 718                 if (pack_lockfile)
 719                         cmd.out = -1;
 720                 *av++ = cmd_name = "index-pack";
 721                 *av++ = "--stdin";
 722                 if (!args->quiet && !args->no_progress)
 723                         *av++ = "-v";
 724                 if (args->use_thin_pack)
 725                         *av++ = "--fix-thin";
 726                 if (args->lock_pack || unpack_limit) {
 727                         int s = sprintf(keep_arg,
 728                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 729                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 730                                 strcpy(keep_arg + s, "localhost");
 731                         *av++ = keep_arg;
 732                 }
 733                 if (args->check_self_contained_and_connected)
 734                         *av++ = "--check-self-contained-and-connected";
 735         }
 736         else {
 737                 *av++ = cmd_name = "unpack-objects";
 738                 if (args->quiet || args->no_progress)
 739                         *av++ = "-q";
 740                 args->check_self_contained_and_connected = 0;
 741         }
 742         if (*hdr_arg)
 743                 *av++ = hdr_arg;
 744         if (fetch_fsck_objects >= 0
 745             ? fetch_fsck_objects
 746             : transfer_fsck_objects >= 0
 747             ? transfer_fsck_objects
 748             : 0)
 749                 *av++ = "--strict";
 750         *av++ = NULL;
 751
 752         cmd.in = demux.out;
 753         cmd.git_cmd = 1;
 754         if (start_command(&cmd))
 755                 die("fetch-pack: unable to fork off %s", cmd_name);
 756         if (do_keep && pack_lockfile) {
 757                 *pack_lockfile = index_pack_lockfile(cmd.out);
 758                 close(cmd.out);
 759         }
 760
 761         if (!use_sideband)
 762                 /* Closed by start_command() */
 763                 xd[0] = -1;
 764
 765         ret = finish_command(&cmd);
 766         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 767                 args->self_contained_and_connected =
 768                         args->check_self_contained_and_connected &&
 769                         ret == 0;
 770         else
 771                 die("%s failed", cmd_name);
 772         if (use_sideband && finish_async(&demux))
 773                 die("error in sideband demultiplexer");
 774         return 0;
 775 }
 776
 777 static int cmp_ref_by_name(const void *a_, const void *b_)
 778 {
 779         const struct ref *a = *((const struct ref **)a_);
 780         const struct ref *b = *((const struct ref **)b_);
 781         return strcmp(a->name, b->name);
 782 }
 783
 784 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 785                                  int fd[2],
 786                                  const struct ref *orig_ref,
 787                                  struct ref **sought, int nr_sought,
 788                                  struct shallow_info *si,
 789                                  char **pack_lockfile)
 790 {
 791         struct ref *ref = copy_ref_list(orig_ref);
 792         unsigned char sha1[20];
 793         const char *agent_feature;
 794         int agent_len;
 795
 796         sort_ref_list(&ref, ref_compare_name);
 797         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 798
 799         if (is_repository_shallow() && !server_supports("shallow"))
 800                 die("Server does not support shallow clients");
 801         if (server_supports("multi_ack_detailed")) {
 802                 if (args->verbose)
 803                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 804                 multi_ack = 2;
 805                 if (server_supports("no-done")) {
 806                         if (args->verbose)
 807                                 fprintf(stderr, "Server supports no-done\n");
 808                         if (args->stateless_rpc)
 809                                 no_done = 1;
 810                 }
 811         }
 812         else if (server_supports("multi_ack")) {
 813                 if (args->verbose)
 814                         fprintf(stderr, "Server supports multi_ack\n");
 815                 multi_ack = 1;
 816         }
 817         if (server_supports("side-band-64k")) {
 818                 if (args->verbose)
 819                         fprintf(stderr, "Server supports side-band-64k\n");
 820                 use_sideband = 2;
 821         }
 822         else if (server_supports("side-band")) {
 823                 if (args->verbose)
 824                         fprintf(stderr, "Server supports side-band\n");
 825                 use_sideband = 1;
 826         }
 827         if (server_supports("allow-tip-sha1-in-want")) {
 828                 if (args->verbose)
 829                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 830                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 831         }
 832         if (server_supports("allow-reachable-sha1-in-want")) {
 833                 if (args->verbose)
 834                         fprintf(stderr, "Server supports allow-reachable-sha1-in-want\n");
 835                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 836         }
 837         if (!server_supports("thin-pack"))
 838                 args->use_thin_pack = 0;
 839         if (!server_supports("no-progress"))
 840                 args->no_progress = 0;
 841         if (!server_supports("include-tag"))
 842                 args->include_tag = 0;
 843         if (server_supports("ofs-delta")) {
 844                 if (args->verbose)
 845                         fprintf(stderr, "Server supports ofs-delta\n");
 846         } else
 847                 prefer_ofs_delta = 0;
 848
 849         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 850                 agent_supported = 1;
 851                 if (args->verbose && agent_len)
 852                         fprintf(stderr, "Server version is %.*s\n",
 853                                 agent_len, agent_feature);
 854         }
 855
 856         if (everything_local(args, &ref, sought, nr_sought)) {
 857                 packet_flush(fd[1]);
 858                 goto all_done;
 859         }
 860         if (find_common(args, fd, sha1, ref) < 0)
 861                 if (!args->keep_pack)
 862                         /* When cloning, it is not unusual to have
 863                          * no common commit.
 864                          */
 865                         warning("no common commits");
 866
 867         if (args->stateless_rpc)
 868                 packet_flush(fd[1]);
 869         if (args->depth > 0)
 870                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 871                                         NULL);
 872         else if (si->nr_ours || si->nr_theirs)
 873                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 874         else
 875                 alternate_shallow_file = NULL;
 876         if (get_pack(args, fd, pack_lockfile))
 877                 die("git fetch-pack: fetch failed.");
 878
 879  all_done:
 880         return ref;
 881 }
 882
 883 static void fetch_pack_config(void)
 884 {
 885         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 886         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 887         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 888         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 889         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 890
 891         git_config(git_default_config, NULL);
 892 }
 893
 894 static void fetch_pack_setup(void)
 895 {
 896         static int did_setup;
 897         if (did_setup)
 898                 return;
 899         fetch_pack_config();
 900         if (0 <= transfer_unpack_limit)
 901                 unpack_limit = transfer_unpack_limit;
 902         else if (0 <= fetch_unpack_limit)
 903                 unpack_limit = fetch_unpack_limit;
 904         did_setup = 1;
 905 }
 906
 907 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 908 {
 909         struct string_list names = STRING_LIST_INIT_NODUP;
 910         int src, dst;
 911
 912         for (src = dst = 0; src < nr; src++) {
 913                 struct string_list_item *item;
 914                 item = string_list_insert(&names, ref[src]->name);
 915                 if (item->util)
 916                         continue; /* already have it */
 917                 item->util = ref[src];
 918                 if (src != dst)
 919                         ref[dst] = ref[src];
 920                 dst++;
 921         }
 922         for (src = dst; src < nr; src++)
 923                 ref[src] = NULL;
 924         string_list_clear(&names, 0);
 925         return dst;
 926 }
 927
 928 static void update_shallow(struct fetch_pack_args *args,
 929                            struct ref **sought, int nr_sought,
 930                            struct shallow_info *si)
 931 {
 932         struct sha1_array ref = SHA1_ARRAY_INIT;
 933         int *status;
 934         int i;
 935
 936         if (args->depth > 0 && alternate_shallow_file) {
 937                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 938                         unlink_or_warn(git_path("shallow"));
 939                         rollback_lock_file(&shallow_lock);
 940                 } else
 941                         commit_lock_file(&shallow_lock);
 942                 return;
 943         }
 944
 945         if (!si->shallow || !si->shallow->nr)
 946                 return;
 947
 948         if (args->cloning) {
 949                 /*
 950                  * remote is shallow, but this is a clone, there are
 951                  * no objects in repo to worry about. Accept any
 952                  * shallow points that exist in the pack (iow in repo
 953                  * after get_pack() and reprepare_packed_git())
 954                  */
 955                 struct sha1_array extra = SHA1_ARRAY_INIT;
 956                 unsigned char (*sha1)[20] = si->shallow->sha1;
 957                 for (i = 0; i < si->shallow->nr; i++)
 958                         if (has_sha1_file(sha1[i]))
 959                                 sha1_array_append(&extra, sha1[i]);
 960                 if (extra.nr) {
 961                         setup_alternate_shallow(&shallow_lock,
 962                                                 &alternate_shallow_file,
 963                                                 &extra);
 964                         commit_lock_file(&shallow_lock);
 965                 }
 966                 sha1_array_clear(&extra);
 967                 return;
 968         }
 969
 970         if (!si->nr_ours && !si->nr_theirs)
 971                 return;
 972
 973         remove_nonexistent_theirs_shallow(si);
 974         if (!si->nr_ours && !si->nr_theirs)
 975                 return;
 976         for (i = 0; i < nr_sought; i++)
 977                 sha1_array_append(&ref, sought[i]->old_sha1);
 978         si->ref = &ref;
 979
 980         if (args->update_shallow) {
 981                 /*
 982                  * remote is also shallow, .git/shallow may be updated
 983                  * so all refs can be accepted. Make sure we only add
 984                  * shallow roots that are actually reachable from new
 985                  * refs.
 986                  */
 987                 struct sha1_array extra = SHA1_ARRAY_INIT;
 988                 unsigned char (*sha1)[20] = si->shallow->sha1;
 989                 assign_shallow_commits_to_refs(si, NULL, NULL);
 990                 if (!si->nr_ours && !si->nr_theirs) {
 991                         sha1_array_clear(&ref);
 992                         return;
 993                 }
 994                 for (i = 0; i < si->nr_ours; i++)
 995                         sha1_array_append(&extra, sha1[si->ours[i]]);
 996                 for (i = 0; i < si->nr_theirs; i++)
 997                         sha1_array_append(&extra, sha1[si->theirs[i]]);
 998                 setup_alternate_shallow(&shallow_lock,
 999                                         &alternate_shallow_file,
1000                                         &extra);
1001                 commit_lock_file(&shallow_lock);
1002                 sha1_array_clear(&extra);
1003                 sha1_array_clear(&ref);
1004                 return;
1005         }
1006
1007         /*
1008          * remote is also shallow, check what ref is safe to update
1009          * without updating .git/shallow
1010          */
1011         status = xcalloc(nr_sought, sizeof(*status));
1012         assign_shallow_commits_to_refs(si, NULL, status);
1013         if (si->nr_ours || si->nr_theirs) {
1014                 for (i = 0; i < nr_sought; i++)
1015                         if (status[i])
1016                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1017         }
1018         free(status);
1019         sha1_array_clear(&ref);
1020 }
1021
1022 struct ref *fetch_pack(struct fetch_pack_args *args,
1023                        int fd[], struct child_process *conn,
1024                        const struct ref *ref,
1025                        const char *dest,
1026                        struct ref **sought, int nr_sought,
1027                        struct sha1_array *shallow,
1028                        char **pack_lockfile)
1029 {
1030         struct ref *ref_cpy;
1031         struct shallow_info si;
1032
1033         fetch_pack_setup();
1034         if (nr_sought)
1035                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1036
1037         if (!ref) {
1038                 packet_flush(fd[1]);
1039                 die("no matching remote head");
1040         }
1041         prepare_shallow_info(&si, shallow);
1042         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1043                                 &si, pack_lockfile);
1044         reprepare_packed_git();
1045         update_shallow(args, sought, nr_sought, &si);
1046         clear_shallow_info(&si);
1047         return ref_cpy;
1048 }