git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband;
  47 /* Allow specifying sha1 if it is a ref tip. */
  48 #define ALLOW_TIP_SHA1  01
  49 static unsigned int allow_unadvertised_object_request;
  50
  51 static void rev_list_push(struct commit *commit, int mark)
  52 {
  53         if (!(commit->object.flags & mark)) {
  54                 commit->object.flags |= mark;
  55
  56                 if (parse_commit(commit))
  57                         return;
  58
  59                 prio_queue_put(&rev_list, commit);
  60
  61                 if (!(commit->object.flags & COMMON))
  62                         non_common_revs++;
  63         }
  64 }
  65
  66 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  67 {
  68         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  69
  70         if (o && o->type == OBJ_COMMIT)
  71                 rev_list_push((struct commit *)o, SEEN);
  72
  73         return 0;
  74 }
  75
  76 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  77 {
  78         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  79
  80         if (o && o->type == OBJ_COMMIT)
  81                 clear_commit_marks((struct commit *)o,
  82                                    COMMON | COMMON_REF | SEEN | POPPED);
  83         return 0;
  84 }
  85
  86 /*
  87    This function marks a rev and its ancestors as common.
  88    In some cases, it is desirable to mark only the ancestors (for example
  89    when only the server does not yet know that they are common).
  90 */
  91
  92 static void mark_common(struct commit *commit,
  93                 int ancestors_only, int dont_parse)
  94 {
  95         if (commit != NULL && !(commit->object.flags & COMMON)) {
  96                 struct object *o = (struct object *)commit;
  97
  98                 if (!ancestors_only)
  99                         o->flags |= COMMON;
 100
 101                 if (!(o->flags & SEEN))
 102                         rev_list_push(commit, SEEN);
 103                 else {
 104                         struct commit_list *parents;
 105
 106                         if (!ancestors_only && !(o->flags & POPPED))
 107                                 non_common_revs--;
 108                         if (!o->parsed && !dont_parse)
 109                                 if (parse_commit(commit))
 110                                         return;
 111
 112                         for (parents = commit->parents;
 113                                         parents;
 114                                         parents = parents->next)
 115                                 mark_common(parents->item, 0, dont_parse);
 116                 }
 117         }
 118 }
 119
 120 /*
 121   Get the next rev to send, ignoring the common.
 122 */
 123
 124 static const unsigned char *get_rev(void)
 125 {
 126         struct commit *commit = NULL;
 127
 128         while (commit == NULL) {
 129                 unsigned int mark;
 130                 struct commit_list *parents;
 131
 132                 if (rev_list.nr == 0 || non_common_revs == 0)
 133                         return NULL;
 134
 135                 commit = prio_queue_get(&rev_list);
 136                 parse_commit(commit);
 137                 parents = commit->parents;
 138
 139                 commit->object.flags |= POPPED;
 140                 if (!(commit->object.flags & COMMON))
 141                         non_common_revs--;
 142
 143                 if (commit->object.flags & COMMON) {
 144                         /* do not send "have", and ignore ancestors */
 145                         commit = NULL;
 146                         mark = COMMON | SEEN;
 147                 } else if (commit->object.flags & COMMON_REF)
 148                         /* send "have", and ignore ancestors */
 149                         mark = COMMON | SEEN;
 150                 else
 151                         /* send "have", also for its ancestors */
 152                         mark = SEEN;
 153
 154                 while (parents) {
 155                         if (!(parents->item->object.flags & SEEN))
 156                                 rev_list_push(parents->item, mark);
 157                         if (mark & COMMON)
 158                                 mark_common(parents->item, 1, 0);
 159                         parents = parents->next;
 160                 }
 161         }
 162
 163         return commit->object.sha1;
 164 }
 165
 166 enum ack_type {
 167         NAK = 0,
 168         ACK,
 169         ACK_continue,
 170         ACK_common,
 171         ACK_ready
 172 };
 173
 174 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 175 {
 176         if (args->stateless_rpc && args->depth > 0) {
 177                 /* If we sent a depth we will get back "duplicate"
 178                  * shallow and unshallow commands every time there
 179                  * is a block of have lines exchanged.
 180                  */
 181                 char *line;
 182                 while ((line = packet_read_line(fd, NULL))) {
 183                         if (starts_with(line, "shallow "))
 184                                 continue;
 185                         if (starts_with(line, "unshallow "))
 186                                 continue;
 187                         die("git fetch-pack: expected shallow list");
 188                 }
 189         }
 190 }
 191
 192 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 193 {
 194         int len;
 195         char *line = packet_read_line(fd, &len);
 196         const char *arg;
 197
 198         if (!len)
 199                 die("git fetch-pack: expected ACK/NAK, got EOF");
 200         if (!strcmp(line, "NAK"))
 201                 return NAK;
 202         if (skip_prefix(line, "ACK ", &arg)) {
 203                 if (!get_sha1_hex(arg, result_sha1)) {
 204                         arg += 40;
 205                         len -= arg - line;
 206                         if (len < 1)
 207                                 return ACK;
 208                         if (strstr(arg, "continue"))
 209                                 return ACK_continue;
 210                         if (strstr(arg, "common"))
 211                                 return ACK_common;
 212                         if (strstr(arg, "ready"))
 213                                 return ACK_ready;
 214                         return ACK;
 215                 }
 216         }
 217         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 218 }
 219
 220 static void send_request(struct fetch_pack_args *args,
 221                          int fd, struct strbuf *buf)
 222 {
 223         if (args->stateless_rpc) {
 224                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 225                 packet_flush(fd);
 226         } else
 227                 write_or_die(fd, buf->buf, buf->len);
 228 }
 229
 230 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 231 {
 232         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 233 }
 234
 235 #define INITIAL_FLUSH 16
 236 #define PIPESAFE_FLUSH 32
 237 #define LARGE_FLUSH 1024
 238
 239 static int next_flush(struct fetch_pack_args *args, int count)
 240 {
 241         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 242
 243         if (count < flush_limit)
 244                 count <<= 1;
 245         else
 246                 count += flush_limit;
 247         return count;
 248 }
 249
 250 static int find_common(struct fetch_pack_args *args,
 251                        int fd[2], unsigned char *result_sha1,
 252                        struct ref *refs)
 253 {
 254         int fetching;
 255         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 256         const unsigned char *sha1;
 257         unsigned in_vain = 0;
 258         int got_continue = 0;
 259         int got_ready = 0;
 260         struct strbuf req_buf = STRBUF_INIT;
 261         size_t state_len = 0;
 262
 263         if (args->stateless_rpc && multi_ack == 1)
 264                 die("--stateless-rpc requires multi_ack_detailed");
 265         if (marked)
 266                 for_each_ref(clear_marks, NULL);
 267         marked = 1;
 268
 269         for_each_ref(rev_list_insert_ref, NULL);
 270         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 271
 272         fetching = 0;
 273         for ( ; refs ; refs = refs->next) {
 274                 unsigned char *remote = refs->old_sha1;
 275                 const char *remote_hex;
 276                 struct object *o;
 277
 278                 /*
 279                  * If that object is complete (i.e. it is an ancestor of a
 280                  * local ref), we tell them we have it but do not have to
 281                  * tell them about its ancestors, which they already know
 282                  * about.
 283                  *
 284                  * We use lookup_object here because we are only
 285                  * interested in the case we *know* the object is
 286                  * reachable and we have already scanned it.
 287                  */
 288                 if (((o = lookup_object(remote)) != NULL) &&
 289                                 (o->flags & COMPLETE)) {
 290                         continue;
 291                 }
 292
 293                 remote_hex = sha1_to_hex(remote);
 294                 if (!fetching) {
 295                         struct strbuf c = STRBUF_INIT;
 296                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 297                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 298                         if (no_done)            strbuf_addstr(&c, " no-done");
 299                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 300                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 301                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 302                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 303                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 304                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 305                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 306                                                             git_user_agent_sanitized());
 307                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 308                         strbuf_release(&c);
 309                 } else
 310                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 311                 fetching++;
 312         }
 313
 314         if (!fetching) {
 315                 strbuf_release(&req_buf);
 316                 packet_flush(fd[1]);
 317                 return 1;
 318         }
 319
 320         if (is_repository_shallow())
 321                 write_shallow_commits(&req_buf, 1, NULL);
 322         if (args->depth > 0)
 323                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 324         packet_buf_flush(&req_buf);
 325         state_len = req_buf.len;
 326
 327         if (args->depth > 0) {
 328                 char *line;
 329                 const char *arg;
 330                 unsigned char sha1[20];
 331
 332                 send_request(args, fd[1], &req_buf);
 333                 while ((line = packet_read_line(fd[0], NULL))) {
 334                         if (skip_prefix(line, "shallow ", &arg)) {
 335                                 if (get_sha1_hex(arg, sha1))
 336                                         die("invalid shallow line: %s", line);
 337                                 register_shallow(sha1);
 338                                 continue;
 339                         }
 340                         if (skip_prefix(line, "unshallow ", &arg)) {
 341                                 if (get_sha1_hex(arg, sha1))
 342                                         die("invalid unshallow line: %s", line);
 343                                 if (!lookup_object(sha1))
 344                                         die("object not found: %s", line);
 345                                 /* make sure that it is parsed as shallow */
 346                                 if (!parse_object(sha1))
 347                                         die("error in object: %s", line);
 348                                 if (unregister_shallow(sha1))
 349                                         die("no shallow found: %s", line);
 350                                 continue;
 351                         }
 352                         die("expected shallow/unshallow, got %s", line);
 353                 }
 354         } else if (!args->stateless_rpc)
 355                 send_request(args, fd[1], &req_buf);
 356
 357         if (!args->stateless_rpc) {
 358                 /* If we aren't using the stateless-rpc interface
 359                  * we don't need to retain the headers.
 360                  */
 361                 strbuf_setlen(&req_buf, 0);
 362                 state_len = 0;
 363         }
 364
 365         flushes = 0;
 366         retval = -1;
 367         while ((sha1 = get_rev())) {
 368                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 369                 if (args->verbose)
 370                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 371                 in_vain++;
 372                 if (flush_at <= ++count) {
 373                         int ack;
 374
 375                         packet_buf_flush(&req_buf);
 376                         send_request(args, fd[1], &req_buf);
 377                         strbuf_setlen(&req_buf, state_len);
 378                         flushes++;
 379                         flush_at = next_flush(args, count);
 380
 381                         /*
 382                          * We keep one window "ahead" of the other side, and
 383                          * will wait for an ACK only on the next one
 384                          */
 385                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 386                                 continue;
 387
 388                         consume_shallow_list(args, fd[0]);
 389                         do {
 390                                 ack = get_ack(fd[0], result_sha1);
 391                                 if (args->verbose && ack)
 392                                         fprintf(stderr, "got ack %d %s\n", ack,
 393                                                         sha1_to_hex(result_sha1));
 394                                 switch (ack) {
 395                                 case ACK:
 396                                         flushes = 0;
 397                                         multi_ack = 0;
 398                                         retval = 0;
 399                                         goto done;
 400                                 case ACK_common:
 401                                 case ACK_ready:
 402                                 case ACK_continue: {
 403                                         struct commit *commit =
 404                                                 lookup_commit(result_sha1);
 405                                         if (!commit)
 406                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 407                                         if (args->stateless_rpc
 408                                          && ack == ACK_common
 409                                          && !(commit->object.flags & COMMON)) {
 410                                                 /* We need to replay the have for this object
 411                                                  * on the next RPC request so the peer knows
 412                                                  * it is in common with us.
 413                                                  */
 414                                                 const char *hex = sha1_to_hex(result_sha1);
 415                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 416                                                 state_len = req_buf.len;
 417                                         }
 418                                         mark_common(commit, 0, 1);
 419                                         retval = 0;
 420                                         in_vain = 0;
 421                                         got_continue = 1;
 422                                         if (ack == ACK_ready) {
 423                                                 clear_prio_queue(&rev_list);
 424                                                 got_ready = 1;
 425                                         }
 426                                         break;
 427                                         }
 428                                 }
 429                         } while (ack);
 430                         flushes--;
 431                         if (got_continue && MAX_IN_VAIN < in_vain) {
 432                                 if (args->verbose)
 433                                         fprintf(stderr, "giving up\n");
 434                                 break; /* give up */
 435                         }
 436                 }
 437         }
 438 done:
 439         if (!got_ready || !no_done) {
 440                 packet_buf_write(&req_buf, "done\n");
 441                 send_request(args, fd[1], &req_buf);
 442         }
 443         if (args->verbose)
 444                 fprintf(stderr, "done\n");
 445         if (retval != 0) {
 446                 multi_ack = 0;
 447                 flushes++;
 448         }
 449         strbuf_release(&req_buf);
 450
 451         if (!got_ready || !no_done)
 452                 consume_shallow_list(args, fd[0]);
 453         while (flushes || multi_ack) {
 454                 int ack = get_ack(fd[0], result_sha1);
 455                 if (ack) {
 456                         if (args->verbose)
 457                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 458                                         sha1_to_hex(result_sha1));
 459                         if (ack == ACK)
 460                                 return 0;
 461                         multi_ack = 1;
 462                         continue;
 463                 }
 464                 flushes--;
 465         }
 466         /* it is no error to fetch into a completely empty repo */
 467         return count ? retval : 0;
 468 }
 469
 470 static struct commit_list *complete;
 471
 472 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 473 {
 474         struct object *o = parse_object(sha1);
 475
 476         while (o && o->type == OBJ_TAG) {
 477                 struct tag *t = (struct tag *) o;
 478                 if (!t->tagged)
 479                         break; /* broken repository */
 480                 o->flags |= COMPLETE;
 481                 o = parse_object(t->tagged->sha1);
 482         }
 483         if (o && o->type == OBJ_COMMIT) {
 484                 struct commit *commit = (struct commit *)o;
 485                 if (!(commit->object.flags & COMPLETE)) {
 486                         commit->object.flags |= COMPLETE;
 487                         commit_list_insert(commit, &complete);
 488                 }
 489         }
 490         return 0;
 491 }
 492
 493 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 494                                          unsigned long cutoff)
 495 {
 496         while (complete && cutoff <= complete->item->date) {
 497                 if (args->verbose)
 498                         fprintf(stderr, "Marking %s as complete\n",
 499                                 sha1_to_hex(complete->item->object.sha1));
 500                 pop_most_recent_commit(&complete, COMPLETE);
 501         }
 502 }
 503
 504 static void filter_refs(struct fetch_pack_args *args,
 505                         struct ref **refs,
 506                         struct ref **sought, int nr_sought)
 507 {
 508         struct ref *newlist = NULL;
 509         struct ref **newtail = &newlist;
 510         struct ref *ref, *next;
 511         int i;
 512
 513         i = 0;
 514         for (ref = *refs; ref; ref = next) {
 515                 int keep = 0;
 516                 next = ref->next;
 517
 518                 if (starts_with(ref->name, "refs/") &&
 519                     check_refname_format(ref->name, 0))
 520                         ; /* trash */
 521                 else {
 522                         while (i < nr_sought) {
 523                                 int cmp = strcmp(ref->name, sought[i]->name);
 524                                 if (cmp < 0)
 525                                         break; /* definitely do not have it */
 526                                 else if (cmp == 0) {
 527                                         keep = 1; /* definitely have it */
 528                                         sought[i]->matched = 1;
 529                                 }
 530                                 i++;
 531                         }
 532                 }
 533
 534                 if (!keep && args->fetch_all &&
 535                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 536                         keep = 1;
 537
 538                 if (keep) {
 539                         *newtail = ref;
 540                         ref->next = NULL;
 541                         newtail = &ref->next;
 542                 } else {
 543                         free(ref);
 544                 }
 545         }
 546
 547         /* Append unmatched requests to the list */
 548         if ((allow_unadvertised_object_request & ALLOW_TIP_SHA1)) {
 549                 for (i = 0; i < nr_sought; i++) {
 550                         unsigned char sha1[20];
 551
 552                         ref = sought[i];
 553                         if (ref->matched)
 554                                 continue;
 555                         if (get_sha1_hex(ref->name, sha1) ||
 556                             ref->name[40] != '\0' ||
 557                             hashcmp(sha1, ref->old_sha1))
 558                                 continue;
 559
 560                         ref->matched = 1;
 561                         *newtail = copy_ref(ref);
 562                         newtail = &(*newtail)->next;
 563                 }
 564         }
 565         *refs = newlist;
 566 }
 567
 568 static void mark_alternate_complete(const struct ref *ref, void *unused)
 569 {
 570         mark_complete(NULL, ref->old_sha1, 0, NULL);
 571 }
 572
 573 static int everything_local(struct fetch_pack_args *args,
 574                             struct ref **refs,
 575                             struct ref **sought, int nr_sought)
 576 {
 577         struct ref *ref;
 578         int retval;
 579         unsigned long cutoff = 0;
 580
 581         save_commit_buffer = 0;
 582
 583         for (ref = *refs; ref; ref = ref->next) {
 584                 struct object *o;
 585
 586                 if (!has_sha1_file(ref->old_sha1))
 587                         continue;
 588
 589                 o = parse_object(ref->old_sha1);
 590                 if (!o)
 591                         continue;
 592
 593                 /* We already have it -- which may mean that we were
 594                  * in sync with the other side at some time after
 595                  * that (it is OK if we guess wrong here).
 596                  */
 597                 if (o->type == OBJ_COMMIT) {
 598                         struct commit *commit = (struct commit *)o;
 599                         if (!cutoff || cutoff < commit->date)
 600                                 cutoff = commit->date;
 601                 }
 602         }
 603
 604         if (!args->depth) {
 605                 for_each_ref(mark_complete, NULL);
 606                 for_each_alternate_ref(mark_alternate_complete, NULL);
 607                 commit_list_sort_by_date(&complete);
 608                 if (cutoff)
 609                         mark_recent_complete_commits(args, cutoff);
 610         }
 611
 612         /*
 613          * Mark all complete remote refs as common refs.
 614          * Don't mark them common yet; the server has to be told so first.
 615          */
 616         for (ref = *refs; ref; ref = ref->next) {
 617                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 618                                              NULL, 0);
 619
 620                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 621                         continue;
 622
 623                 if (!(o->flags & SEEN)) {
 624                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 625
 626                         mark_common((struct commit *)o, 1, 1);
 627                 }
 628         }
 629
 630         filter_refs(args, refs, sought, nr_sought);
 631
 632         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 633                 const unsigned char *remote = ref->old_sha1;
 634                 struct object *o;
 635
 636                 o = lookup_object(remote);
 637                 if (!o || !(o->flags & COMPLETE)) {
 638                         retval = 0;
 639                         if (!args->verbose)
 640                                 continue;
 641                         fprintf(stderr,
 642                                 "want %s (%s)\n", sha1_to_hex(remote),
 643                                 ref->name);
 644                         continue;
 645                 }
 646                 if (!args->verbose)
 647                         continue;
 648                 fprintf(stderr,
 649                         "already have %s (%s)\n", sha1_to_hex(remote),
 650                         ref->name);
 651         }
 652         return retval;
 653 }
 654
 655 static int sideband_demux(int in, int out, void *data)
 656 {
 657         int *xd = data;
 658
 659         int ret = recv_sideband("fetch-pack", xd[0], out);
 660         close(out);
 661         return ret;
 662 }
 663
 664 static int get_pack(struct fetch_pack_args *args,
 665                     int xd[2], char **pack_lockfile)
 666 {
 667         struct async demux;
 668         const char *argv[22];
 669         char keep_arg[256];
 670         char hdr_arg[256];
 671         const char **av, *cmd_name;
 672         int do_keep = args->keep_pack;
 673         struct child_process cmd = CHILD_PROCESS_INIT;
 674         int ret;
 675
 676         memset(&demux, 0, sizeof(demux));
 677         if (use_sideband) {
 678                 /* xd[] is talking with upload-pack; subprocess reads from
 679                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 680                  * through demux->out.
 681                  */
 682                 demux.proc = sideband_demux;
 683                 demux.data = xd;
 684                 demux.out = -1;
 685                 if (start_async(&demux))
 686                         die("fetch-pack: unable to fork off sideband"
 687                             " demultiplexer");
 688         }
 689         else
 690                 demux.out = xd[0];
 691
 692         cmd.argv = argv;
 693         av = argv;
 694         *hdr_arg = 0;
 695         if (!args->keep_pack && unpack_limit) {
 696                 struct pack_header header;
 697
 698                 if (read_pack_header(demux.out, &header))
 699                         die("protocol error: bad pack header");
 700                 snprintf(hdr_arg, sizeof(hdr_arg),
 701                          "--pack_header=%"PRIu32",%"PRIu32,
 702                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 703                 if (ntohl(header.hdr_entries) < unpack_limit)
 704                         do_keep = 0;
 705                 else
 706                         do_keep = 1;
 707         }
 708
 709         if (alternate_shallow_file) {
 710                 *av++ = "--shallow-file";
 711                 *av++ = alternate_shallow_file;
 712         }
 713
 714         if (do_keep) {
 715                 if (pack_lockfile)
 716                         cmd.out = -1;
 717                 *av++ = cmd_name = "index-pack";
 718                 *av++ = "--stdin";
 719                 if (!args->quiet && !args->no_progress)
 720                         *av++ = "-v";
 721                 if (args->use_thin_pack)
 722                         *av++ = "--fix-thin";
 723                 if (args->lock_pack || unpack_limit) {
 724                         int s = sprintf(keep_arg,
 725                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 726                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 727                                 strcpy(keep_arg + s, "localhost");
 728                         *av++ = keep_arg;
 729                 }
 730                 if (args->check_self_contained_and_connected)
 731                         *av++ = "--check-self-contained-and-connected";
 732         }
 733         else {
 734                 *av++ = cmd_name = "unpack-objects";
 735                 if (args->quiet || args->no_progress)
 736                         *av++ = "-q";
 737                 args->check_self_contained_and_connected = 0;
 738         }
 739         if (*hdr_arg)
 740                 *av++ = hdr_arg;
 741         if (fetch_fsck_objects >= 0
 742             ? fetch_fsck_objects
 743             : transfer_fsck_objects >= 0
 744             ? transfer_fsck_objects
 745             : 0)
 746                 *av++ = "--strict";
 747         *av++ = NULL;
 748
 749         cmd.in = demux.out;
 750         cmd.git_cmd = 1;
 751         if (start_command(&cmd))
 752                 die("fetch-pack: unable to fork off %s", cmd_name);
 753         if (do_keep && pack_lockfile) {
 754                 *pack_lockfile = index_pack_lockfile(cmd.out);
 755                 close(cmd.out);
 756         }
 757
 758         if (!use_sideband)
 759                 /* Closed by start_command() */
 760                 xd[0] = -1;
 761
 762         ret = finish_command(&cmd);
 763         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 764                 args->self_contained_and_connected =
 765                         args->check_self_contained_and_connected &&
 766                         ret == 0;
 767         else
 768                 die("%s failed", cmd_name);
 769         if (use_sideband && finish_async(&demux))
 770                 die("error in sideband demultiplexer");
 771         return 0;
 772 }
 773
 774 static int cmp_ref_by_name(const void *a_, const void *b_)
 775 {
 776         const struct ref *a = *((const struct ref **)a_);
 777         const struct ref *b = *((const struct ref **)b_);
 778         return strcmp(a->name, b->name);
 779 }
 780
 781 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 782                                  int fd[2],
 783                                  const struct ref *orig_ref,
 784                                  struct ref **sought, int nr_sought,
 785                                  struct shallow_info *si,
 786                                  char **pack_lockfile)
 787 {
 788         struct ref *ref = copy_ref_list(orig_ref);
 789         unsigned char sha1[20];
 790         const char *agent_feature;
 791         int agent_len;
 792
 793         sort_ref_list(&ref, ref_compare_name);
 794         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 795
 796         if (is_repository_shallow() && !server_supports("shallow"))
 797                 die("Server does not support shallow clients");
 798         if (server_supports("multi_ack_detailed")) {
 799                 if (args->verbose)
 800                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 801                 multi_ack = 2;
 802                 if (server_supports("no-done")) {
 803                         if (args->verbose)
 804                                 fprintf(stderr, "Server supports no-done\n");
 805                         if (args->stateless_rpc)
 806                                 no_done = 1;
 807                 }
 808         }
 809         else if (server_supports("multi_ack")) {
 810                 if (args->verbose)
 811                         fprintf(stderr, "Server supports multi_ack\n");
 812                 multi_ack = 1;
 813         }
 814         if (server_supports("side-band-64k")) {
 815                 if (args->verbose)
 816                         fprintf(stderr, "Server supports side-band-64k\n");
 817                 use_sideband = 2;
 818         }
 819         else if (server_supports("side-band")) {
 820                 if (args->verbose)
 821                         fprintf(stderr, "Server supports side-band\n");
 822                 use_sideband = 1;
 823         }
 824         if (server_supports("allow-tip-sha1-in-want")) {
 825                 if (args->verbose)
 826                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 827                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 828         }
 829         if (!server_supports("thin-pack"))
 830                 args->use_thin_pack = 0;
 831         if (!server_supports("no-progress"))
 832                 args->no_progress = 0;
 833         if (!server_supports("include-tag"))
 834                 args->include_tag = 0;
 835         if (server_supports("ofs-delta")) {
 836                 if (args->verbose)
 837                         fprintf(stderr, "Server supports ofs-delta\n");
 838         } else
 839                 prefer_ofs_delta = 0;
 840
 841         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 842                 agent_supported = 1;
 843                 if (args->verbose && agent_len)
 844                         fprintf(stderr, "Server version is %.*s\n",
 845                                 agent_len, agent_feature);
 846         }
 847
 848         if (everything_local(args, &ref, sought, nr_sought)) {
 849                 packet_flush(fd[1]);
 850                 goto all_done;
 851         }
 852         if (find_common(args, fd, sha1, ref) < 0)
 853                 if (!args->keep_pack)
 854                         /* When cloning, it is not unusual to have
 855                          * no common commit.
 856                          */
 857                         warning("no common commits");
 858
 859         if (args->stateless_rpc)
 860                 packet_flush(fd[1]);
 861         if (args->depth > 0)
 862                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 863                                         NULL);
 864         else if (si->nr_ours || si->nr_theirs)
 865                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 866         else
 867                 alternate_shallow_file = NULL;
 868         if (get_pack(args, fd, pack_lockfile))
 869                 die("git fetch-pack: fetch failed.");
 870
 871  all_done:
 872         return ref;
 873 }
 874
 875 static void fetch_pack_config(void)
 876 {
 877         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 878         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 879         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 880         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 881         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 882
 883         git_config(git_default_config, NULL);
 884 }
 885
 886 static void fetch_pack_setup(void)
 887 {
 888         static int did_setup;
 889         if (did_setup)
 890                 return;
 891         fetch_pack_config();
 892         if (0 <= transfer_unpack_limit)
 893                 unpack_limit = transfer_unpack_limit;
 894         else if (0 <= fetch_unpack_limit)
 895                 unpack_limit = fetch_unpack_limit;
 896         did_setup = 1;
 897 }
 898
 899 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 900 {
 901         struct string_list names = STRING_LIST_INIT_NODUP;
 902         int src, dst;
 903
 904         for (src = dst = 0; src < nr; src++) {
 905                 struct string_list_item *item;
 906                 item = string_list_insert(&names, ref[src]->name);
 907                 if (item->util)
 908                         continue; /* already have it */
 909                 item->util = ref[src];
 910                 if (src != dst)
 911                         ref[dst] = ref[src];
 912                 dst++;
 913         }
 914         for (src = dst; src < nr; src++)
 915                 ref[src] = NULL;
 916         string_list_clear(&names, 0);
 917         return dst;
 918 }
 919
 920 static void update_shallow(struct fetch_pack_args *args,
 921                            struct ref **sought, int nr_sought,
 922                            struct shallow_info *si)
 923 {
 924         struct sha1_array ref = SHA1_ARRAY_INIT;
 925         int *status;
 926         int i;
 927
 928         if (args->depth > 0 && alternate_shallow_file) {
 929                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 930                         unlink_or_warn(git_path("shallow"));
 931                         rollback_lock_file(&shallow_lock);
 932                 } else
 933                         commit_lock_file(&shallow_lock);
 934                 return;
 935         }
 936
 937         if (!si->shallow || !si->shallow->nr)
 938                 return;
 939
 940         if (args->cloning) {
 941                 /*
 942                  * remote is shallow, but this is a clone, there are
 943                  * no objects in repo to worry about. Accept any
 944                  * shallow points that exist in the pack (iow in repo
 945                  * after get_pack() and reprepare_packed_git())
 946                  */
 947                 struct sha1_array extra = SHA1_ARRAY_INIT;
 948                 unsigned char (*sha1)[20] = si->shallow->sha1;
 949                 for (i = 0; i < si->shallow->nr; i++)
 950                         if (has_sha1_file(sha1[i]))
 951                                 sha1_array_append(&extra, sha1[i]);
 952                 if (extra.nr) {
 953                         setup_alternate_shallow(&shallow_lock,
 954                                                 &alternate_shallow_file,
 955                                                 &extra);
 956                         commit_lock_file(&shallow_lock);
 957                 }
 958                 sha1_array_clear(&extra);
 959                 return;
 960         }
 961
 962         if (!si->nr_ours && !si->nr_theirs)
 963                 return;
 964
 965         remove_nonexistent_theirs_shallow(si);
 966         if (!si->nr_ours && !si->nr_theirs)
 967                 return;
 968         for (i = 0; i < nr_sought; i++)
 969                 sha1_array_append(&ref, sought[i]->old_sha1);
 970         si->ref = &ref;
 971
 972         if (args->update_shallow) {
 973                 /*
 974                  * remote is also shallow, .git/shallow may be updated
 975                  * so all refs can be accepted. Make sure we only add
 976                  * shallow roots that are actually reachable from new
 977                  * refs.
 978                  */
 979                 struct sha1_array extra = SHA1_ARRAY_INIT;
 980                 unsigned char (*sha1)[20] = si->shallow->sha1;
 981                 assign_shallow_commits_to_refs(si, NULL, NULL);
 982                 if (!si->nr_ours && !si->nr_theirs) {
 983                         sha1_array_clear(&ref);
 984                         return;
 985                 }
 986                 for (i = 0; i < si->nr_ours; i++)
 987                         sha1_array_append(&extra, sha1[si->ours[i]]);
 988                 for (i = 0; i < si->nr_theirs; i++)
 989                         sha1_array_append(&extra, sha1[si->theirs[i]]);
 990                 setup_alternate_shallow(&shallow_lock,
 991                                         &alternate_shallow_file,
 992                                         &extra);
 993                 commit_lock_file(&shallow_lock);
 994                 sha1_array_clear(&extra);
 995                 sha1_array_clear(&ref);
 996                 return;
 997         }
 998
 999         /*
1000          * remote is also shallow, check what ref is safe to update
1001          * without updating .git/shallow
1002          */
1003         status = xcalloc(nr_sought, sizeof(*status));
1004         assign_shallow_commits_to_refs(si, NULL, status);
1005         if (si->nr_ours || si->nr_theirs) {
1006                 for (i = 0; i < nr_sought; i++)
1007                         if (status[i])
1008                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1009         }
1010         free(status);
1011         sha1_array_clear(&ref);
1012 }
1013
1014 struct ref *fetch_pack(struct fetch_pack_args *args,
1015                        int fd[], struct child_process *conn,
1016                        const struct ref *ref,
1017                        const char *dest,
1018                        struct ref **sought, int nr_sought,
1019                        struct sha1_array *shallow,
1020                        char **pack_lockfile)
1021 {
1022         struct ref *ref_cpy;
1023         struct shallow_info si;
1024
1025         fetch_pack_setup();
1026         if (nr_sought)
1027                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1028
1029         if (!ref) {
1030                 packet_flush(fd[1]);
1031                 die("no matching remote head");
1032         }
1033         prepare_shallow_info(&si, shallow);
1034         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1035                                 &si, pack_lockfile);
1036         reprepare_packed_git();
1037         update_shallow(args, sought, nr_sought, &si);
1038         clear_shallow_info(&si);
1039         return ref_cpy;
1040 }