git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "connect.h"
  13 #include "transport.h"
  14 #include "version.h"
  15 #include "prio-queue.h"
  16 #include "sha1-array.h"
  17
  18 static int transfer_unpack_limit = -1;
  19 static int fetch_unpack_limit = -1;
  20 static int unpack_limit = 100;
  21 static int prefer_ofs_delta = 1;
  22 static int no_done;
  23 static int fetch_fsck_objects = -1;
  24 static int transfer_fsck_objects = -1;
  25 static int agent_supported;
  26 static struct lock_file shallow_lock;
  27 static const char *alternate_shallow_file;
  28
  29 #define COMPLETE        (1U << 0)
  30 #define COMMON          (1U << 1)
  31 #define COMMON_REF      (1U << 2)
  32 #define SEEN            (1U << 3)
  33 #define POPPED          (1U << 4)
  34
  35 static int marked;
  36
  37 /*
  38  * After sending this many "have"s if we do not get any new ACK , we
  39  * give up traversing our history.
  40  */
  41 #define MAX_IN_VAIN 256
  42
  43 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  44 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  45
  46 static void rev_list_push(struct commit *commit, int mark)
  47 {
  48         if (!(commit->object.flags & mark)) {
  49                 commit->object.flags |= mark;
  50
  51                 if (parse_commit(commit))
  52                         return;
  53
  54                 prio_queue_put(&rev_list, commit);
  55
  56                 if (!(commit->object.flags & COMMON))
  57                         non_common_revs++;
  58         }
  59 }
  60
  61 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  62 {
  63         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  64
  65         if (o && o->type == OBJ_COMMIT)
  66                 rev_list_push((struct commit *)o, SEEN);
  67
  68         return 0;
  69 }
  70
  71 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  72 {
  73         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  74
  75         if (o && o->type == OBJ_COMMIT)
  76                 clear_commit_marks((struct commit *)o,
  77                                    COMMON | COMMON_REF | SEEN | POPPED);
  78         return 0;
  79 }
  80
  81 /*
  82    This function marks a rev and its ancestors as common.
  83    In some cases, it is desirable to mark only the ancestors (for example
  84    when only the server does not yet know that they are common).
  85 */
  86
  87 static void mark_common(struct commit *commit,
  88                 int ancestors_only, int dont_parse)
  89 {
  90         if (commit != NULL && !(commit->object.flags & COMMON)) {
  91                 struct object *o = (struct object *)commit;
  92
  93                 if (!ancestors_only)
  94                         o->flags |= COMMON;
  95
  96                 if (!(o->flags & SEEN))
  97                         rev_list_push(commit, SEEN);
  98                 else {
  99                         struct commit_list *parents;
 100
 101                         if (!ancestors_only && !(o->flags & POPPED))
 102                                 non_common_revs--;
 103                         if (!o->parsed && !dont_parse)
 104                                 if (parse_commit(commit))
 105                                         return;
 106
 107                         for (parents = commit->parents;
 108                                         parents;
 109                                         parents = parents->next)
 110                                 mark_common(parents->item, 0, dont_parse);
 111                 }
 112         }
 113 }
 114
 115 /*
 116   Get the next rev to send, ignoring the common.
 117 */
 118
 119 static const unsigned char *get_rev(void)
 120 {
 121         struct commit *commit = NULL;
 122
 123         while (commit == NULL) {
 124                 unsigned int mark;
 125                 struct commit_list *parents;
 126
 127                 if (rev_list.nr == 0 || non_common_revs == 0)
 128                         return NULL;
 129
 130                 commit = prio_queue_get(&rev_list);
 131                 parse_commit(commit);
 132                 parents = commit->parents;
 133
 134                 commit->object.flags |= POPPED;
 135                 if (!(commit->object.flags & COMMON))
 136                         non_common_revs--;
 137
 138                 if (commit->object.flags & COMMON) {
 139                         /* do not send "have", and ignore ancestors */
 140                         commit = NULL;
 141                         mark = COMMON | SEEN;
 142                 } else if (commit->object.flags & COMMON_REF)
 143                         /* send "have", and ignore ancestors */
 144                         mark = COMMON | SEEN;
 145                 else
 146                         /* send "have", also for its ancestors */
 147                         mark = SEEN;
 148
 149                 while (parents) {
 150                         if (!(parents->item->object.flags & SEEN))
 151                                 rev_list_push(parents->item, mark);
 152                         if (mark & COMMON)
 153                                 mark_common(parents->item, 1, 0);
 154                         parents = parents->next;
 155                 }
 156         }
 157
 158         return commit->object.sha1;
 159 }
 160
 161 enum ack_type {
 162         NAK = 0,
 163         ACK,
 164         ACK_continue,
 165         ACK_common,
 166         ACK_ready
 167 };
 168
 169 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 170 {
 171         if (args->stateless_rpc && args->depth > 0) {
 172                 /* If we sent a depth we will get back "duplicate"
 173                  * shallow and unshallow commands every time there
 174                  * is a block of have lines exchanged.
 175                  */
 176                 char *line;
 177                 while ((line = packet_read_line(fd, NULL))) {
 178                         if (starts_with(line, "shallow "))
 179                                 continue;
 180                         if (starts_with(line, "unshallow "))
 181                                 continue;
 182                         die("git fetch-pack: expected shallow list");
 183                 }
 184         }
 185 }
 186
 187 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 188 {
 189         int len;
 190         char *line = packet_read_line(fd, &len);
 191
 192         if (!len)
 193                 die("git fetch-pack: expected ACK/NAK, got EOF");
 194         if (!strcmp(line, "NAK"))
 195                 return NAK;
 196         if (starts_with(line, "ACK ")) {
 197                 if (!get_sha1_hex(line+4, result_sha1)) {
 198                         if (len < 45)
 199                                 return ACK;
 200                         if (strstr(line+45, "continue"))
 201                                 return ACK_continue;
 202                         if (strstr(line+45, "common"))
 203                                 return ACK_common;
 204                         if (strstr(line+45, "ready"))
 205                                 return ACK_ready;
 206                         return ACK;
 207                 }
 208         }
 209         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 210 }
 211
 212 static void send_request(struct fetch_pack_args *args,
 213                          int fd, struct strbuf *buf)
 214 {
 215         if (args->stateless_rpc) {
 216                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 217                 packet_flush(fd);
 218         } else
 219                 write_or_die(fd, buf->buf, buf->len);
 220 }
 221
 222 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 223 {
 224         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 225 }
 226
 227 #define INITIAL_FLUSH 16
 228 #define PIPESAFE_FLUSH 32
 229 #define LARGE_FLUSH 1024
 230
 231 static int next_flush(struct fetch_pack_args *args, int count)
 232 {
 233         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 234
 235         if (count < flush_limit)
 236                 count <<= 1;
 237         else
 238                 count += flush_limit;
 239         return count;
 240 }
 241
 242 static int find_common(struct fetch_pack_args *args,
 243                        int fd[2], unsigned char *result_sha1,
 244                        struct ref *refs)
 245 {
 246         int fetching;
 247         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 248         const unsigned char *sha1;
 249         unsigned in_vain = 0;
 250         int got_continue = 0;
 251         int got_ready = 0;
 252         struct strbuf req_buf = STRBUF_INIT;
 253         size_t state_len = 0;
 254
 255         if (args->stateless_rpc && multi_ack == 1)
 256                 die("--stateless-rpc requires multi_ack_detailed");
 257         if (marked)
 258                 for_each_ref(clear_marks, NULL);
 259         marked = 1;
 260
 261         for_each_ref(rev_list_insert_ref, NULL);
 262         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 263
 264         fetching = 0;
 265         for ( ; refs ; refs = refs->next) {
 266                 unsigned char *remote = refs->old_sha1;
 267                 const char *remote_hex;
 268                 struct object *o;
 269
 270                 /*
 271                  * If that object is complete (i.e. it is an ancestor of a
 272                  * local ref), we tell them we have it but do not have to
 273                  * tell them about its ancestors, which they already know
 274                  * about.
 275                  *
 276                  * We use lookup_object here because we are only
 277                  * interested in the case we *know* the object is
 278                  * reachable and we have already scanned it.
 279                  */
 280                 if (((o = lookup_object(remote)) != NULL) &&
 281                                 (o->flags & COMPLETE)) {
 282                         continue;
 283                 }
 284
 285                 remote_hex = sha1_to_hex(remote);
 286                 if (!fetching) {
 287                         struct strbuf c = STRBUF_INIT;
 288                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 289                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 290                         if (no_done)            strbuf_addstr(&c, " no-done");
 291                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 292                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 293                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 294                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 295                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 296                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 297                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 298                                                             git_user_agent_sanitized());
 299                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 300                         strbuf_release(&c);
 301                 } else
 302                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 303                 fetching++;
 304         }
 305
 306         if (!fetching) {
 307                 strbuf_release(&req_buf);
 308                 packet_flush(fd[1]);
 309                 return 1;
 310         }
 311
 312         if (is_repository_shallow())
 313                 write_shallow_commits(&req_buf, 1, NULL);
 314         if (args->depth > 0)
 315                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 316         packet_buf_flush(&req_buf);
 317         state_len = req_buf.len;
 318
 319         if (args->depth > 0) {
 320                 char *line;
 321                 unsigned char sha1[20];
 322
 323                 send_request(args, fd[1], &req_buf);
 324                 while ((line = packet_read_line(fd[0], NULL))) {
 325                         if (starts_with(line, "shallow ")) {
 326                                 if (get_sha1_hex(line + 8, sha1))
 327                                         die("invalid shallow line: %s", line);
 328                                 register_shallow(sha1);
 329                                 continue;
 330                         }
 331                         if (starts_with(line, "unshallow ")) {
 332                                 if (get_sha1_hex(line + 10, sha1))
 333                                         die("invalid unshallow line: %s", line);
 334                                 if (!lookup_object(sha1))
 335                                         die("object not found: %s", line);
 336                                 /* make sure that it is parsed as shallow */
 337                                 if (!parse_object(sha1))
 338                                         die("error in object: %s", line);
 339                                 if (unregister_shallow(sha1))
 340                                         die("no shallow found: %s", line);
 341                                 continue;
 342                         }
 343                         die("expected shallow/unshallow, got %s", line);
 344                 }
 345         } else if (!args->stateless_rpc)
 346                 send_request(args, fd[1], &req_buf);
 347
 348         if (!args->stateless_rpc) {
 349                 /* If we aren't using the stateless-rpc interface
 350                  * we don't need to retain the headers.
 351                  */
 352                 strbuf_setlen(&req_buf, 0);
 353                 state_len = 0;
 354         }
 355
 356         flushes = 0;
 357         retval = -1;
 358         while ((sha1 = get_rev())) {
 359                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 360                 if (args->verbose)
 361                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 362                 in_vain++;
 363                 if (flush_at <= ++count) {
 364                         int ack;
 365
 366                         packet_buf_flush(&req_buf);
 367                         send_request(args, fd[1], &req_buf);
 368                         strbuf_setlen(&req_buf, state_len);
 369                         flushes++;
 370                         flush_at = next_flush(args, count);
 371
 372                         /*
 373                          * We keep one window "ahead" of the other side, and
 374                          * will wait for an ACK only on the next one
 375                          */
 376                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 377                                 continue;
 378
 379                         consume_shallow_list(args, fd[0]);
 380                         do {
 381                                 ack = get_ack(fd[0], result_sha1);
 382                                 if (args->verbose && ack)
 383                                         fprintf(stderr, "got ack %d %s\n", ack,
 384                                                         sha1_to_hex(result_sha1));
 385                                 switch (ack) {
 386                                 case ACK:
 387                                         flushes = 0;
 388                                         multi_ack = 0;
 389                                         retval = 0;
 390                                         goto done;
 391                                 case ACK_common:
 392                                 case ACK_ready:
 393                                 case ACK_continue: {
 394                                         struct commit *commit =
 395                                                 lookup_commit(result_sha1);
 396                                         if (!commit)
 397                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 398                                         if (args->stateless_rpc
 399                                          && ack == ACK_common
 400                                          && !(commit->object.flags & COMMON)) {
 401                                                 /* We need to replay the have for this object
 402                                                  * on the next RPC request so the peer knows
 403                                                  * it is in common with us.
 404                                                  */
 405                                                 const char *hex = sha1_to_hex(result_sha1);
 406                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 407                                                 state_len = req_buf.len;
 408                                         }
 409                                         mark_common(commit, 0, 1);
 410                                         retval = 0;
 411                                         in_vain = 0;
 412                                         got_continue = 1;
 413                                         if (ack == ACK_ready) {
 414                                                 clear_prio_queue(&rev_list);
 415                                                 got_ready = 1;
 416                                         }
 417                                         break;
 418                                         }
 419                                 }
 420                         } while (ack);
 421                         flushes--;
 422                         if (got_continue && MAX_IN_VAIN < in_vain) {
 423                                 if (args->verbose)
 424                                         fprintf(stderr, "giving up\n");
 425                                 break; /* give up */
 426                         }
 427                 }
 428         }
 429 done:
 430         if (!got_ready || !no_done) {
 431                 packet_buf_write(&req_buf, "done\n");
 432                 send_request(args, fd[1], &req_buf);
 433         }
 434         if (args->verbose)
 435                 fprintf(stderr, "done\n");
 436         if (retval != 0) {
 437                 multi_ack = 0;
 438                 flushes++;
 439         }
 440         strbuf_release(&req_buf);
 441
 442         if (!got_ready || !no_done)
 443                 consume_shallow_list(args, fd[0]);
 444         while (flushes || multi_ack) {
 445                 int ack = get_ack(fd[0], result_sha1);
 446                 if (ack) {
 447                         if (args->verbose)
 448                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 449                                         sha1_to_hex(result_sha1));
 450                         if (ack == ACK)
 451                                 return 0;
 452                         multi_ack = 1;
 453                         continue;
 454                 }
 455                 flushes--;
 456         }
 457         /* it is no error to fetch into a completely empty repo */
 458         return count ? retval : 0;
 459 }
 460
 461 static struct commit_list *complete;
 462
 463 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 464 {
 465         struct object *o = parse_object(sha1);
 466
 467         while (o && o->type == OBJ_TAG) {
 468                 struct tag *t = (struct tag *) o;
 469                 if (!t->tagged)
 470                         break; /* broken repository */
 471                 o->flags |= COMPLETE;
 472                 o = parse_object(t->tagged->sha1);
 473         }
 474         if (o && o->type == OBJ_COMMIT) {
 475                 struct commit *commit = (struct commit *)o;
 476                 if (!(commit->object.flags & COMPLETE)) {
 477                         commit->object.flags |= COMPLETE;
 478                         commit_list_insert(commit, &complete);
 479                 }
 480         }
 481         return 0;
 482 }
 483
 484 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 485                                          unsigned long cutoff)
 486 {
 487         while (complete && cutoff <= complete->item->date) {
 488                 if (args->verbose)
 489                         fprintf(stderr, "Marking %s as complete\n",
 490                                 sha1_to_hex(complete->item->object.sha1));
 491                 pop_most_recent_commit(&complete, COMPLETE);
 492         }
 493 }
 494
 495 static void filter_refs(struct fetch_pack_args *args,
 496                         struct ref **refs,
 497                         struct ref **sought, int nr_sought)
 498 {
 499         struct ref *newlist = NULL;
 500         struct ref **newtail = &newlist;
 501         struct ref *ref, *next;
 502         int i;
 503
 504         i = 0;
 505         for (ref = *refs; ref; ref = next) {
 506                 int keep = 0;
 507                 next = ref->next;
 508
 509                 if (!memcmp(ref->name, "refs/", 5) &&
 510                     check_refname_format(ref->name, 0))
 511                         ; /* trash */
 512                 else {
 513                         while (i < nr_sought) {
 514                                 int cmp = strcmp(ref->name, sought[i]->name);
 515                                 if (cmp < 0)
 516                                         break; /* definitely do not have it */
 517                                 else if (cmp == 0) {
 518                                         keep = 1; /* definitely have it */
 519                                         sought[i]->matched = 1;
 520                                 }
 521                                 i++;
 522                         }
 523                 }
 524
 525                 if (!keep && args->fetch_all &&
 526                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 527                         keep = 1;
 528
 529                 if (keep) {
 530                         *newtail = ref;
 531                         ref->next = NULL;
 532                         newtail = &ref->next;
 533                 } else {
 534                         free(ref);
 535                 }
 536         }
 537
 538         /* Append unmatched requests to the list */
 539         if (allow_tip_sha1_in_want) {
 540                 for (i = 0; i < nr_sought; i++) {
 541                         ref = sought[i];
 542                         if (ref->matched)
 543                                 continue;
 544                         if (get_sha1_hex(ref->name, ref->old_sha1))
 545                                 continue;
 546
 547                         ref->matched = 1;
 548                         *newtail = ref;
 549                         ref->next = NULL;
 550                         newtail = &ref->next;
 551                 }
 552         }
 553         *refs = newlist;
 554 }
 555
 556 static void mark_alternate_complete(const struct ref *ref, void *unused)
 557 {
 558         mark_complete(NULL, ref->old_sha1, 0, NULL);
 559 }
 560
 561 static int everything_local(struct fetch_pack_args *args,
 562                             struct ref **refs,
 563                             struct ref **sought, int nr_sought)
 564 {
 565         struct ref *ref;
 566         int retval;
 567         unsigned long cutoff = 0;
 568
 569         save_commit_buffer = 0;
 570
 571         for (ref = *refs; ref; ref = ref->next) {
 572                 struct object *o;
 573
 574                 if (!has_sha1_file(ref->old_sha1))
 575                         continue;
 576
 577                 o = parse_object(ref->old_sha1);
 578                 if (!o)
 579                         continue;
 580
 581                 /* We already have it -- which may mean that we were
 582                  * in sync with the other side at some time after
 583                  * that (it is OK if we guess wrong here).
 584                  */
 585                 if (o->type == OBJ_COMMIT) {
 586                         struct commit *commit = (struct commit *)o;
 587                         if (!cutoff || cutoff < commit->date)
 588                                 cutoff = commit->date;
 589                 }
 590         }
 591
 592         if (!args->depth) {
 593                 for_each_ref(mark_complete, NULL);
 594                 for_each_alternate_ref(mark_alternate_complete, NULL);
 595                 commit_list_sort_by_date(&complete);
 596                 if (cutoff)
 597                         mark_recent_complete_commits(args, cutoff);
 598         }
 599
 600         /*
 601          * Mark all complete remote refs as common refs.
 602          * Don't mark them common yet; the server has to be told so first.
 603          */
 604         for (ref = *refs; ref; ref = ref->next) {
 605                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 606                                              NULL, 0);
 607
 608                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 609                         continue;
 610
 611                 if (!(o->flags & SEEN)) {
 612                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 613
 614                         mark_common((struct commit *)o, 1, 1);
 615                 }
 616         }
 617
 618         filter_refs(args, refs, sought, nr_sought);
 619
 620         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 621                 const unsigned char *remote = ref->old_sha1;
 622                 unsigned char local[20];
 623                 struct object *o;
 624
 625                 o = lookup_object(remote);
 626                 if (!o || !(o->flags & COMPLETE)) {
 627                         retval = 0;
 628                         if (!args->verbose)
 629                                 continue;
 630                         fprintf(stderr,
 631                                 "want %s (%s)\n", sha1_to_hex(remote),
 632                                 ref->name);
 633                         continue;
 634                 }
 635
 636                 hashcpy(ref->new_sha1, local);
 637                 if (!args->verbose)
 638                         continue;
 639                 fprintf(stderr,
 640                         "already have %s (%s)\n", sha1_to_hex(remote),
 641                         ref->name);
 642         }
 643         return retval;
 644 }
 645
 646 static int sideband_demux(int in, int out, void *data)
 647 {
 648         int *xd = data;
 649
 650         int ret = recv_sideband("fetch-pack", xd[0], out);
 651         close(out);
 652         return ret;
 653 }
 654
 655 static int get_pack(struct fetch_pack_args *args,
 656                     int xd[2], char **pack_lockfile)
 657 {
 658         struct async demux;
 659         const char *argv[22];
 660         char keep_arg[256];
 661         char hdr_arg[256];
 662         const char **av, *cmd_name;
 663         int do_keep = args->keep_pack;
 664         struct child_process cmd;
 665         int ret;
 666
 667         memset(&demux, 0, sizeof(demux));
 668         if (use_sideband) {
 669                 /* xd[] is talking with upload-pack; subprocess reads from
 670                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 671                  * through demux->out.
 672                  */
 673                 demux.proc = sideband_demux;
 674                 demux.data = xd;
 675                 demux.out = -1;
 676                 if (start_async(&demux))
 677                         die("fetch-pack: unable to fork off sideband"
 678                             " demultiplexer");
 679         }
 680         else
 681                 demux.out = xd[0];
 682
 683         memset(&cmd, 0, sizeof(cmd));
 684         cmd.argv = argv;
 685         av = argv;
 686         *hdr_arg = 0;
 687         if (!args->keep_pack && unpack_limit) {
 688                 struct pack_header header;
 689
 690                 if (read_pack_header(demux.out, &header))
 691                         die("protocol error: bad pack header");
 692                 snprintf(hdr_arg, sizeof(hdr_arg),
 693                          "--pack_header=%"PRIu32",%"PRIu32,
 694                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 695                 if (ntohl(header.hdr_entries) < unpack_limit)
 696                         do_keep = 0;
 697                 else
 698                         do_keep = 1;
 699         }
 700
 701         if (alternate_shallow_file) {
 702                 *av++ = "--shallow-file";
 703                 *av++ = alternate_shallow_file;
 704         }
 705
 706         if (do_keep) {
 707                 if (pack_lockfile)
 708                         cmd.out = -1;
 709                 *av++ = cmd_name = "index-pack";
 710                 *av++ = "--stdin";
 711                 if (!args->quiet && !args->no_progress)
 712                         *av++ = "-v";
 713                 if (args->use_thin_pack)
 714                         *av++ = "--fix-thin";
 715                 if (args->lock_pack || unpack_limit) {
 716                         int s = sprintf(keep_arg,
 717                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 718                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 719                                 strcpy(keep_arg + s, "localhost");
 720                         *av++ = keep_arg;
 721                 }
 722                 if (args->check_self_contained_and_connected)
 723                         *av++ = "--check-self-contained-and-connected";
 724         }
 725         else {
 726                 *av++ = cmd_name = "unpack-objects";
 727                 if (args->quiet || args->no_progress)
 728                         *av++ = "-q";
 729                 args->check_self_contained_and_connected = 0;
 730         }
 731         if (*hdr_arg)
 732                 *av++ = hdr_arg;
 733         if (fetch_fsck_objects >= 0
 734             ? fetch_fsck_objects
 735             : transfer_fsck_objects >= 0
 736             ? transfer_fsck_objects
 737             : 0)
 738                 *av++ = "--strict";
 739         *av++ = NULL;
 740
 741         cmd.in = demux.out;
 742         cmd.git_cmd = 1;
 743         if (start_command(&cmd))
 744                 die("fetch-pack: unable to fork off %s", cmd_name);
 745         if (do_keep && pack_lockfile) {
 746                 *pack_lockfile = index_pack_lockfile(cmd.out);
 747                 close(cmd.out);
 748         }
 749
 750         if (!use_sideband)
 751                 /* Closed by start_command() */
 752                 xd[0] = -1;
 753
 754         ret = finish_command(&cmd);
 755         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 756                 args->self_contained_and_connected =
 757                         args->check_self_contained_and_connected &&
 758                         ret == 0;
 759         else
 760                 die("%s failed", cmd_name);
 761         if (use_sideband && finish_async(&demux))
 762                 die("error in sideband demultiplexer");
 763         return 0;
 764 }
 765
 766 static int cmp_ref_by_name(const void *a_, const void *b_)
 767 {
 768         const struct ref *a = *((const struct ref **)a_);
 769         const struct ref *b = *((const struct ref **)b_);
 770         return strcmp(a->name, b->name);
 771 }
 772
 773 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 774                                  int fd[2],
 775                                  const struct ref *orig_ref,
 776                                  struct ref **sought, int nr_sought,
 777                                  struct shallow_info *si,
 778                                  char **pack_lockfile)
 779 {
 780         struct ref *ref = copy_ref_list(orig_ref);
 781         unsigned char sha1[20];
 782         const char *agent_feature;
 783         int agent_len;
 784
 785         sort_ref_list(&ref, ref_compare_name);
 786         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 787
 788         if (is_repository_shallow() && !server_supports("shallow"))
 789                 die("Server does not support shallow clients");
 790         if (server_supports("multi_ack_detailed")) {
 791                 if (args->verbose)
 792                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 793                 multi_ack = 2;
 794                 if (server_supports("no-done")) {
 795                         if (args->verbose)
 796                                 fprintf(stderr, "Server supports no-done\n");
 797                         if (args->stateless_rpc)
 798                                 no_done = 1;
 799                 }
 800         }
 801         else if (server_supports("multi_ack")) {
 802                 if (args->verbose)
 803                         fprintf(stderr, "Server supports multi_ack\n");
 804                 multi_ack = 1;
 805         }
 806         if (server_supports("side-band-64k")) {
 807                 if (args->verbose)
 808                         fprintf(stderr, "Server supports side-band-64k\n");
 809                 use_sideband = 2;
 810         }
 811         else if (server_supports("side-band")) {
 812                 if (args->verbose)
 813                         fprintf(stderr, "Server supports side-band\n");
 814                 use_sideband = 1;
 815         }
 816         if (server_supports("allow-tip-sha1-in-want")) {
 817                 if (args->verbose)
 818                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 819                 allow_tip_sha1_in_want = 1;
 820         }
 821         if (!server_supports("thin-pack"))
 822                 args->use_thin_pack = 0;
 823         if (!server_supports("no-progress"))
 824                 args->no_progress = 0;
 825         if (!server_supports("include-tag"))
 826                 args->include_tag = 0;
 827         if (server_supports("ofs-delta")) {
 828                 if (args->verbose)
 829                         fprintf(stderr, "Server supports ofs-delta\n");
 830         } else
 831                 prefer_ofs_delta = 0;
 832
 833         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 834                 agent_supported = 1;
 835                 if (args->verbose && agent_len)
 836                         fprintf(stderr, "Server version is %.*s\n",
 837                                 agent_len, agent_feature);
 838         }
 839
 840         if (everything_local(args, &ref, sought, nr_sought)) {
 841                 packet_flush(fd[1]);
 842                 goto all_done;
 843         }
 844         if (find_common(args, fd, sha1, ref) < 0)
 845                 if (!args->keep_pack)
 846                         /* When cloning, it is not unusual to have
 847                          * no common commit.
 848                          */
 849                         warning("no common commits");
 850
 851         if (args->stateless_rpc)
 852                 packet_flush(fd[1]);
 853         if (args->depth > 0)
 854                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 855                                         NULL);
 856         else if (si->nr_ours || si->nr_theirs)
 857                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 858         else
 859                 alternate_shallow_file = NULL;
 860         if (get_pack(args, fd, pack_lockfile))
 861                 die("git fetch-pack: fetch failed.");
 862
 863  all_done:
 864         return ref;
 865 }
 866
 867 static int fetch_pack_config(const char *var, const char *value, void *cb)
 868 {
 869         if (strcmp(var, "fetch.unpacklimit") == 0) {
 870                 fetch_unpack_limit = git_config_int(var, value);
 871                 return 0;
 872         }
 873
 874         if (strcmp(var, "transfer.unpacklimit") == 0) {
 875                 transfer_unpack_limit = git_config_int(var, value);
 876                 return 0;
 877         }
 878
 879         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 880                 prefer_ofs_delta = git_config_bool(var, value);
 881                 return 0;
 882         }
 883
 884         if (!strcmp(var, "fetch.fsckobjects")) {
 885                 fetch_fsck_objects = git_config_bool(var, value);
 886                 return 0;
 887         }
 888
 889         if (!strcmp(var, "transfer.fsckobjects")) {
 890                 transfer_fsck_objects = git_config_bool(var, value);
 891                 return 0;
 892         }
 893
 894         return git_default_config(var, value, cb);
 895 }
 896
 897 static void fetch_pack_setup(void)
 898 {
 899         static int did_setup;
 900         if (did_setup)
 901                 return;
 902         git_config(fetch_pack_config, NULL);
 903         if (0 <= transfer_unpack_limit)
 904                 unpack_limit = transfer_unpack_limit;
 905         else if (0 <= fetch_unpack_limit)
 906                 unpack_limit = fetch_unpack_limit;
 907         did_setup = 1;
 908 }
 909
 910 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 911 {
 912         struct string_list names = STRING_LIST_INIT_NODUP;
 913         int src, dst;
 914
 915         for (src = dst = 0; src < nr; src++) {
 916                 struct string_list_item *item;
 917                 item = string_list_insert(&names, ref[src]->name);
 918                 if (item->util)
 919                         continue; /* already have it */
 920                 item->util = ref[src];
 921                 if (src != dst)
 922                         ref[dst] = ref[src];
 923                 dst++;
 924         }
 925         for (src = dst; src < nr; src++)
 926                 ref[src] = NULL;
 927         string_list_clear(&names, 0);
 928         return dst;
 929 }
 930
 931 static void update_shallow(struct fetch_pack_args *args,
 932                            struct ref **sought, int nr_sought,
 933                            struct shallow_info *si)
 934 {
 935         struct sha1_array ref = SHA1_ARRAY_INIT;
 936         int *status;
 937         int i;
 938
 939         if (args->depth > 0 && alternate_shallow_file) {
 940                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 941                         unlink_or_warn(git_path("shallow"));
 942                         rollback_lock_file(&shallow_lock);
 943                 } else
 944                         commit_lock_file(&shallow_lock);
 945                 return;
 946         }
 947
 948         if (!si->shallow || !si->shallow->nr)
 949                 return;
 950
 951         if (alternate_shallow_file) {
 952                 /*
 953                  * The temporary shallow file is only useful for
 954                  * index-pack and unpack-objects because it may
 955                  * contain more roots than we want. Delete it.
 956                  */
 957                 if (*alternate_shallow_file)
 958                         unlink(alternate_shallow_file);
 959                 free((char *)alternate_shallow_file);
 960         }
 961
 962         if (args->cloning) {
 963                 /*
 964                  * remote is shallow, but this is a clone, there are
 965                  * no objects in repo to worry about. Accept any
 966                  * shallow points that exist in the pack (iow in repo
 967                  * after get_pack() and reprepare_packed_git())
 968                  */
 969                 struct sha1_array extra = SHA1_ARRAY_INIT;
 970                 unsigned char (*sha1)[20] = si->shallow->sha1;
 971                 for (i = 0; i < si->shallow->nr; i++)
 972                         if (has_sha1_file(sha1[i]))
 973                                 sha1_array_append(&extra, sha1[i]);
 974                 if (extra.nr) {
 975                         setup_alternate_shallow(&shallow_lock,
 976                                                 &alternate_shallow_file,
 977                                                 &extra);
 978                         commit_lock_file(&shallow_lock);
 979                 }
 980                 sha1_array_clear(&extra);
 981                 return;
 982         }
 983
 984         if (!si->nr_ours && !si->nr_theirs)
 985                 return;
 986
 987         remove_nonexistent_theirs_shallow(si);
 988         if (!si->nr_ours && !si->nr_theirs)
 989                 return;
 990         for (i = 0; i < nr_sought; i++)
 991                 sha1_array_append(&ref, sought[i]->old_sha1);
 992         si->ref = &ref;
 993
 994         if (args->update_shallow) {
 995                 /*
 996                  * remote is also shallow, .git/shallow may be updated
 997                  * so all refs can be accepted. Make sure we only add
 998                  * shallow roots that are actually reachable from new
 999                  * refs.
1000                  */
1001                 struct sha1_array extra = SHA1_ARRAY_INIT;
1002                 unsigned char (*sha1)[20] = si->shallow->sha1;
1003                 assign_shallow_commits_to_refs(si, NULL, NULL);
1004                 if (!si->nr_ours && !si->nr_theirs) {
1005                         sha1_array_clear(&ref);
1006                         return;
1007                 }
1008                 for (i = 0; i < si->nr_ours; i++)
1009                         sha1_array_append(&extra, sha1[si->ours[i]]);
1010                 for (i = 0; i < si->nr_theirs; i++)
1011                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1012                 setup_alternate_shallow(&shallow_lock,
1013                                         &alternate_shallow_file,
1014                                         &extra);
1015                 commit_lock_file(&shallow_lock);
1016                 sha1_array_clear(&extra);
1017                 sha1_array_clear(&ref);
1018                 return;
1019         }
1020
1021         /*
1022          * remote is also shallow, check what ref is safe to update
1023          * without updating .git/shallow
1024          */
1025         status = xcalloc(nr_sought, sizeof(*status));
1026         assign_shallow_commits_to_refs(si, NULL, status);
1027         if (si->nr_ours || si->nr_theirs) {
1028                 for (i = 0; i < nr_sought; i++)
1029                         if (status[i])
1030                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1031         }
1032         free(status);
1033         sha1_array_clear(&ref);
1034 }
1035
1036 struct ref *fetch_pack(struct fetch_pack_args *args,
1037                        int fd[], struct child_process *conn,
1038                        const struct ref *ref,
1039                        const char *dest,
1040                        struct ref **sought, int nr_sought,
1041                        struct sha1_array *shallow,
1042                        char **pack_lockfile)
1043 {
1044         struct ref *ref_cpy;
1045         struct shallow_info si;
1046
1047         fetch_pack_setup();
1048         if (nr_sought)
1049                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1050
1051         if (!ref) {
1052                 packet_flush(fd[1]);
1053                 die("no matching remote head");
1054         }
1055         prepare_shallow_info(&si, shallow);
1056         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1057                                 &si, pack_lockfile);
1058         reprepare_packed_git();
1059         update_shallow(args, sought, nr_sought, &si);
1060         clear_shallow_info(&si);
1061         return ref_cpy;
1062 }