git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  47
  48 static void rev_list_push(struct commit *commit, int mark)
  49 {
  50         if (!(commit->object.flags & mark)) {
  51                 commit->object.flags |= mark;
  52
  53                 if (parse_commit(commit))
  54                         return;
  55
  56                 prio_queue_put(&rev_list, commit);
  57
  58                 if (!(commit->object.flags & COMMON))
  59                         non_common_revs++;
  60         }
  61 }
  62
  63 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  64 {
  65         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  66
  67         if (o && o->type == OBJ_COMMIT)
  68                 rev_list_push((struct commit *)o, SEEN);
  69
  70         return 0;
  71 }
  72
  73 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  74 {
  75         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  76
  77         if (o && o->type == OBJ_COMMIT)
  78                 clear_commit_marks((struct commit *)o,
  79                                    COMMON | COMMON_REF | SEEN | POPPED);
  80         return 0;
  81 }
  82
  83 /*
  84    This function marks a rev and its ancestors as common.
  85    In some cases, it is desirable to mark only the ancestors (for example
  86    when only the server does not yet know that they are common).
  87 */
  88
  89 static void mark_common(struct commit *commit,
  90                 int ancestors_only, int dont_parse)
  91 {
  92         if (commit != NULL && !(commit->object.flags & COMMON)) {
  93                 struct object *o = (struct object *)commit;
  94
  95                 if (!ancestors_only)
  96                         o->flags |= COMMON;
  97
  98                 if (!(o->flags & SEEN))
  99                         rev_list_push(commit, SEEN);
 100                 else {
 101                         struct commit_list *parents;
 102
 103                         if (!ancestors_only && !(o->flags & POPPED))
 104                                 non_common_revs--;
 105                         if (!o->parsed && !dont_parse)
 106                                 if (parse_commit(commit))
 107                                         return;
 108
 109                         for (parents = commit->parents;
 110                                         parents;
 111                                         parents = parents->next)
 112                                 mark_common(parents->item, 0, dont_parse);
 113                 }
 114         }
 115 }
 116
 117 /*
 118   Get the next rev to send, ignoring the common.
 119 */
 120
 121 static const unsigned char *get_rev(void)
 122 {
 123         struct commit *commit = NULL;
 124
 125         while (commit == NULL) {
 126                 unsigned int mark;
 127                 struct commit_list *parents;
 128
 129                 if (rev_list.nr == 0 || non_common_revs == 0)
 130                         return NULL;
 131
 132                 commit = prio_queue_get(&rev_list);
 133                 parse_commit(commit);
 134                 parents = commit->parents;
 135
 136                 commit->object.flags |= POPPED;
 137                 if (!(commit->object.flags & COMMON))
 138                         non_common_revs--;
 139
 140                 if (commit->object.flags & COMMON) {
 141                         /* do not send "have", and ignore ancestors */
 142                         commit = NULL;
 143                         mark = COMMON | SEEN;
 144                 } else if (commit->object.flags & COMMON_REF)
 145                         /* send "have", and ignore ancestors */
 146                         mark = COMMON | SEEN;
 147                 else
 148                         /* send "have", also for its ancestors */
 149                         mark = SEEN;
 150
 151                 while (parents) {
 152                         if (!(parents->item->object.flags & SEEN))
 153                                 rev_list_push(parents->item, mark);
 154                         if (mark & COMMON)
 155                                 mark_common(parents->item, 1, 0);
 156                         parents = parents->next;
 157                 }
 158         }
 159
 160         return commit->object.sha1;
 161 }
 162
 163 enum ack_type {
 164         NAK = 0,
 165         ACK,
 166         ACK_continue,
 167         ACK_common,
 168         ACK_ready
 169 };
 170
 171 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 172 {
 173         if (args->stateless_rpc && args->depth > 0) {
 174                 /* If we sent a depth we will get back "duplicate"
 175                  * shallow and unshallow commands every time there
 176                  * is a block of have lines exchanged.
 177                  */
 178                 char *line;
 179                 while ((line = packet_read_line(fd, NULL))) {
 180                         if (starts_with(line, "shallow "))
 181                                 continue;
 182                         if (starts_with(line, "unshallow "))
 183                                 continue;
 184                         die("git fetch-pack: expected shallow list");
 185                 }
 186         }
 187 }
 188
 189 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 190 {
 191         int len;
 192         char *line = packet_read_line(fd, &len);
 193         const char *arg;
 194
 195         if (!len)
 196                 die("git fetch-pack: expected ACK/NAK, got EOF");
 197         if (!strcmp(line, "NAK"))
 198                 return NAK;
 199         if (skip_prefix(line, "ACK ", &arg)) {
 200                 if (!get_sha1_hex(arg, result_sha1)) {
 201                         arg += 40;
 202                         len -= arg - line;
 203                         if (len < 1)
 204                                 return ACK;
 205                         if (strstr(arg, "continue"))
 206                                 return ACK_continue;
 207                         if (strstr(arg, "common"))
 208                                 return ACK_common;
 209                         if (strstr(arg, "ready"))
 210                                 return ACK_ready;
 211                         return ACK;
 212                 }
 213         }
 214         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 215 }
 216
 217 static void send_request(struct fetch_pack_args *args,
 218                          int fd, struct strbuf *buf)
 219 {
 220         if (args->stateless_rpc) {
 221                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 222                 packet_flush(fd);
 223         } else
 224                 write_or_die(fd, buf->buf, buf->len);
 225 }
 226
 227 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 228 {
 229         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 230 }
 231
 232 #define INITIAL_FLUSH 16
 233 #define PIPESAFE_FLUSH 32
 234 #define LARGE_FLUSH 1024
 235
 236 static int next_flush(struct fetch_pack_args *args, int count)
 237 {
 238         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 239
 240         if (count < flush_limit)
 241                 count <<= 1;
 242         else
 243                 count += flush_limit;
 244         return count;
 245 }
 246
 247 static int find_common(struct fetch_pack_args *args,
 248                        int fd[2], unsigned char *result_sha1,
 249                        struct ref *refs)
 250 {
 251         int fetching;
 252         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 253         const unsigned char *sha1;
 254         unsigned in_vain = 0;
 255         int got_continue = 0;
 256         int got_ready = 0;
 257         struct strbuf req_buf = STRBUF_INIT;
 258         size_t state_len = 0;
 259
 260         if (args->stateless_rpc && multi_ack == 1)
 261                 die("--stateless-rpc requires multi_ack_detailed");
 262         if (marked)
 263                 for_each_ref(clear_marks, NULL);
 264         marked = 1;
 265
 266         for_each_ref(rev_list_insert_ref, NULL);
 267         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 268
 269         fetching = 0;
 270         for ( ; refs ; refs = refs->next) {
 271                 unsigned char *remote = refs->old_sha1;
 272                 const char *remote_hex;
 273                 struct object *o;
 274
 275                 /*
 276                  * If that object is complete (i.e. it is an ancestor of a
 277                  * local ref), we tell them we have it but do not have to
 278                  * tell them about its ancestors, which they already know
 279                  * about.
 280                  *
 281                  * We use lookup_object here because we are only
 282                  * interested in the case we *know* the object is
 283                  * reachable and we have already scanned it.
 284                  */
 285                 if (((o = lookup_object(remote)) != NULL) &&
 286                                 (o->flags & COMPLETE)) {
 287                         continue;
 288                 }
 289
 290                 remote_hex = sha1_to_hex(remote);
 291                 if (!fetching) {
 292                         struct strbuf c = STRBUF_INIT;
 293                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 294                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 295                         if (no_done)            strbuf_addstr(&c, " no-done");
 296                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 297                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 298                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 299                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 300                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 301                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 302                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 303                                                             git_user_agent_sanitized());
 304                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 305                         strbuf_release(&c);
 306                 } else
 307                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 308                 fetching++;
 309         }
 310
 311         if (!fetching) {
 312                 strbuf_release(&req_buf);
 313                 packet_flush(fd[1]);
 314                 return 1;
 315         }
 316
 317         if (is_repository_shallow())
 318                 write_shallow_commits(&req_buf, 1, NULL);
 319         if (args->depth > 0)
 320                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 321         packet_buf_flush(&req_buf);
 322         state_len = req_buf.len;
 323
 324         if (args->depth > 0) {
 325                 char *line;
 326                 const char *arg;
 327                 unsigned char sha1[20];
 328
 329                 send_request(args, fd[1], &req_buf);
 330                 while ((line = packet_read_line(fd[0], NULL))) {
 331                         if (skip_prefix(line, "shallow ", &arg)) {
 332                                 if (get_sha1_hex(arg, sha1))
 333                                         die("invalid shallow line: %s", line);
 334                                 register_shallow(sha1);
 335                                 continue;
 336                         }
 337                         if (skip_prefix(line, "unshallow ", &arg)) {
 338                                 if (get_sha1_hex(arg, sha1))
 339                                         die("invalid unshallow line: %s", line);
 340                                 if (!lookup_object(sha1))
 341                                         die("object not found: %s", line);
 342                                 /* make sure that it is parsed as shallow */
 343                                 if (!parse_object(sha1))
 344                                         die("error in object: %s", line);
 345                                 if (unregister_shallow(sha1))
 346                                         die("no shallow found: %s", line);
 347                                 continue;
 348                         }
 349                         die("expected shallow/unshallow, got %s", line);
 350                 }
 351         } else if (!args->stateless_rpc)
 352                 send_request(args, fd[1], &req_buf);
 353
 354         if (!args->stateless_rpc) {
 355                 /* If we aren't using the stateless-rpc interface
 356                  * we don't need to retain the headers.
 357                  */
 358                 strbuf_setlen(&req_buf, 0);
 359                 state_len = 0;
 360         }
 361
 362         flushes = 0;
 363         retval = -1;
 364         while ((sha1 = get_rev())) {
 365                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 366                 if (args->verbose)
 367                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 368                 in_vain++;
 369                 if (flush_at <= ++count) {
 370                         int ack;
 371
 372                         packet_buf_flush(&req_buf);
 373                         send_request(args, fd[1], &req_buf);
 374                         strbuf_setlen(&req_buf, state_len);
 375                         flushes++;
 376                         flush_at = next_flush(args, count);
 377
 378                         /*
 379                          * We keep one window "ahead" of the other side, and
 380                          * will wait for an ACK only on the next one
 381                          */
 382                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 383                                 continue;
 384
 385                         consume_shallow_list(args, fd[0]);
 386                         do {
 387                                 ack = get_ack(fd[0], result_sha1);
 388                                 if (args->verbose && ack)
 389                                         fprintf(stderr, "got ack %d %s\n", ack,
 390                                                         sha1_to_hex(result_sha1));
 391                                 switch (ack) {
 392                                 case ACK:
 393                                         flushes = 0;
 394                                         multi_ack = 0;
 395                                         retval = 0;
 396                                         goto done;
 397                                 case ACK_common:
 398                                 case ACK_ready:
 399                                 case ACK_continue: {
 400                                         struct commit *commit =
 401                                                 lookup_commit(result_sha1);
 402                                         if (!commit)
 403                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 404                                         if (args->stateless_rpc
 405                                          && ack == ACK_common
 406                                          && !(commit->object.flags & COMMON)) {
 407                                                 /* We need to replay the have for this object
 408                                                  * on the next RPC request so the peer knows
 409                                                  * it is in common with us.
 410                                                  */
 411                                                 const char *hex = sha1_to_hex(result_sha1);
 412                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 413                                                 state_len = req_buf.len;
 414                                         }
 415                                         mark_common(commit, 0, 1);
 416                                         retval = 0;
 417                                         in_vain = 0;
 418                                         got_continue = 1;
 419                                         if (ack == ACK_ready) {
 420                                                 clear_prio_queue(&rev_list);
 421                                                 got_ready = 1;
 422                                         }
 423                                         break;
 424                                         }
 425                                 }
 426                         } while (ack);
 427                         flushes--;
 428                         if (got_continue && MAX_IN_VAIN < in_vain) {
 429                                 if (args->verbose)
 430                                         fprintf(stderr, "giving up\n");
 431                                 break; /* give up */
 432                         }
 433                 }
 434         }
 435 done:
 436         if (!got_ready || !no_done) {
 437                 packet_buf_write(&req_buf, "done\n");
 438                 send_request(args, fd[1], &req_buf);
 439         }
 440         if (args->verbose)
 441                 fprintf(stderr, "done\n");
 442         if (retval != 0) {
 443                 multi_ack = 0;
 444                 flushes++;
 445         }
 446         strbuf_release(&req_buf);
 447
 448         if (!got_ready || !no_done)
 449                 consume_shallow_list(args, fd[0]);
 450         while (flushes || multi_ack) {
 451                 int ack = get_ack(fd[0], result_sha1);
 452                 if (ack) {
 453                         if (args->verbose)
 454                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 455                                         sha1_to_hex(result_sha1));
 456                         if (ack == ACK)
 457                                 return 0;
 458                         multi_ack = 1;
 459                         continue;
 460                 }
 461                 flushes--;
 462         }
 463         /* it is no error to fetch into a completely empty repo */
 464         return count ? retval : 0;
 465 }
 466
 467 static struct commit_list *complete;
 468
 469 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 470 {
 471         struct object *o = parse_object(sha1);
 472
 473         while (o && o->type == OBJ_TAG) {
 474                 struct tag *t = (struct tag *) o;
 475                 if (!t->tagged)
 476                         break; /* broken repository */
 477                 o->flags |= COMPLETE;
 478                 o = parse_object(t->tagged->sha1);
 479         }
 480         if (o && o->type == OBJ_COMMIT) {
 481                 struct commit *commit = (struct commit *)o;
 482                 if (!(commit->object.flags & COMPLETE)) {
 483                         commit->object.flags |= COMPLETE;
 484                         commit_list_insert(commit, &complete);
 485                 }
 486         }
 487         return 0;
 488 }
 489
 490 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 491                                          unsigned long cutoff)
 492 {
 493         while (complete && cutoff <= complete->item->date) {
 494                 if (args->verbose)
 495                         fprintf(stderr, "Marking %s as complete\n",
 496                                 sha1_to_hex(complete->item->object.sha1));
 497                 pop_most_recent_commit(&complete, COMPLETE);
 498         }
 499 }
 500
 501 static void filter_refs(struct fetch_pack_args *args,
 502                         struct ref **refs,
 503                         struct ref **sought, int nr_sought)
 504 {
 505         struct ref *newlist = NULL;
 506         struct ref **newtail = &newlist;
 507         struct ref *ref, *next;
 508         int i;
 509
 510         i = 0;
 511         for (ref = *refs; ref; ref = next) {
 512                 int keep = 0;
 513                 next = ref->next;
 514
 515                 if (starts_with(ref->name, "refs/") &&
 516                     check_refname_format(ref->name, 0))
 517                         ; /* trash */
 518                 else {
 519                         while (i < nr_sought) {
 520                                 int cmp = strcmp(ref->name, sought[i]->name);
 521                                 if (cmp < 0)
 522                                         break; /* definitely do not have it */
 523                                 else if (cmp == 0) {
 524                                         keep = 1; /* definitely have it */
 525                                         sought[i]->matched = 1;
 526                                 }
 527                                 i++;
 528                         }
 529                 }
 530
 531                 if (!keep && args->fetch_all &&
 532                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 533                         keep = 1;
 534
 535                 if (keep) {
 536                         *newtail = ref;
 537                         ref->next = NULL;
 538                         newtail = &ref->next;
 539                 } else {
 540                         free(ref);
 541                 }
 542         }
 543
 544         /* Append unmatched requests to the list */
 545         if (allow_tip_sha1_in_want) {
 546                 for (i = 0; i < nr_sought; i++) {
 547                         unsigned char sha1[20];
 548
 549                         ref = sought[i];
 550                         if (ref->matched)
 551                                 continue;
 552                         if (get_sha1_hex(ref->name, sha1) ||
 553                             ref->name[40] != '\0' ||
 554                             hashcmp(sha1, ref->old_sha1))
 555                                 continue;
 556
 557                         ref->matched = 1;
 558                         *newtail = copy_ref(ref);
 559                         newtail = &(*newtail)->next;
 560                 }
 561         }
 562         *refs = newlist;
 563 }
 564
 565 static void mark_alternate_complete(const struct ref *ref, void *unused)
 566 {
 567         mark_complete(NULL, ref->old_sha1, 0, NULL);
 568 }
 569
 570 static int everything_local(struct fetch_pack_args *args,
 571                             struct ref **refs,
 572                             struct ref **sought, int nr_sought)
 573 {
 574         struct ref *ref;
 575         int retval;
 576         unsigned long cutoff = 0;
 577
 578         save_commit_buffer = 0;
 579
 580         for (ref = *refs; ref; ref = ref->next) {
 581                 struct object *o;
 582
 583                 if (!has_sha1_file(ref->old_sha1))
 584                         continue;
 585
 586                 o = parse_object(ref->old_sha1);
 587                 if (!o)
 588                         continue;
 589
 590                 /* We already have it -- which may mean that we were
 591                  * in sync with the other side at some time after
 592                  * that (it is OK if we guess wrong here).
 593                  */
 594                 if (o->type == OBJ_COMMIT) {
 595                         struct commit *commit = (struct commit *)o;
 596                         if (!cutoff || cutoff < commit->date)
 597                                 cutoff = commit->date;
 598                 }
 599         }
 600
 601         if (!args->depth) {
 602                 for_each_ref(mark_complete, NULL);
 603                 for_each_alternate_ref(mark_alternate_complete, NULL);
 604                 commit_list_sort_by_date(&complete);
 605                 if (cutoff)
 606                         mark_recent_complete_commits(args, cutoff);
 607         }
 608
 609         /*
 610          * Mark all complete remote refs as common refs.
 611          * Don't mark them common yet; the server has to be told so first.
 612          */
 613         for (ref = *refs; ref; ref = ref->next) {
 614                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 615                                              NULL, 0);
 616
 617                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 618                         continue;
 619
 620                 if (!(o->flags & SEEN)) {
 621                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 622
 623                         mark_common((struct commit *)o, 1, 1);
 624                 }
 625         }
 626
 627         filter_refs(args, refs, sought, nr_sought);
 628
 629         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 630                 const unsigned char *remote = ref->old_sha1;
 631                 unsigned char local[20];
 632                 struct object *o;
 633
 634                 o = lookup_object(remote);
 635                 if (!o || !(o->flags & COMPLETE)) {
 636                         retval = 0;
 637                         if (!args->verbose)
 638                                 continue;
 639                         fprintf(stderr,
 640                                 "want %s (%s)\n", sha1_to_hex(remote),
 641                                 ref->name);
 642                         continue;
 643                 }
 644
 645                 hashcpy(ref->new_sha1, local);
 646                 if (!args->verbose)
 647                         continue;
 648                 fprintf(stderr,
 649                         "already have %s (%s)\n", sha1_to_hex(remote),
 650                         ref->name);
 651         }
 652         return retval;
 653 }
 654
 655 static int sideband_demux(int in, int out, void *data)
 656 {
 657         int *xd = data;
 658
 659         int ret = recv_sideband("fetch-pack", xd[0], out);
 660         close(out);
 661         return ret;
 662 }
 663
 664 static int get_pack(struct fetch_pack_args *args,
 665                     int xd[2], char **pack_lockfile)
 666 {
 667         struct async demux;
 668         const char *argv[22];
 669         char keep_arg[256];
 670         char hdr_arg[256];
 671         const char **av, *cmd_name;
 672         int do_keep = args->keep_pack;
 673         struct child_process cmd = CHILD_PROCESS_INIT;
 674         int ret;
 675
 676         memset(&demux, 0, sizeof(demux));
 677         if (use_sideband) {
 678                 /* xd[] is talking with upload-pack; subprocess reads from
 679                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 680                  * through demux->out.
 681                  */
 682                 demux.proc = sideband_demux;
 683                 demux.data = xd;
 684                 demux.out = -1;
 685                 if (start_async(&demux))
 686                         die("fetch-pack: unable to fork off sideband"
 687                             " demultiplexer");
 688         }
 689         else
 690                 demux.out = xd[0];
 691
 692         cmd.argv = argv;
 693         av = argv;
 694         *hdr_arg = 0;
 695         if (!args->keep_pack && unpack_limit) {
 696                 struct pack_header header;
 697
 698                 if (read_pack_header(demux.out, &header))
 699                         die("protocol error: bad pack header");
 700                 snprintf(hdr_arg, sizeof(hdr_arg),
 701                          "--pack_header=%"PRIu32",%"PRIu32,
 702                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 703                 if (ntohl(header.hdr_entries) < unpack_limit)
 704                         do_keep = 0;
 705                 else
 706                         do_keep = 1;
 707         }
 708
 709         if (alternate_shallow_file) {
 710                 *av++ = "--shallow-file";
 711                 *av++ = alternate_shallow_file;
 712         }
 713
 714         if (do_keep) {
 715                 if (pack_lockfile)
 716                         cmd.out = -1;
 717                 *av++ = cmd_name = "index-pack";
 718                 *av++ = "--stdin";
 719                 if (!args->quiet && !args->no_progress)
 720                         *av++ = "-v";
 721                 if (args->use_thin_pack)
 722                         *av++ = "--fix-thin";
 723                 if (args->lock_pack || unpack_limit) {
 724                         int s = sprintf(keep_arg,
 725                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 726                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 727                                 strcpy(keep_arg + s, "localhost");
 728                         *av++ = keep_arg;
 729                 }
 730                 if (args->check_self_contained_and_connected)
 731                         *av++ = "--check-self-contained-and-connected";
 732         }
 733         else {
 734                 *av++ = cmd_name = "unpack-objects";
 735                 if (args->quiet || args->no_progress)
 736                         *av++ = "-q";
 737                 args->check_self_contained_and_connected = 0;
 738         }
 739         if (*hdr_arg)
 740                 *av++ = hdr_arg;
 741         if (fetch_fsck_objects >= 0
 742             ? fetch_fsck_objects
 743             : transfer_fsck_objects >= 0
 744             ? transfer_fsck_objects
 745             : 0)
 746                 *av++ = "--strict";
 747         *av++ = NULL;
 748
 749         cmd.in = demux.out;
 750         cmd.git_cmd = 1;
 751         if (start_command(&cmd))
 752                 die("fetch-pack: unable to fork off %s", cmd_name);
 753         if (do_keep && pack_lockfile) {
 754                 *pack_lockfile = index_pack_lockfile(cmd.out);
 755                 close(cmd.out);
 756         }
 757
 758         if (!use_sideband)
 759                 /* Closed by start_command() */
 760                 xd[0] = -1;
 761
 762         ret = finish_command(&cmd);
 763         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 764                 args->self_contained_and_connected =
 765                         args->check_self_contained_and_connected &&
 766                         ret == 0;
 767         else
 768                 die("%s failed", cmd_name);
 769         if (use_sideband && finish_async(&demux))
 770                 die("error in sideband demultiplexer");
 771         return 0;
 772 }
 773
 774 static int cmp_ref_by_name(const void *a_, const void *b_)
 775 {
 776         const struct ref *a = *((const struct ref **)a_);
 777         const struct ref *b = *((const struct ref **)b_);
 778         return strcmp(a->name, b->name);
 779 }
 780
 781 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 782                                  int fd[2],
 783                                  const struct ref *orig_ref,
 784                                  struct ref **sought, int nr_sought,
 785                                  struct shallow_info *si,
 786                                  char **pack_lockfile)
 787 {
 788         struct ref *ref = copy_ref_list(orig_ref);
 789         unsigned char sha1[20];
 790         const char *agent_feature;
 791         int agent_len;
 792
 793         sort_ref_list(&ref, ref_compare_name);
 794         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 795
 796         if (is_repository_shallow() && !server_supports("shallow"))
 797                 die("Server does not support shallow clients");
 798         if (server_supports("multi_ack_detailed")) {
 799                 if (args->verbose)
 800                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 801                 multi_ack = 2;
 802                 if (server_supports("no-done")) {
 803                         if (args->verbose)
 804                                 fprintf(stderr, "Server supports no-done\n");
 805                         if (args->stateless_rpc)
 806                                 no_done = 1;
 807                 }
 808         }
 809         else if (server_supports("multi_ack")) {
 810                 if (args->verbose)
 811                         fprintf(stderr, "Server supports multi_ack\n");
 812                 multi_ack = 1;
 813         }
 814         if (server_supports("side-band-64k")) {
 815                 if (args->verbose)
 816                         fprintf(stderr, "Server supports side-band-64k\n");
 817                 use_sideband = 2;
 818         }
 819         else if (server_supports("side-band")) {
 820                 if (args->verbose)
 821                         fprintf(stderr, "Server supports side-band\n");
 822                 use_sideband = 1;
 823         }
 824         if (server_supports("allow-tip-sha1-in-want")) {
 825                 if (args->verbose)
 826                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 827                 allow_tip_sha1_in_want = 1;
 828         }
 829         if (!server_supports("thin-pack"))
 830                 args->use_thin_pack = 0;
 831         if (!server_supports("no-progress"))
 832                 args->no_progress = 0;
 833         if (!server_supports("include-tag"))
 834                 args->include_tag = 0;
 835         if (server_supports("ofs-delta")) {
 836                 if (args->verbose)
 837                         fprintf(stderr, "Server supports ofs-delta\n");
 838         } else
 839                 prefer_ofs_delta = 0;
 840
 841         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 842                 agent_supported = 1;
 843                 if (args->verbose && agent_len)
 844                         fprintf(stderr, "Server version is %.*s\n",
 845                                 agent_len, agent_feature);
 846         }
 847
 848         if (everything_local(args, &ref, sought, nr_sought)) {
 849                 packet_flush(fd[1]);
 850                 goto all_done;
 851         }
 852         if (find_common(args, fd, sha1, ref) < 0)
 853                 if (!args->keep_pack)
 854                         /* When cloning, it is not unusual to have
 855                          * no common commit.
 856                          */
 857                         warning("no common commits");
 858
 859         if (args->stateless_rpc)
 860                 packet_flush(fd[1]);
 861         if (args->depth > 0)
 862                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 863                                         NULL);
 864         else if (si->nr_ours || si->nr_theirs)
 865                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 866         else
 867                 alternate_shallow_file = NULL;
 868         if (get_pack(args, fd, pack_lockfile))
 869                 die("git fetch-pack: fetch failed.");
 870
 871  all_done:
 872         return ref;
 873 }
 874
 875 static void fetch_pack_config(void)
 876 {
 877         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 878         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 879         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 880         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 881         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 882
 883         git_config(git_default_config, NULL);
 884 }
 885
 886 static void fetch_pack_setup(void)
 887 {
 888         static int did_setup;
 889         if (did_setup)
 890                 return;
 891         fetch_pack_config();
 892         if (0 <= transfer_unpack_limit)
 893                 unpack_limit = transfer_unpack_limit;
 894         else if (0 <= fetch_unpack_limit)
 895                 unpack_limit = fetch_unpack_limit;
 896         did_setup = 1;
 897 }
 898
 899 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 900 {
 901         struct string_list names = STRING_LIST_INIT_NODUP;
 902         int src, dst;
 903
 904         for (src = dst = 0; src < nr; src++) {
 905                 struct string_list_item *item;
 906                 item = string_list_insert(&names, ref[src]->name);
 907                 if (item->util)
 908                         continue; /* already have it */
 909                 item->util = ref[src];
 910                 if (src != dst)
 911                         ref[dst] = ref[src];
 912                 dst++;
 913         }
 914         for (src = dst; src < nr; src++)
 915                 ref[src] = NULL;
 916         string_list_clear(&names, 0);
 917         return dst;
 918 }
 919
 920 static void update_shallow(struct fetch_pack_args *args,
 921                            struct ref **sought, int nr_sought,
 922                            struct shallow_info *si)
 923 {
 924         struct sha1_array ref = SHA1_ARRAY_INIT;
 925         int *status;
 926         int i;
 927
 928         if (args->depth > 0 && alternate_shallow_file) {
 929                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 930                         unlink_or_warn(git_path("shallow"));
 931                         rollback_lock_file(&shallow_lock);
 932                 } else
 933                         commit_lock_file(&shallow_lock);
 934                 return;
 935         }
 936
 937         if (!si->shallow || !si->shallow->nr)
 938                 return;
 939
 940         if (args->cloning) {
 941                 /*
 942                  * remote is shallow, but this is a clone, there are
 943                  * no objects in repo to worry about. Accept any
 944                  * shallow points that exist in the pack (iow in repo
 945                  * after get_pack() and reprepare_packed_git())
 946                  */
 947                 struct sha1_array extra = SHA1_ARRAY_INIT;
 948                 unsigned char (*sha1)[20] = si->shallow->sha1;
 949                 for (i = 0; i < si->shallow->nr; i++)
 950                         if (has_sha1_file(sha1[i]))
 951                                 sha1_array_append(&extra, sha1[i]);
 952                 if (extra.nr) {
 953                         setup_alternate_shallow(&shallow_lock,
 954                                                 &alternate_shallow_file,
 955                                                 &extra);
 956                         commit_lock_file(&shallow_lock);
 957                 }
 958                 sha1_array_clear(&extra);
 959                 return;
 960         }
 961
 962         if (!si->nr_ours && !si->nr_theirs)
 963                 return;
 964
 965         remove_nonexistent_theirs_shallow(si);
 966         if (!si->nr_ours && !si->nr_theirs)
 967                 return;
 968         for (i = 0; i < nr_sought; i++)
 969                 sha1_array_append(&ref, sought[i]->old_sha1);
 970         si->ref = &ref;
 971
 972         if (args->update_shallow) {
 973                 /*
 974                  * remote is also shallow, .git/shallow may be updated
 975                  * so all refs can be accepted. Make sure we only add
 976                  * shallow roots that are actually reachable from new
 977                  * refs.
 978                  */
 979                 struct sha1_array extra = SHA1_ARRAY_INIT;
 980                 unsigned char (*sha1)[20] = si->shallow->sha1;
 981                 assign_shallow_commits_to_refs(si, NULL, NULL);
 982                 if (!si->nr_ours && !si->nr_theirs) {
 983                         sha1_array_clear(&ref);
 984                         return;
 985                 }
 986                 for (i = 0; i < si->nr_ours; i++)
 987                         sha1_array_append(&extra, sha1[si->ours[i]]);
 988                 for (i = 0; i < si->nr_theirs; i++)
 989                         sha1_array_append(&extra, sha1[si->theirs[i]]);
 990                 setup_alternate_shallow(&shallow_lock,
 991                                         &alternate_shallow_file,
 992                                         &extra);
 993                 commit_lock_file(&shallow_lock);
 994                 sha1_array_clear(&extra);
 995                 sha1_array_clear(&ref);
 996                 return;
 997         }
 998
 999         /*
1000          * remote is also shallow, check what ref is safe to update
1001          * without updating .git/shallow
1002          */
1003         status = xcalloc(nr_sought, sizeof(*status));
1004         assign_shallow_commits_to_refs(si, NULL, status);
1005         if (si->nr_ours || si->nr_theirs) {
1006                 for (i = 0; i < nr_sought; i++)
1007                         if (status[i])
1008                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1009         }
1010         free(status);
1011         sha1_array_clear(&ref);
1012 }
1013
1014 struct ref *fetch_pack(struct fetch_pack_args *args,
1015                        int fd[], struct child_process *conn,
1016                        const struct ref *ref,
1017                        const char *dest,
1018                        struct ref **sought, int nr_sought,
1019                        struct sha1_array *shallow,
1020                        char **pack_lockfile)
1021 {
1022         struct ref *ref_cpy;
1023         struct shallow_info si;
1024
1025         fetch_pack_setup();
1026         if (nr_sought)
1027                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1028
1029         if (!ref) {
1030                 packet_flush(fd[1]);
1031                 die("no matching remote head");
1032         }
1033         prepare_shallow_info(&si, shallow);
1034         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1035                                 &si, pack_lockfile);
1036         reprepare_packed_git();
1037         update_shallow(args, sought, nr_sought, &si);
1038         clear_shallow_info(&si);
1039         return ref_cpy;
1040 }