git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband, allow_tip_sha1_in_want;
  47
  48 static void rev_list_push(struct commit *commit, int mark)
  49 {
  50         if (!(commit->object.flags & mark)) {
  51                 commit->object.flags |= mark;
  52
  53                 if (parse_commit(commit))
  54                         return;
  55
  56                 prio_queue_put(&rev_list, commit);
  57
  58                 if (!(commit->object.flags & COMMON))
  59                         non_common_revs++;
  60         }
  61 }
  62
  63 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  64 {
  65         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  66
  67         if (o && o->type == OBJ_COMMIT)
  68                 rev_list_push((struct commit *)o, SEEN);
  69
  70         return 0;
  71 }
  72
  73 static int clear_marks(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
  74 {
  75         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  76
  77         if (o && o->type == OBJ_COMMIT)
  78                 clear_commit_marks((struct commit *)o,
  79                                    COMMON | COMMON_REF | SEEN | POPPED);
  80         return 0;
  81 }
  82
  83 /*
  84    This function marks a rev and its ancestors as common.
  85    In some cases, it is desirable to mark only the ancestors (for example
  86    when only the server does not yet know that they are common).
  87 */
  88
  89 static void mark_common(struct commit *commit,
  90                 int ancestors_only, int dont_parse)
  91 {
  92         if (commit != NULL && !(commit->object.flags & COMMON)) {
  93                 struct object *o = (struct object *)commit;
  94
  95                 if (!ancestors_only)
  96                         o->flags |= COMMON;
  97
  98                 if (!(o->flags & SEEN))
  99                         rev_list_push(commit, SEEN);
 100                 else {
 101                         struct commit_list *parents;
 102
 103                         if (!ancestors_only && !(o->flags & POPPED))
 104                                 non_common_revs--;
 105                         if (!o->parsed && !dont_parse)
 106                                 if (parse_commit(commit))
 107                                         return;
 108
 109                         for (parents = commit->parents;
 110                                         parents;
 111                                         parents = parents->next)
 112                                 mark_common(parents->item, 0, dont_parse);
 113                 }
 114         }
 115 }
 116
 117 /*
 118   Get the next rev to send, ignoring the common.
 119 */
 120
 121 static const unsigned char *get_rev(void)
 122 {
 123         struct commit *commit = NULL;
 124
 125         while (commit == NULL) {
 126                 unsigned int mark;
 127                 struct commit_list *parents;
 128
 129                 if (rev_list.nr == 0 || non_common_revs == 0)
 130                         return NULL;
 131
 132                 commit = prio_queue_get(&rev_list);
 133                 parse_commit(commit);
 134                 parents = commit->parents;
 135
 136                 commit->object.flags |= POPPED;
 137                 if (!(commit->object.flags & COMMON))
 138                         non_common_revs--;
 139
 140                 if (commit->object.flags & COMMON) {
 141                         /* do not send "have", and ignore ancestors */
 142                         commit = NULL;
 143                         mark = COMMON | SEEN;
 144                 } else if (commit->object.flags & COMMON_REF)
 145                         /* send "have", and ignore ancestors */
 146                         mark = COMMON | SEEN;
 147                 else
 148                         /* send "have", also for its ancestors */
 149                         mark = SEEN;
 150
 151                 while (parents) {
 152                         if (!(parents->item->object.flags & SEEN))
 153                                 rev_list_push(parents->item, mark);
 154                         if (mark & COMMON)
 155                                 mark_common(parents->item, 1, 0);
 156                         parents = parents->next;
 157                 }
 158         }
 159
 160         return commit->object.sha1;
 161 }
 162
 163 enum ack_type {
 164         NAK = 0,
 165         ACK,
 166         ACK_continue,
 167         ACK_common,
 168         ACK_ready
 169 };
 170
 171 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 172 {
 173         if (args->stateless_rpc && args->depth > 0) {
 174                 /* If we sent a depth we will get back "duplicate"
 175                  * shallow and unshallow commands every time there
 176                  * is a block of have lines exchanged.
 177                  */
 178                 char *line;
 179                 while ((line = packet_read_line(fd, NULL))) {
 180                         if (starts_with(line, "shallow "))
 181                                 continue;
 182                         if (starts_with(line, "unshallow "))
 183                                 continue;
 184                         die("git fetch-pack: expected shallow list");
 185                 }
 186         }
 187 }
 188
 189 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 190 {
 191         int len;
 192         char *line = packet_read_line(fd, &len);
 193         const char *arg;
 194
 195         if (!len)
 196                 die("git fetch-pack: expected ACK/NAK, got EOF");
 197         if (!strcmp(line, "NAK"))
 198                 return NAK;
 199         if (skip_prefix(line, "ACK ", &arg)) {
 200                 if (!get_sha1_hex(arg, result_sha1)) {
 201                         arg += 40;
 202                         len -= arg - line;
 203                         if (len < 1)
 204                                 return ACK;
 205                         if (strstr(arg, "continue"))
 206                                 return ACK_continue;
 207                         if (strstr(arg, "common"))
 208                                 return ACK_common;
 209                         if (strstr(arg, "ready"))
 210                                 return ACK_ready;
 211                         return ACK;
 212                 }
 213         }
 214         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 215 }
 216
 217 static void send_request(struct fetch_pack_args *args,
 218                          int fd, struct strbuf *buf)
 219 {
 220         if (args->stateless_rpc) {
 221                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 222                 packet_flush(fd);
 223         } else
 224                 write_or_die(fd, buf->buf, buf->len);
 225 }
 226
 227 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 228 {
 229         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 230 }
 231
 232 #define INITIAL_FLUSH 16
 233 #define PIPESAFE_FLUSH 32
 234 #define LARGE_FLUSH 1024
 235
 236 static int next_flush(struct fetch_pack_args *args, int count)
 237 {
 238         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 239
 240         if (count < flush_limit)
 241                 count <<= 1;
 242         else
 243                 count += flush_limit;
 244         return count;
 245 }
 246
 247 static int find_common(struct fetch_pack_args *args,
 248                        int fd[2], unsigned char *result_sha1,
 249                        struct ref *refs)
 250 {
 251         int fetching;
 252         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 253         const unsigned char *sha1;
 254         unsigned in_vain = 0;
 255         int got_continue = 0;
 256         int got_ready = 0;
 257         struct strbuf req_buf = STRBUF_INIT;
 258         size_t state_len = 0;
 259         struct each_ref_fn_sha1_adapter wrapped_rev_list_insert_ref =
 260                 {rev_list_insert_ref, NULL};
 261
 262         if (args->stateless_rpc && multi_ack == 1)
 263                 die("--stateless-rpc requires multi_ack_detailed");
 264         if (marked) {
 265                 struct each_ref_fn_sha1_adapter wrapped_clear_marks =
 266                         {clear_marks, NULL};
 267
 268                 for_each_ref(each_ref_fn_adapter, &wrapped_clear_marks);
 269         }
 270         marked = 1;
 271
 272         for_each_ref(each_ref_fn_adapter, &wrapped_rev_list_insert_ref);
 273         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 274
 275         fetching = 0;
 276         for ( ; refs ; refs = refs->next) {
 277                 unsigned char *remote = refs->old_sha1;
 278                 const char *remote_hex;
 279                 struct object *o;
 280
 281                 /*
 282                  * If that object is complete (i.e. it is an ancestor of a
 283                  * local ref), we tell them we have it but do not have to
 284                  * tell them about its ancestors, which they already know
 285                  * about.
 286                  *
 287                  * We use lookup_object here because we are only
 288                  * interested in the case we *know* the object is
 289                  * reachable and we have already scanned it.
 290                  */
 291                 if (((o = lookup_object(remote)) != NULL) &&
 292                                 (o->flags & COMPLETE)) {
 293                         continue;
 294                 }
 295
 296                 remote_hex = sha1_to_hex(remote);
 297                 if (!fetching) {
 298                         struct strbuf c = STRBUF_INIT;
 299                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 300                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 301                         if (no_done)            strbuf_addstr(&c, " no-done");
 302                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 303                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 304                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 305                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 306                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 307                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 308                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 309                                                             git_user_agent_sanitized());
 310                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 311                         strbuf_release(&c);
 312                 } else
 313                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 314                 fetching++;
 315         }
 316
 317         if (!fetching) {
 318                 strbuf_release(&req_buf);
 319                 packet_flush(fd[1]);
 320                 return 1;
 321         }
 322
 323         if (is_repository_shallow())
 324                 write_shallow_commits(&req_buf, 1, NULL);
 325         if (args->depth > 0)
 326                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 327         packet_buf_flush(&req_buf);
 328         state_len = req_buf.len;
 329
 330         if (args->depth > 0) {
 331                 char *line;
 332                 const char *arg;
 333                 unsigned char sha1[20];
 334
 335                 send_request(args, fd[1], &req_buf);
 336                 while ((line = packet_read_line(fd[0], NULL))) {
 337                         if (skip_prefix(line, "shallow ", &arg)) {
 338                                 if (get_sha1_hex(arg, sha1))
 339                                         die("invalid shallow line: %s", line);
 340                                 register_shallow(sha1);
 341                                 continue;
 342                         }
 343                         if (skip_prefix(line, "unshallow ", &arg)) {
 344                                 if (get_sha1_hex(arg, sha1))
 345                                         die("invalid unshallow line: %s", line);
 346                                 if (!lookup_object(sha1))
 347                                         die("object not found: %s", line);
 348                                 /* make sure that it is parsed as shallow */
 349                                 if (!parse_object(sha1))
 350                                         die("error in object: %s", line);
 351                                 if (unregister_shallow(sha1))
 352                                         die("no shallow found: %s", line);
 353                                 continue;
 354                         }
 355                         die("expected shallow/unshallow, got %s", line);
 356                 }
 357         } else if (!args->stateless_rpc)
 358                 send_request(args, fd[1], &req_buf);
 359
 360         if (!args->stateless_rpc) {
 361                 /* If we aren't using the stateless-rpc interface
 362                  * we don't need to retain the headers.
 363                  */
 364                 strbuf_setlen(&req_buf, 0);
 365                 state_len = 0;
 366         }
 367
 368         flushes = 0;
 369         retval = -1;
 370         while ((sha1 = get_rev())) {
 371                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 372                 if (args->verbose)
 373                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 374                 in_vain++;
 375                 if (flush_at <= ++count) {
 376                         int ack;
 377
 378                         packet_buf_flush(&req_buf);
 379                         send_request(args, fd[1], &req_buf);
 380                         strbuf_setlen(&req_buf, state_len);
 381                         flushes++;
 382                         flush_at = next_flush(args, count);
 383
 384                         /*
 385                          * We keep one window "ahead" of the other side, and
 386                          * will wait for an ACK only on the next one
 387                          */
 388                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 389                                 continue;
 390
 391                         consume_shallow_list(args, fd[0]);
 392                         do {
 393                                 ack = get_ack(fd[0], result_sha1);
 394                                 if (args->verbose && ack)
 395                                         fprintf(stderr, "got ack %d %s\n", ack,
 396                                                         sha1_to_hex(result_sha1));
 397                                 switch (ack) {
 398                                 case ACK:
 399                                         flushes = 0;
 400                                         multi_ack = 0;
 401                                         retval = 0;
 402                                         goto done;
 403                                 case ACK_common:
 404                                 case ACK_ready:
 405                                 case ACK_continue: {
 406                                         struct commit *commit =
 407                                                 lookup_commit(result_sha1);
 408                                         if (!commit)
 409                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 410                                         if (args->stateless_rpc
 411                                          && ack == ACK_common
 412                                          && !(commit->object.flags & COMMON)) {
 413                                                 /* We need to replay the have for this object
 414                                                  * on the next RPC request so the peer knows
 415                                                  * it is in common with us.
 416                                                  */
 417                                                 const char *hex = sha1_to_hex(result_sha1);
 418                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 419                                                 state_len = req_buf.len;
 420                                         }
 421                                         mark_common(commit, 0, 1);
 422                                         retval = 0;
 423                                         in_vain = 0;
 424                                         got_continue = 1;
 425                                         if (ack == ACK_ready) {
 426                                                 clear_prio_queue(&rev_list);
 427                                                 got_ready = 1;
 428                                         }
 429                                         break;
 430                                         }
 431                                 }
 432                         } while (ack);
 433                         flushes--;
 434                         if (got_continue && MAX_IN_VAIN < in_vain) {
 435                                 if (args->verbose)
 436                                         fprintf(stderr, "giving up\n");
 437                                 break; /* give up */
 438                         }
 439                 }
 440         }
 441 done:
 442         if (!got_ready || !no_done) {
 443                 packet_buf_write(&req_buf, "done\n");
 444                 send_request(args, fd[1], &req_buf);
 445         }
 446         if (args->verbose)
 447                 fprintf(stderr, "done\n");
 448         if (retval != 0) {
 449                 multi_ack = 0;
 450                 flushes++;
 451         }
 452         strbuf_release(&req_buf);
 453
 454         if (!got_ready || !no_done)
 455                 consume_shallow_list(args, fd[0]);
 456         while (flushes || multi_ack) {
 457                 int ack = get_ack(fd[0], result_sha1);
 458                 if (ack) {
 459                         if (args->verbose)
 460                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 461                                         sha1_to_hex(result_sha1));
 462                         if (ack == ACK)
 463                                 return 0;
 464                         multi_ack = 1;
 465                         continue;
 466                 }
 467                 flushes--;
 468         }
 469         /* it is no error to fetch into a completely empty repo */
 470         return count ? retval : 0;
 471 }
 472
 473 static struct commit_list *complete;
 474
 475 static int mark_complete(const char *refname, const unsigned char *sha1, int flag, void *cb_data)
 476 {
 477         struct object *o = parse_object(sha1);
 478
 479         while (o && o->type == OBJ_TAG) {
 480                 struct tag *t = (struct tag *) o;
 481                 if (!t->tagged)
 482                         break; /* broken repository */
 483                 o->flags |= COMPLETE;
 484                 o = parse_object(t->tagged->sha1);
 485         }
 486         if (o && o->type == OBJ_COMMIT) {
 487                 struct commit *commit = (struct commit *)o;
 488                 if (!(commit->object.flags & COMPLETE)) {
 489                         commit->object.flags |= COMPLETE;
 490                         commit_list_insert(commit, &complete);
 491                 }
 492         }
 493         return 0;
 494 }
 495
 496 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 497                                          unsigned long cutoff)
 498 {
 499         while (complete && cutoff <= complete->item->date) {
 500                 if (args->verbose)
 501                         fprintf(stderr, "Marking %s as complete\n",
 502                                 sha1_to_hex(complete->item->object.sha1));
 503                 pop_most_recent_commit(&complete, COMPLETE);
 504         }
 505 }
 506
 507 static void filter_refs(struct fetch_pack_args *args,
 508                         struct ref **refs,
 509                         struct ref **sought, int nr_sought)
 510 {
 511         struct ref *newlist = NULL;
 512         struct ref **newtail = &newlist;
 513         struct ref *ref, *next;
 514         int i;
 515
 516         i = 0;
 517         for (ref = *refs; ref; ref = next) {
 518                 int keep = 0;
 519                 next = ref->next;
 520
 521                 if (starts_with(ref->name, "refs/") &&
 522                     check_refname_format(ref->name, 0))
 523                         ; /* trash */
 524                 else {
 525                         while (i < nr_sought) {
 526                                 int cmp = strcmp(ref->name, sought[i]->name);
 527                                 if (cmp < 0)
 528                                         break; /* definitely do not have it */
 529                                 else if (cmp == 0) {
 530                                         keep = 1; /* definitely have it */
 531                                         sought[i]->matched = 1;
 532                                 }
 533                                 i++;
 534                         }
 535                 }
 536
 537                 if (!keep && args->fetch_all &&
 538                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 539                         keep = 1;
 540
 541                 if (keep) {
 542                         *newtail = ref;
 543                         ref->next = NULL;
 544                         newtail = &ref->next;
 545                 } else {
 546                         free(ref);
 547                 }
 548         }
 549
 550         /* Append unmatched requests to the list */
 551         if (allow_tip_sha1_in_want) {
 552                 for (i = 0; i < nr_sought; i++) {
 553                         unsigned char sha1[20];
 554
 555                         ref = sought[i];
 556                         if (ref->matched)
 557                                 continue;
 558                         if (get_sha1_hex(ref->name, sha1) ||
 559                             ref->name[40] != '\0' ||
 560                             hashcmp(sha1, ref->old_sha1))
 561                                 continue;
 562
 563                         ref->matched = 1;
 564                         *newtail = copy_ref(ref);
 565                         newtail = &(*newtail)->next;
 566                 }
 567         }
 568         *refs = newlist;
 569 }
 570
 571 static void mark_alternate_complete(const struct ref *ref, void *unused)
 572 {
 573         mark_complete(NULL, ref->old_sha1, 0, NULL);
 574 }
 575
 576 static int everything_local(struct fetch_pack_args *args,
 577                             struct ref **refs,
 578                             struct ref **sought, int nr_sought)
 579 {
 580         struct ref *ref;
 581         int retval;
 582         unsigned long cutoff = 0;
 583
 584         save_commit_buffer = 0;
 585
 586         for (ref = *refs; ref; ref = ref->next) {
 587                 struct object *o;
 588
 589                 if (!has_sha1_file(ref->old_sha1))
 590                         continue;
 591
 592                 o = parse_object(ref->old_sha1);
 593                 if (!o)
 594                         continue;
 595
 596                 /* We already have it -- which may mean that we were
 597                  * in sync with the other side at some time after
 598                  * that (it is OK if we guess wrong here).
 599                  */
 600                 if (o->type == OBJ_COMMIT) {
 601                         struct commit *commit = (struct commit *)o;
 602                         if (!cutoff || cutoff < commit->date)
 603                                 cutoff = commit->date;
 604                 }
 605         }
 606
 607         if (!args->depth) {
 608                 struct each_ref_fn_sha1_adapter wrapped_mark_complete =
 609                         {mark_complete, NULL};
 610
 611                 for_each_ref(each_ref_fn_adapter, &wrapped_mark_complete);
 612                 for_each_alternate_ref(mark_alternate_complete, NULL);
 613                 commit_list_sort_by_date(&complete);
 614                 if (cutoff)
 615                         mark_recent_complete_commits(args, cutoff);
 616         }
 617
 618         /*
 619          * Mark all complete remote refs as common refs.
 620          * Don't mark them common yet; the server has to be told so first.
 621          */
 622         for (ref = *refs; ref; ref = ref->next) {
 623                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 624                                              NULL, 0);
 625
 626                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 627                         continue;
 628
 629                 if (!(o->flags & SEEN)) {
 630                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 631
 632                         mark_common((struct commit *)o, 1, 1);
 633                 }
 634         }
 635
 636         filter_refs(args, refs, sought, nr_sought);
 637
 638         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 639                 const unsigned char *remote = ref->old_sha1;
 640                 struct object *o;
 641
 642                 o = lookup_object(remote);
 643                 if (!o || !(o->flags & COMPLETE)) {
 644                         retval = 0;
 645                         if (!args->verbose)
 646                                 continue;
 647                         fprintf(stderr,
 648                                 "want %s (%s)\n", sha1_to_hex(remote),
 649                                 ref->name);
 650                         continue;
 651                 }
 652                 if (!args->verbose)
 653                         continue;
 654                 fprintf(stderr,
 655                         "already have %s (%s)\n", sha1_to_hex(remote),
 656                         ref->name);
 657         }
 658         return retval;
 659 }
 660
 661 static int sideband_demux(int in, int out, void *data)
 662 {
 663         int *xd = data;
 664
 665         int ret = recv_sideband("fetch-pack", xd[0], out);
 666         close(out);
 667         return ret;
 668 }
 669
 670 static int get_pack(struct fetch_pack_args *args,
 671                     int xd[2], char **pack_lockfile)
 672 {
 673         struct async demux;
 674         const char *argv[22];
 675         char keep_arg[256];
 676         char hdr_arg[256];
 677         const char **av, *cmd_name;
 678         int do_keep = args->keep_pack;
 679         struct child_process cmd = CHILD_PROCESS_INIT;
 680         int ret;
 681
 682         memset(&demux, 0, sizeof(demux));
 683         if (use_sideband) {
 684                 /* xd[] is talking with upload-pack; subprocess reads from
 685                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 686                  * through demux->out.
 687                  */
 688                 demux.proc = sideband_demux;
 689                 demux.data = xd;
 690                 demux.out = -1;
 691                 if (start_async(&demux))
 692                         die("fetch-pack: unable to fork off sideband"
 693                             " demultiplexer");
 694         }
 695         else
 696                 demux.out = xd[0];
 697
 698         cmd.argv = argv;
 699         av = argv;
 700         *hdr_arg = 0;
 701         if (!args->keep_pack && unpack_limit) {
 702                 struct pack_header header;
 703
 704                 if (read_pack_header(demux.out, &header))
 705                         die("protocol error: bad pack header");
 706                 snprintf(hdr_arg, sizeof(hdr_arg),
 707                          "--pack_header=%"PRIu32",%"PRIu32,
 708                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 709                 if (ntohl(header.hdr_entries) < unpack_limit)
 710                         do_keep = 0;
 711                 else
 712                         do_keep = 1;
 713         }
 714
 715         if (alternate_shallow_file) {
 716                 *av++ = "--shallow-file";
 717                 *av++ = alternate_shallow_file;
 718         }
 719
 720         if (do_keep) {
 721                 if (pack_lockfile)
 722                         cmd.out = -1;
 723                 *av++ = cmd_name = "index-pack";
 724                 *av++ = "--stdin";
 725                 if (!args->quiet && !args->no_progress)
 726                         *av++ = "-v";
 727                 if (args->use_thin_pack)
 728                         *av++ = "--fix-thin";
 729                 if (args->lock_pack || unpack_limit) {
 730                         int s = sprintf(keep_arg,
 731                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 732                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 733                                 strcpy(keep_arg + s, "localhost");
 734                         *av++ = keep_arg;
 735                 }
 736                 if (args->check_self_contained_and_connected)
 737                         *av++ = "--check-self-contained-and-connected";
 738         }
 739         else {
 740                 *av++ = cmd_name = "unpack-objects";
 741                 if (args->quiet || args->no_progress)
 742                         *av++ = "-q";
 743                 args->check_self_contained_and_connected = 0;
 744         }
 745         if (*hdr_arg)
 746                 *av++ = hdr_arg;
 747         if (fetch_fsck_objects >= 0
 748             ? fetch_fsck_objects
 749             : transfer_fsck_objects >= 0
 750             ? transfer_fsck_objects
 751             : 0)
 752                 *av++ = "--strict";
 753         *av++ = NULL;
 754
 755         cmd.in = demux.out;
 756         cmd.git_cmd = 1;
 757         if (start_command(&cmd))
 758                 die("fetch-pack: unable to fork off %s", cmd_name);
 759         if (do_keep && pack_lockfile) {
 760                 *pack_lockfile = index_pack_lockfile(cmd.out);
 761                 close(cmd.out);
 762         }
 763
 764         if (!use_sideband)
 765                 /* Closed by start_command() */
 766                 xd[0] = -1;
 767
 768         ret = finish_command(&cmd);
 769         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 770                 args->self_contained_and_connected =
 771                         args->check_self_contained_and_connected &&
 772                         ret == 0;
 773         else
 774                 die("%s failed", cmd_name);
 775         if (use_sideband && finish_async(&demux))
 776                 die("error in sideband demultiplexer");
 777         return 0;
 778 }
 779
 780 static int cmp_ref_by_name(const void *a_, const void *b_)
 781 {
 782         const struct ref *a = *((const struct ref **)a_);
 783         const struct ref *b = *((const struct ref **)b_);
 784         return strcmp(a->name, b->name);
 785 }
 786
 787 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 788                                  int fd[2],
 789                                  const struct ref *orig_ref,
 790                                  struct ref **sought, int nr_sought,
 791                                  struct shallow_info *si,
 792                                  char **pack_lockfile)
 793 {
 794         struct ref *ref = copy_ref_list(orig_ref);
 795         unsigned char sha1[20];
 796         const char *agent_feature;
 797         int agent_len;
 798
 799         sort_ref_list(&ref, ref_compare_name);
 800         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 801
 802         if (is_repository_shallow() && !server_supports("shallow"))
 803                 die("Server does not support shallow clients");
 804         if (server_supports("multi_ack_detailed")) {
 805                 if (args->verbose)
 806                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 807                 multi_ack = 2;
 808                 if (server_supports("no-done")) {
 809                         if (args->verbose)
 810                                 fprintf(stderr, "Server supports no-done\n");
 811                         if (args->stateless_rpc)
 812                                 no_done = 1;
 813                 }
 814         }
 815         else if (server_supports("multi_ack")) {
 816                 if (args->verbose)
 817                         fprintf(stderr, "Server supports multi_ack\n");
 818                 multi_ack = 1;
 819         }
 820         if (server_supports("side-band-64k")) {
 821                 if (args->verbose)
 822                         fprintf(stderr, "Server supports side-band-64k\n");
 823                 use_sideband = 2;
 824         }
 825         else if (server_supports("side-band")) {
 826                 if (args->verbose)
 827                         fprintf(stderr, "Server supports side-band\n");
 828                 use_sideband = 1;
 829         }
 830         if (server_supports("allow-tip-sha1-in-want")) {
 831                 if (args->verbose)
 832                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 833                 allow_tip_sha1_in_want = 1;
 834         }
 835         if (!server_supports("thin-pack"))
 836                 args->use_thin_pack = 0;
 837         if (!server_supports("no-progress"))
 838                 args->no_progress = 0;
 839         if (!server_supports("include-tag"))
 840                 args->include_tag = 0;
 841         if (server_supports("ofs-delta")) {
 842                 if (args->verbose)
 843                         fprintf(stderr, "Server supports ofs-delta\n");
 844         } else
 845                 prefer_ofs_delta = 0;
 846
 847         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 848                 agent_supported = 1;
 849                 if (args->verbose && agent_len)
 850                         fprintf(stderr, "Server version is %.*s\n",
 851                                 agent_len, agent_feature);
 852         }
 853
 854         if (everything_local(args, &ref, sought, nr_sought)) {
 855                 packet_flush(fd[1]);
 856                 goto all_done;
 857         }
 858         if (find_common(args, fd, sha1, ref) < 0)
 859                 if (!args->keep_pack)
 860                         /* When cloning, it is not unusual to have
 861                          * no common commit.
 862                          */
 863                         warning("no common commits");
 864
 865         if (args->stateless_rpc)
 866                 packet_flush(fd[1]);
 867         if (args->depth > 0)
 868                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 869                                         NULL);
 870         else if (si->nr_ours || si->nr_theirs)
 871                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 872         else
 873                 alternate_shallow_file = NULL;
 874         if (get_pack(args, fd, pack_lockfile))
 875                 die("git fetch-pack: fetch failed.");
 876
 877  all_done:
 878         return ref;
 879 }
 880
 881 static void fetch_pack_config(void)
 882 {
 883         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 884         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 885         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 886         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 887         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 888
 889         git_config(git_default_config, NULL);
 890 }
 891
 892 static void fetch_pack_setup(void)
 893 {
 894         static int did_setup;
 895         if (did_setup)
 896                 return;
 897         fetch_pack_config();
 898         if (0 <= transfer_unpack_limit)
 899                 unpack_limit = transfer_unpack_limit;
 900         else if (0 <= fetch_unpack_limit)
 901                 unpack_limit = fetch_unpack_limit;
 902         did_setup = 1;
 903 }
 904
 905 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 906 {
 907         struct string_list names = STRING_LIST_INIT_NODUP;
 908         int src, dst;
 909
 910         for (src = dst = 0; src < nr; src++) {
 911                 struct string_list_item *item;
 912                 item = string_list_insert(&names, ref[src]->name);
 913                 if (item->util)
 914                         continue; /* already have it */
 915                 item->util = ref[src];
 916                 if (src != dst)
 917                         ref[dst] = ref[src];
 918                 dst++;
 919         }
 920         for (src = dst; src < nr; src++)
 921                 ref[src] = NULL;
 922         string_list_clear(&names, 0);
 923         return dst;
 924 }
 925
 926 static void update_shallow(struct fetch_pack_args *args,
 927                            struct ref **sought, int nr_sought,
 928                            struct shallow_info *si)
 929 {
 930         struct sha1_array ref = SHA1_ARRAY_INIT;
 931         int *status;
 932         int i;
 933
 934         if (args->depth > 0 && alternate_shallow_file) {
 935                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 936                         unlink_or_warn(git_path("shallow"));
 937                         rollback_lock_file(&shallow_lock);
 938                 } else
 939                         commit_lock_file(&shallow_lock);
 940                 return;
 941         }
 942
 943         if (!si->shallow || !si->shallow->nr)
 944                 return;
 945
 946         if (args->cloning) {
 947                 /*
 948                  * remote is shallow, but this is a clone, there are
 949                  * no objects in repo to worry about. Accept any
 950                  * shallow points that exist in the pack (iow in repo
 951                  * after get_pack() and reprepare_packed_git())
 952                  */
 953                 struct sha1_array extra = SHA1_ARRAY_INIT;
 954                 unsigned char (*sha1)[20] = si->shallow->sha1;
 955                 for (i = 0; i < si->shallow->nr; i++)
 956                         if (has_sha1_file(sha1[i]))
 957                                 sha1_array_append(&extra, sha1[i]);
 958                 if (extra.nr) {
 959                         setup_alternate_shallow(&shallow_lock,
 960                                                 &alternate_shallow_file,
 961                                                 &extra);
 962                         commit_lock_file(&shallow_lock);
 963                 }
 964                 sha1_array_clear(&extra);
 965                 return;
 966         }
 967
 968         if (!si->nr_ours && !si->nr_theirs)
 969                 return;
 970
 971         remove_nonexistent_theirs_shallow(si);
 972         if (!si->nr_ours && !si->nr_theirs)
 973                 return;
 974         for (i = 0; i < nr_sought; i++)
 975                 sha1_array_append(&ref, sought[i]->old_sha1);
 976         si->ref = &ref;
 977
 978         if (args->update_shallow) {
 979                 /*
 980                  * remote is also shallow, .git/shallow may be updated
 981                  * so all refs can be accepted. Make sure we only add
 982                  * shallow roots that are actually reachable from new
 983                  * refs.
 984                  */
 985                 struct sha1_array extra = SHA1_ARRAY_INIT;
 986                 unsigned char (*sha1)[20] = si->shallow->sha1;
 987                 assign_shallow_commits_to_refs(si, NULL, NULL);
 988                 if (!si->nr_ours && !si->nr_theirs) {
 989                         sha1_array_clear(&ref);
 990                         return;
 991                 }
 992                 for (i = 0; i < si->nr_ours; i++)
 993                         sha1_array_append(&extra, sha1[si->ours[i]]);
 994                 for (i = 0; i < si->nr_theirs; i++)
 995                         sha1_array_append(&extra, sha1[si->theirs[i]]);
 996                 setup_alternate_shallow(&shallow_lock,
 997                                         &alternate_shallow_file,
 998                                         &extra);
 999                 commit_lock_file(&shallow_lock);
1000                 sha1_array_clear(&extra);
1001                 sha1_array_clear(&ref);
1002                 return;
1003         }
1004
1005         /*
1006          * remote is also shallow, check what ref is safe to update
1007          * without updating .git/shallow
1008          */
1009         status = xcalloc(nr_sought, sizeof(*status));
1010         assign_shallow_commits_to_refs(si, NULL, status);
1011         if (si->nr_ours || si->nr_theirs) {
1012                 for (i = 0; i < nr_sought; i++)
1013                         if (status[i])
1014                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1015         }
1016         free(status);
1017         sha1_array_clear(&ref);
1018 }
1019
1020 struct ref *fetch_pack(struct fetch_pack_args *args,
1021                        int fd[], struct child_process *conn,
1022                        const struct ref *ref,
1023                        const char *dest,
1024                        struct ref **sought, int nr_sought,
1025                        struct sha1_array *shallow,
1026                        char **pack_lockfile)
1027 {
1028         struct ref *ref_cpy;
1029         struct shallow_info si;
1030
1031         fetch_pack_setup();
1032         if (nr_sought)
1033                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1034
1035         if (!ref) {
1036                 packet_flush(fd[1]);
1037                 die("no matching remote head");
1038         }
1039         prepare_shallow_info(&si, shallow);
1040         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1041                                 &si, pack_lockfile);
1042         reprepare_packed_git();
1043         update_shallow(args, sought, nr_sought, &si);
1044         clear_shallow_info(&si);
1045         return ref_cpy;
1046 }