git.oblomov.eu Git - git/blob - builtin/fetch-pack.c

   1 #include "builtin.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "pack.h"
   8 #include "sideband.h"
   9 #include "fetch-pack.h"
  10 #include "remote.h"
  11 #include "run-command.h"
  12 #include "transport.h"
  13
  14 static int transfer_unpack_limit = -1;
  15 static int fetch_unpack_limit = -1;
  16 static int unpack_limit = 100;
  17 static int prefer_ofs_delta = 1;
  18 static int no_done;
  19 static int fetch_fsck_objects = -1;
  20 static int transfer_fsck_objects = -1;
  21 static struct fetch_pack_args args = {
  22         /* .uploadpack = */ "git-upload-pack",
  23 };
  24
  25 static const char fetch_pack_usage[] =
  26 "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
  27
  28 #define COMPLETE        (1U << 0)
  29 #define COMMON          (1U << 1)
  30 #define COMMON_REF      (1U << 2)
  31 #define SEEN            (1U << 3)
  32 #define POPPED          (1U << 4)
  33
  34 static int marked;
  35
  36 /*
  37  * After sending this many "have"s if we do not get any new ACK , we
  38  * give up traversing our history.
  39  */
  40 #define MAX_IN_VAIN 256
  41
  42 static struct commit_list *rev_list;
  43 static int non_common_revs, multi_ack, use_sideband;
  44
  45 static void rev_list_push(struct commit *commit, int mark)
  46 {
  47         if (!(commit->object.flags & mark)) {
  48                 commit->object.flags |= mark;
  49
  50                 if (!(commit->object.parsed))
  51                         if (parse_commit(commit))
  52                                 return;
  53
  54                 commit_list_insert_by_date(commit, &rev_list);
  55
  56                 if (!(commit->object.flags & COMMON))
  57                         non_common_revs++;
  58         }
  59 }
  60
  61 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  62 {
  63         struct object *o = deref_tag(parse_object(sha1), path, 0);
  64
  65         if (o && o->type == OBJ_COMMIT)
  66                 rev_list_push((struct commit *)o, SEEN);
  67
  68         return 0;
  69 }
  70
  71 static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  72 {
  73         struct object *o = deref_tag(parse_object(sha1), path, 0);
  74
  75         if (o && o->type == OBJ_COMMIT)
  76                 clear_commit_marks((struct commit *)o,
  77                                    COMMON | COMMON_REF | SEEN | POPPED);
  78         return 0;
  79 }
  80
  81 /*
  82    This function marks a rev and its ancestors as common.
  83    In some cases, it is desirable to mark only the ancestors (for example
  84    when only the server does not yet know that they are common).
  85 */
  86
  87 static void mark_common(struct commit *commit,
  88                 int ancestors_only, int dont_parse)
  89 {
  90         if (commit != NULL && !(commit->object.flags & COMMON)) {
  91                 struct object *o = (struct object *)commit;
  92
  93                 if (!ancestors_only)
  94                         o->flags |= COMMON;
  95
  96                 if (!(o->flags & SEEN))
  97                         rev_list_push(commit, SEEN);
  98                 else {
  99                         struct commit_list *parents;
 100
 101                         if (!ancestors_only && !(o->flags & POPPED))
 102                                 non_common_revs--;
 103                         if (!o->parsed && !dont_parse)
 104                                 if (parse_commit(commit))
 105                                         return;
 106
 107                         for (parents = commit->parents;
 108                                         parents;
 109                                         parents = parents->next)
 110                                 mark_common(parents->item, 0, dont_parse);
 111                 }
 112         }
 113 }
 114
 115 /*
 116   Get the next rev to send, ignoring the common.
 117 */
 118
 119 static const unsigned char *get_rev(void)
 120 {
 121         struct commit *commit = NULL;
 122
 123         while (commit == NULL) {
 124                 unsigned int mark;
 125                 struct commit_list *parents;
 126
 127                 if (rev_list == NULL || non_common_revs == 0)
 128                         return NULL;
 129
 130                 commit = rev_list->item;
 131                 if (!commit->object.parsed)
 132                         parse_commit(commit);
 133                 parents = commit->parents;
 134
 135                 commit->object.flags |= POPPED;
 136                 if (!(commit->object.flags & COMMON))
 137                         non_common_revs--;
 138
 139                 if (commit->object.flags & COMMON) {
 140                         /* do not send "have", and ignore ancestors */
 141                         commit = NULL;
 142                         mark = COMMON | SEEN;
 143                 } else if (commit->object.flags & COMMON_REF)
 144                         /* send "have", and ignore ancestors */
 145                         mark = COMMON | SEEN;
 146                 else
 147                         /* send "have", also for its ancestors */
 148                         mark = SEEN;
 149
 150                 while (parents) {
 151                         if (!(parents->item->object.flags & SEEN))
 152                                 rev_list_push(parents->item, mark);
 153                         if (mark & COMMON)
 154                                 mark_common(parents->item, 1, 0);
 155                         parents = parents->next;
 156                 }
 157
 158                 rev_list = rev_list->next;
 159         }
 160
 161         return commit->object.sha1;
 162 }
 163
 164 enum ack_type {
 165         NAK = 0,
 166         ACK,
 167         ACK_continue,
 168         ACK_common,
 169         ACK_ready
 170 };
 171
 172 static void consume_shallow_list(int fd)
 173 {
 174         if (args.stateless_rpc && args.depth > 0) {
 175                 /* If we sent a depth we will get back "duplicate"
 176                  * shallow and unshallow commands every time there
 177                  * is a block of have lines exchanged.
 178                  */
 179                 char line[1000];
 180                 while (packet_read_line(fd, line, sizeof(line))) {
 181                         if (!prefixcmp(line, "shallow "))
 182                                 continue;
 183                         if (!prefixcmp(line, "unshallow "))
 184                                 continue;
 185                         die("git fetch-pack: expected shallow list");
 186                 }
 187         }
 188 }
 189
 190 struct write_shallow_data {
 191         struct strbuf *out;
 192         int use_pack_protocol;
 193         int count;
 194 };
 195
 196 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
 197 {
 198         struct write_shallow_data *data = cb_data;
 199         const char *hex = sha1_to_hex(graft->sha1);
 200         data->count++;
 201         if (data->use_pack_protocol)
 202                 packet_buf_write(data->out, "shallow %s", hex);
 203         else {
 204                 strbuf_addstr(data->out, hex);
 205                 strbuf_addch(data->out, '\n');
 206         }
 207         return 0;
 208 }
 209
 210 static int write_shallow_commits(struct strbuf *out, int use_pack_protocol)
 211 {
 212         struct write_shallow_data data;
 213         data.out = out;
 214         data.use_pack_protocol = use_pack_protocol;
 215         data.count = 0;
 216         for_each_commit_graft(write_one_shallow, &data);
 217         return data.count;
 218 }
 219
 220 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 221 {
 222         static char line[1000];
 223         int len = packet_read_line(fd, line, sizeof(line));
 224
 225         if (!len)
 226                 die("git fetch-pack: expected ACK/NAK, got EOF");
 227         if (line[len-1] == '\n')
 228                 line[--len] = 0;
 229         if (!strcmp(line, "NAK"))
 230                 return NAK;
 231         if (!prefixcmp(line, "ACK ")) {
 232                 if (!get_sha1_hex(line+4, result_sha1)) {
 233                         if (strstr(line+45, "continue"))
 234                                 return ACK_continue;
 235                         if (strstr(line+45, "common"))
 236                                 return ACK_common;
 237                         if (strstr(line+45, "ready"))
 238                                 return ACK_ready;
 239                         return ACK;
 240                 }
 241         }
 242         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 243 }
 244
 245 static void send_request(int fd, struct strbuf *buf)
 246 {
 247         if (args.stateless_rpc) {
 248                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 249                 packet_flush(fd);
 250         } else
 251                 safe_write(fd, buf->buf, buf->len);
 252 }
 253
 254 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 255 {
 256         rev_list_insert_ref(NULL, ref->old_sha1, 0, NULL);
 257 }
 258
 259 static void insert_alternate_refs(void)
 260 {
 261         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 262 }
 263
 264 #define INITIAL_FLUSH 16
 265 #define PIPESAFE_FLUSH 32
 266 #define LARGE_FLUSH 1024
 267
 268 static int next_flush(int count)
 269 {
 270         int flush_limit = args.stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 271
 272         if (count < flush_limit)
 273                 count <<= 1;
 274         else
 275                 count += flush_limit;
 276         return count;
 277 }
 278
 279 static int find_common(int fd[2], unsigned char *result_sha1,
 280                        struct ref *refs)
 281 {
 282         int fetching;
 283         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 284         const unsigned char *sha1;
 285         unsigned in_vain = 0;
 286         int got_continue = 0;
 287         int got_ready = 0;
 288         struct strbuf req_buf = STRBUF_INIT;
 289         size_t state_len = 0;
 290
 291         if (args.stateless_rpc && multi_ack == 1)
 292                 die("--stateless-rpc requires multi_ack_detailed");
 293         if (marked)
 294                 for_each_ref(clear_marks, NULL);
 295         marked = 1;
 296
 297         for_each_ref(rev_list_insert_ref, NULL);
 298         insert_alternate_refs();
 299
 300         fetching = 0;
 301         for ( ; refs ; refs = refs->next) {
 302                 unsigned char *remote = refs->old_sha1;
 303                 const char *remote_hex;
 304                 struct object *o;
 305
 306                 /*
 307                  * If that object is complete (i.e. it is an ancestor of a
 308                  * local ref), we tell them we have it but do not have to
 309                  * tell them about its ancestors, which they already know
 310                  * about.
 311                  *
 312                  * We use lookup_object here because we are only
 313                  * interested in the case we *know* the object is
 314                  * reachable and we have already scanned it.
 315                  */
 316                 if (((o = lookup_object(remote)) != NULL) &&
 317                                 (o->flags & COMPLETE)) {
 318                         continue;
 319                 }
 320
 321                 remote_hex = sha1_to_hex(remote);
 322                 if (!fetching) {
 323                         struct strbuf c = STRBUF_INIT;
 324                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 325                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 326                         if (no_done)            strbuf_addstr(&c, " no-done");
 327                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 328                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 329                         if (args.use_thin_pack) strbuf_addstr(&c, " thin-pack");
 330                         if (args.no_progress)   strbuf_addstr(&c, " no-progress");
 331                         if (args.include_tag)   strbuf_addstr(&c, " include-tag");
 332                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 333                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 334                         strbuf_release(&c);
 335                 } else
 336                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 337                 fetching++;
 338         }
 339
 340         if (!fetching) {
 341                 strbuf_release(&req_buf);
 342                 packet_flush(fd[1]);
 343                 return 1;
 344         }
 345
 346         if (is_repository_shallow())
 347                 write_shallow_commits(&req_buf, 1);
 348         if (args.depth > 0)
 349                 packet_buf_write(&req_buf, "deepen %d", args.depth);
 350         packet_buf_flush(&req_buf);
 351         state_len = req_buf.len;
 352
 353         if (args.depth > 0) {
 354                 char line[1024];
 355                 unsigned char sha1[20];
 356
 357                 send_request(fd[1], &req_buf);
 358                 while (packet_read_line(fd[0], line, sizeof(line))) {
 359                         if (!prefixcmp(line, "shallow ")) {
 360                                 if (get_sha1_hex(line + 8, sha1))
 361                                         die("invalid shallow line: %s", line);
 362                                 register_shallow(sha1);
 363                                 continue;
 364                         }
 365                         if (!prefixcmp(line, "unshallow ")) {
 366                                 if (get_sha1_hex(line + 10, sha1))
 367                                         die("invalid unshallow line: %s", line);
 368                                 if (!lookup_object(sha1))
 369                                         die("object not found: %s", line);
 370                                 /* make sure that it is parsed as shallow */
 371                                 if (!parse_object(sha1))
 372                                         die("error in object: %s", line);
 373                                 if (unregister_shallow(sha1))
 374                                         die("no shallow found: %s", line);
 375                                 continue;
 376                         }
 377                         die("expected shallow/unshallow, got %s", line);
 378                 }
 379         } else if (!args.stateless_rpc)
 380                 send_request(fd[1], &req_buf);
 381
 382         if (!args.stateless_rpc) {
 383                 /* If we aren't using the stateless-rpc interface
 384                  * we don't need to retain the headers.
 385                  */
 386                 strbuf_setlen(&req_buf, 0);
 387                 state_len = 0;
 388         }
 389
 390         flushes = 0;
 391         retval = -1;
 392         while ((sha1 = get_rev())) {
 393                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 394                 if (args.verbose)
 395                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 396                 in_vain++;
 397                 if (flush_at <= ++count) {
 398                         int ack;
 399
 400                         packet_buf_flush(&req_buf);
 401                         send_request(fd[1], &req_buf);
 402                         strbuf_setlen(&req_buf, state_len);
 403                         flushes++;
 404                         flush_at = next_flush(count);
 405
 406                         /*
 407                          * We keep one window "ahead" of the other side, and
 408                          * will wait for an ACK only on the next one
 409                          */
 410                         if (!args.stateless_rpc && count == INITIAL_FLUSH)
 411                                 continue;
 412
 413                         consume_shallow_list(fd[0]);
 414                         do {
 415                                 ack = get_ack(fd[0], result_sha1);
 416                                 if (args.verbose && ack)
 417                                         fprintf(stderr, "got ack %d %s\n", ack,
 418                                                         sha1_to_hex(result_sha1));
 419                                 switch (ack) {
 420                                 case ACK:
 421                                         flushes = 0;
 422                                         multi_ack = 0;
 423                                         retval = 0;
 424                                         goto done;
 425                                 case ACK_common:
 426                                 case ACK_ready:
 427                                 case ACK_continue: {
 428                                         struct commit *commit =
 429                                                 lookup_commit(result_sha1);
 430                                         if (!commit)
 431                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 432                                         if (args.stateless_rpc
 433                                          && ack == ACK_common
 434                                          && !(commit->object.flags & COMMON)) {
 435                                                 /* We need to replay the have for this object
 436                                                  * on the next RPC request so the peer knows
 437                                                  * it is in common with us.
 438                                                  */
 439                                                 const char *hex = sha1_to_hex(result_sha1);
 440                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 441                                                 state_len = req_buf.len;
 442                                         }
 443                                         mark_common(commit, 0, 1);
 444                                         retval = 0;
 445                                         in_vain = 0;
 446                                         got_continue = 1;
 447                                         if (ack == ACK_ready) {
 448                                                 rev_list = NULL;
 449                                                 got_ready = 1;
 450                                         }
 451                                         break;
 452                                         }
 453                                 }
 454                         } while (ack);
 455                         flushes--;
 456                         if (got_continue && MAX_IN_VAIN < in_vain) {
 457                                 if (args.verbose)
 458                                         fprintf(stderr, "giving up\n");
 459                                 break; /* give up */
 460                         }
 461                 }
 462         }
 463 done:
 464         if (!got_ready || !no_done) {
 465                 packet_buf_write(&req_buf, "done\n");
 466                 send_request(fd[1], &req_buf);
 467         }
 468         if (args.verbose)
 469                 fprintf(stderr, "done\n");
 470         if (retval != 0) {
 471                 multi_ack = 0;
 472                 flushes++;
 473         }
 474         strbuf_release(&req_buf);
 475
 476         consume_shallow_list(fd[0]);
 477         while (flushes || multi_ack) {
 478                 int ack = get_ack(fd[0], result_sha1);
 479                 if (ack) {
 480                         if (args.verbose)
 481                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 482                                         sha1_to_hex(result_sha1));
 483                         if (ack == ACK)
 484                                 return 0;
 485                         multi_ack = 1;
 486                         continue;
 487                 }
 488                 flushes--;
 489         }
 490         /* it is no error to fetch into a completely empty repo */
 491         return count ? retval : 0;
 492 }
 493
 494 static struct commit_list *complete;
 495
 496 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 497 {
 498         struct object *o = parse_object(sha1);
 499
 500         while (o && o->type == OBJ_TAG) {
 501                 struct tag *t = (struct tag *) o;
 502                 if (!t->tagged)
 503                         break; /* broken repository */
 504                 o->flags |= COMPLETE;
 505                 o = parse_object(t->tagged->sha1);
 506         }
 507         if (o && o->type == OBJ_COMMIT) {
 508                 struct commit *commit = (struct commit *)o;
 509                 if (!(commit->object.flags & COMPLETE)) {
 510                         commit->object.flags |= COMPLETE;
 511                         commit_list_insert_by_date(commit, &complete);
 512                 }
 513         }
 514         return 0;
 515 }
 516
 517 static void mark_recent_complete_commits(unsigned long cutoff)
 518 {
 519         while (complete && cutoff <= complete->item->date) {
 520                 if (args.verbose)
 521                         fprintf(stderr, "Marking %s as complete\n",
 522                                 sha1_to_hex(complete->item->object.sha1));
 523                 pop_most_recent_commit(&complete, COMPLETE);
 524         }
 525 }
 526
 527 static void filter_refs(struct ref **refs, int nr_match, char **match)
 528 {
 529         struct ref **return_refs;
 530         struct ref *newlist = NULL;
 531         struct ref **newtail = &newlist;
 532         struct ref *ref, *next;
 533         struct ref *fastarray[32];
 534
 535         if (nr_match && !args.fetch_all) {
 536                 if (ARRAY_SIZE(fastarray) < nr_match)
 537                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 538                 else {
 539                         return_refs = fastarray;
 540                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 541                 }
 542         }
 543         else
 544                 return_refs = NULL;
 545
 546         for (ref = *refs; ref; ref = next) {
 547                 next = ref->next;
 548                 if (!memcmp(ref->name, "refs/", 5) &&
 549                     check_refname_format(ref->name + 5, 0))
 550                         ; /* trash */
 551                 else if (args.fetch_all &&
 552                          (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
 553                         *newtail = ref;
 554                         ref->next = NULL;
 555                         newtail = &ref->next;
 556                         continue;
 557                 }
 558                 else {
 559                         int i;
 560                         for (i = 0; i < nr_match; i++) {
 561                                 if (!strcmp(ref->name, match[i])) {
 562                                         match[i][0] = '\0';
 563                                         return_refs[i] = ref;
 564                                         break;
 565                                 }
 566                         }
 567                         if (i < nr_match)
 568                                 continue; /* we will link it later */
 569                 }
 570                 free(ref);
 571         }
 572
 573         if (!args.fetch_all) {
 574                 int i;
 575                 for (i = 0; i < nr_match; i++) {
 576                         ref = return_refs[i];
 577                         if (ref) {
 578                                 *newtail = ref;
 579                                 ref->next = NULL;
 580                                 newtail = &ref->next;
 581                         }
 582                 }
 583                 if (return_refs != fastarray)
 584                         free(return_refs);
 585         }
 586         *refs = newlist;
 587 }
 588
 589 static int everything_local(struct ref **refs, int nr_match, char **match)
 590 {
 591         struct ref *ref;
 592         int retval;
 593         unsigned long cutoff = 0;
 594
 595         save_commit_buffer = 0;
 596
 597         for (ref = *refs; ref; ref = ref->next) {
 598                 struct object *o;
 599
 600                 o = parse_object(ref->old_sha1);
 601                 if (!o)
 602                         continue;
 603
 604                 /* We already have it -- which may mean that we were
 605                  * in sync with the other side at some time after
 606                  * that (it is OK if we guess wrong here).
 607                  */
 608                 if (o->type == OBJ_COMMIT) {
 609                         struct commit *commit = (struct commit *)o;
 610                         if (!cutoff || cutoff < commit->date)
 611                                 cutoff = commit->date;
 612                 }
 613         }
 614
 615         if (!args.depth) {
 616                 for_each_ref(mark_complete, NULL);
 617                 if (cutoff)
 618                         mark_recent_complete_commits(cutoff);
 619         }
 620
 621         /*
 622          * Mark all complete remote refs as common refs.
 623          * Don't mark them common yet; the server has to be told so first.
 624          */
 625         for (ref = *refs; ref; ref = ref->next) {
 626                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 627                                              NULL, 0);
 628
 629                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 630                         continue;
 631
 632                 if (!(o->flags & SEEN)) {
 633                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 634
 635                         mark_common((struct commit *)o, 1, 1);
 636                 }
 637         }
 638
 639         filter_refs(refs, nr_match, match);
 640
 641         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 642                 const unsigned char *remote = ref->old_sha1;
 643                 unsigned char local[20];
 644                 struct object *o;
 645
 646                 o = lookup_object(remote);
 647                 if (!o || !(o->flags & COMPLETE)) {
 648                         retval = 0;
 649                         if (!args.verbose)
 650                                 continue;
 651                         fprintf(stderr,
 652                                 "want %s (%s)\n", sha1_to_hex(remote),
 653                                 ref->name);
 654                         continue;
 655                 }
 656
 657                 hashcpy(ref->new_sha1, local);
 658                 if (!args.verbose)
 659                         continue;
 660                 fprintf(stderr,
 661                         "already have %s (%s)\n", sha1_to_hex(remote),
 662                         ref->name);
 663         }
 664         return retval;
 665 }
 666
 667 static int sideband_demux(int in, int out, void *data)
 668 {
 669         int *xd = data;
 670
 671         int ret = recv_sideband("fetch-pack", xd[0], out);
 672         close(out);
 673         return ret;
 674 }
 675
 676 static int get_pack(int xd[2], char **pack_lockfile)
 677 {
 678         struct async demux;
 679         const char *argv[20];
 680         char keep_arg[256];
 681         char hdr_arg[256];
 682         const char **av;
 683         int do_keep = args.keep_pack;
 684         struct child_process cmd;
 685
 686         memset(&demux, 0, sizeof(demux));
 687         if (use_sideband) {
 688                 /* xd[] is talking with upload-pack; subprocess reads from
 689                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 690                  * through demux->out.
 691                  */
 692                 demux.proc = sideband_demux;
 693                 demux.data = xd;
 694                 demux.out = -1;
 695                 if (start_async(&demux))
 696                         die("fetch-pack: unable to fork off sideband"
 697                             " demultiplexer");
 698         }
 699         else
 700                 demux.out = xd[0];
 701
 702         memset(&cmd, 0, sizeof(cmd));
 703         cmd.argv = argv;
 704         av = argv;
 705         *hdr_arg = 0;
 706         if (!args.keep_pack && unpack_limit) {
 707                 struct pack_header header;
 708
 709                 if (read_pack_header(demux.out, &header))
 710                         die("protocol error: bad pack header");
 711                 snprintf(hdr_arg, sizeof(hdr_arg),
 712                          "--pack_header=%"PRIu32",%"PRIu32,
 713                          ntohl(header.hdr_version), ntohl(header.hdr_entries));
 714                 if (ntohl(header.hdr_entries) < unpack_limit)
 715                         do_keep = 0;
 716                 else
 717                         do_keep = 1;
 718         }
 719
 720         if (do_keep) {
 721                 if (pack_lockfile)
 722                         cmd.out = -1;
 723                 *av++ = "index-pack";
 724                 *av++ = "--stdin";
 725                 if (!args.quiet && !args.no_progress)
 726                         *av++ = "-v";
 727                 if (args.use_thin_pack)
 728                         *av++ = "--fix-thin";
 729                 if (args.lock_pack || unpack_limit) {
 730                         int s = sprintf(keep_arg,
 731                                         "--keep=fetch-pack %"PRIuMAX " on ", (uintmax_t) getpid());
 732                         if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 733                                 strcpy(keep_arg + s, "localhost");
 734                         *av++ = keep_arg;
 735                 }
 736         }
 737         else {
 738                 *av++ = "unpack-objects";
 739                 if (args.quiet)
 740                         *av++ = "-q";
 741         }
 742         if (*hdr_arg)
 743                 *av++ = hdr_arg;
 744         if (fetch_fsck_objects >= 0
 745             ? fetch_fsck_objects
 746             : transfer_fsck_objects >= 0
 747             ? transfer_fsck_objects
 748             : 0)
 749                 *av++ = "--strict";
 750         *av++ = NULL;
 751
 752         cmd.in = demux.out;
 753         cmd.git_cmd = 1;
 754         if (start_command(&cmd))
 755                 die("fetch-pack: unable to fork off %s", argv[0]);
 756         if (do_keep && pack_lockfile) {
 757                 *pack_lockfile = index_pack_lockfile(cmd.out);
 758                 close(cmd.out);
 759         }
 760
 761         if (finish_command(&cmd))
 762                 die("%s failed", argv[0]);
 763         if (use_sideband && finish_async(&demux))
 764                 die("error in sideband demultiplexer");
 765         return 0;
 766 }
 767
 768 static struct ref *do_fetch_pack(int fd[2],
 769                 const struct ref *orig_ref,
 770                 int nr_match,
 771                 char **match,
 772                 char **pack_lockfile)
 773 {
 774         struct ref *ref = copy_ref_list(orig_ref);
 775         unsigned char sha1[20];
 776
 777         if (is_repository_shallow() && !server_supports("shallow"))
 778                 die("Server does not support shallow clients");
 779         if (server_supports("multi_ack_detailed")) {
 780                 if (args.verbose)
 781                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 782                 multi_ack = 2;
 783                 if (server_supports("no-done")) {
 784                         if (args.verbose)
 785                                 fprintf(stderr, "Server supports no-done\n");
 786                         if (args.stateless_rpc)
 787                                 no_done = 1;
 788                 }
 789         }
 790         else if (server_supports("multi_ack")) {
 791                 if (args.verbose)
 792                         fprintf(stderr, "Server supports multi_ack\n");
 793                 multi_ack = 1;
 794         }
 795         if (server_supports("side-band-64k")) {
 796                 if (args.verbose)
 797                         fprintf(stderr, "Server supports side-band-64k\n");
 798                 use_sideband = 2;
 799         }
 800         else if (server_supports("side-band")) {
 801                 if (args.verbose)
 802                         fprintf(stderr, "Server supports side-band\n");
 803                 use_sideband = 1;
 804         }
 805         if (server_supports("ofs-delta")) {
 806                 if (args.verbose)
 807                         fprintf(stderr, "Server supports ofs-delta\n");
 808         } else
 809                 prefer_ofs_delta = 0;
 810         if (everything_local(&ref, nr_match, match)) {
 811                 packet_flush(fd[1]);
 812                 goto all_done;
 813         }
 814         if (find_common(fd, sha1, ref) < 0)
 815                 if (!args.keep_pack)
 816                         /* When cloning, it is not unusual to have
 817                          * no common commit.
 818                          */
 819                         warning("no common commits");
 820
 821         if (args.stateless_rpc)
 822                 packet_flush(fd[1]);
 823         if (get_pack(fd, pack_lockfile))
 824                 die("git fetch-pack: fetch failed.");
 825
 826  all_done:
 827         return ref;
 828 }
 829
 830 static int remove_duplicates(int nr_heads, char **heads)
 831 {
 832         int src, dst;
 833
 834         for (src = dst = 0; src < nr_heads; src++) {
 835                 /* If heads[src] is different from any of
 836                  * heads[0..dst], push it in.
 837                  */
 838                 int i;
 839                 for (i = 0; i < dst; i++) {
 840                         if (!strcmp(heads[i], heads[src]))
 841                                 break;
 842                 }
 843                 if (i < dst)
 844                         continue;
 845                 if (src != dst)
 846                         heads[dst] = heads[src];
 847                 dst++;
 848         }
 849         return dst;
 850 }
 851
 852 static int fetch_pack_config(const char *var, const char *value, void *cb)
 853 {
 854         if (strcmp(var, "fetch.unpacklimit") == 0) {
 855                 fetch_unpack_limit = git_config_int(var, value);
 856                 return 0;
 857         }
 858
 859         if (strcmp(var, "transfer.unpacklimit") == 0) {
 860                 transfer_unpack_limit = git_config_int(var, value);
 861                 return 0;
 862         }
 863
 864         if (strcmp(var, "repack.usedeltabaseoffset") == 0) {
 865                 prefer_ofs_delta = git_config_bool(var, value);
 866                 return 0;
 867         }
 868
 869         if (!strcmp(var, "fetch.fsckobjects")) {
 870                 fetch_fsck_objects = git_config_bool(var, value);
 871                 return 0;
 872         }
 873
 874         if (!strcmp(var, "transfer.fsckobjects")) {
 875                 transfer_fsck_objects = git_config_bool(var, value);
 876                 return 0;
 877         }
 878
 879         return git_default_config(var, value, cb);
 880 }
 881
 882 static struct lock_file lock;
 883
 884 static void fetch_pack_setup(void)
 885 {
 886         static int did_setup;
 887         if (did_setup)
 888                 return;
 889         git_config(fetch_pack_config, NULL);
 890         if (0 <= transfer_unpack_limit)
 891                 unpack_limit = transfer_unpack_limit;
 892         else if (0 <= fetch_unpack_limit)
 893                 unpack_limit = fetch_unpack_limit;
 894         did_setup = 1;
 895 }
 896
 897 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
 898 {
 899         int i, ret, nr_heads;
 900         struct ref *ref = NULL;
 901         char *dest = NULL, **heads;
 902         int fd[2];
 903         char *pack_lockfile = NULL;
 904         char **pack_lockfile_ptr = NULL;
 905         struct child_process *conn;
 906
 907         packet_trace_identity("fetch-pack");
 908
 909         nr_heads = 0;
 910         heads = NULL;
 911         for (i = 1; i < argc; i++) {
 912                 const char *arg = argv[i];
 913
 914                 if (*arg == '-') {
 915                         if (!prefixcmp(arg, "--upload-pack=")) {
 916                                 args.uploadpack = arg + 14;
 917                                 continue;
 918                         }
 919                         if (!prefixcmp(arg, "--exec=")) {
 920                                 args.uploadpack = arg + 7;
 921                                 continue;
 922                         }
 923                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 924                                 args.quiet = 1;
 925                                 continue;
 926                         }
 927                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 928                                 args.lock_pack = args.keep_pack;
 929                                 args.keep_pack = 1;
 930                                 continue;
 931                         }
 932                         if (!strcmp("--thin", arg)) {
 933                                 args.use_thin_pack = 1;
 934                                 continue;
 935                         }
 936                         if (!strcmp("--include-tag", arg)) {
 937                                 args.include_tag = 1;
 938                                 continue;
 939                         }
 940                         if (!strcmp("--all", arg)) {
 941                                 args.fetch_all = 1;
 942                                 continue;
 943                         }
 944                         if (!strcmp("-v", arg)) {
 945                                 args.verbose = 1;
 946                                 continue;
 947                         }
 948                         if (!prefixcmp(arg, "--depth=")) {
 949                                 args.depth = strtol(arg + 8, NULL, 0);
 950                                 continue;
 951                         }
 952                         if (!strcmp("--no-progress", arg)) {
 953                                 args.no_progress = 1;
 954                                 continue;
 955                         }
 956                         if (!strcmp("--stateless-rpc", arg)) {
 957                                 args.stateless_rpc = 1;
 958                                 continue;
 959                         }
 960                         if (!strcmp("--lock-pack", arg)) {
 961                                 args.lock_pack = 1;
 962                                 pack_lockfile_ptr = &pack_lockfile;
 963                                 continue;
 964                         }
 965                         usage(fetch_pack_usage);
 966                 }
 967                 dest = (char *)arg;
 968                 heads = (char **)(argv + i + 1);
 969                 nr_heads = argc - i - 1;
 970                 break;
 971         }
 972         if (!dest)
 973                 usage(fetch_pack_usage);
 974
 975         if (args.stateless_rpc) {
 976                 conn = NULL;
 977                 fd[0] = 0;
 978                 fd[1] = 1;
 979         } else {
 980                 conn = git_connect(fd, (char *)dest, args.uploadpack,
 981                                    args.verbose ? CONNECT_VERBOSE : 0);
 982         }
 983
 984         get_remote_heads(fd[0], &ref, 0, NULL);
 985
 986         ref = fetch_pack(&args, fd, conn, ref, dest,
 987                 nr_heads, heads, pack_lockfile_ptr);
 988         if (pack_lockfile) {
 989                 printf("lock %s\n", pack_lockfile);
 990                 fflush(stdout);
 991         }
 992         close(fd[0]);
 993         close(fd[1]);
 994         if (finish_connect(conn))
 995                 ref = NULL;
 996         ret = !ref;
 997
 998         if (!ret && nr_heads) {
 999                 /* If the heads to pull were given, we should have
1000                  * consumed all of them by matching the remote.
1001                  * Otherwise, 'git fetch remote no-such-ref' would
1002                  * silently succeed without issuing an error.
1003                  */
1004                 for (i = 0; i < nr_heads; i++)
1005                         if (heads[i] && heads[i][0]) {
1006                                 error("no such remote ref %s", heads[i]);
1007                                 ret = 1;
1008                         }
1009         }
1010         while (ref) {
1011                 printf("%s %s\n",
1012                        sha1_to_hex(ref->old_sha1), ref->name);
1013                 ref = ref->next;
1014         }
1015
1016         return ret;
1017 }
1018
1019 struct ref *fetch_pack(struct fetch_pack_args *my_args,
1020                        int fd[], struct child_process *conn,
1021                        const struct ref *ref,
1022                 const char *dest,
1023                 int nr_heads,
1024                 char **heads,
1025                 char **pack_lockfile)
1026 {
1027         struct stat st;
1028         struct ref *ref_cpy;
1029
1030         fetch_pack_setup();
1031         if (&args != my_args)
1032                 memcpy(&args, my_args, sizeof(args));
1033         if (args.depth > 0) {
1034                 if (stat(git_path("shallow"), &st))
1035                         st.st_mtime = 0;
1036         }
1037
1038         if (heads && nr_heads)
1039                 nr_heads = remove_duplicates(nr_heads, heads);
1040         if (!ref) {
1041                 packet_flush(fd[1]);
1042                 die("no matching remote head");
1043         }
1044         ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
1045
1046         if (args.depth > 0) {
1047                 struct cache_time mtime;
1048                 struct strbuf sb = STRBUF_INIT;
1049                 char *shallow = git_path("shallow");
1050                 int fd;
1051
1052                 mtime.sec = st.st_mtime;
1053                 mtime.nsec = ST_MTIME_NSEC(st);
1054                 if (stat(shallow, &st)) {
1055                         if (mtime.sec)
1056                                 die("shallow file was removed during fetch");
1057                 } else if (st.st_mtime != mtime.sec
1058 #ifdef USE_NSEC
1059                                 || ST_MTIME_NSEC(st) != mtime.nsec
1060 #endif
1061                           )
1062                         die("shallow file was changed during fetch");
1063
1064                 fd = hold_lock_file_for_update(&lock, shallow,
1065                                                LOCK_DIE_ON_ERROR);
1066                 if (!write_shallow_commits(&sb, 0)
1067                  || write_in_full(fd, sb.buf, sb.len) != sb.len) {
1068                         unlink_or_warn(shallow);
1069                         rollback_lock_file(&lock);
1070                 } else {
1071                         commit_lock_file(&lock);
1072                 }
1073                 strbuf_release(&sb);
1074         }
1075
1076         reprepare_packed_git();
1077         return ref_cpy;
1078 }