git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "sideband.h"
   8 #include <sys/wait.h>
   9
  10 static int keep_pack;
  11 static int quiet;
  12 static int verbose;
  13 static int fetch_all;
  14 static int depth;
  15 static const char fetch_pack_usage[] =
  16 "git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [--depth=<n>] [host:]directory <refs>...";
  17 static const char *exec = "git-upload-pack";
  18
  19 #define COMPLETE        (1U << 0)
  20 #define COMMON          (1U << 1)
  21 #define COMMON_REF      (1U << 2)
  22 #define SEEN            (1U << 3)
  23 #define POPPED          (1U << 4)
  24
  25 /*
  26  * After sending this many "have"s if we do not get any new ACK , we
  27  * give up traversing our history.
  28  */
  29 #define MAX_IN_VAIN 256
  30
  31 static struct commit_list *rev_list;
  32 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
  33
  34 static void rev_list_push(struct commit *commit, int mark)
  35 {
  36         if (!(commit->object.flags & mark)) {
  37                 commit->object.flags |= mark;
  38
  39                 if (!(commit->object.parsed))
  40                         parse_commit(commit);
  41
  42                 insert_by_date(commit, &rev_list);
  43
  44                 if (!(commit->object.flags & COMMON))
  45                         non_common_revs++;
  46         }
  47 }
  48
  49 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  50 {
  51         struct object *o = deref_tag(parse_object(sha1), path, 0);
  52
  53         if (o && o->type == OBJ_COMMIT)
  54                 rev_list_push((struct commit *)o, SEEN);
  55
  56         return 0;
  57 }
  58
  59 /*
  60    This function marks a rev and its ancestors as common.
  61    In some cases, it is desirable to mark only the ancestors (for example
  62    when only the server does not yet know that they are common).
  63 */
  64
  65 static void mark_common(struct commit *commit,
  66                 int ancestors_only, int dont_parse)
  67 {
  68         if (commit != NULL && !(commit->object.flags & COMMON)) {
  69                 struct object *o = (struct object *)commit;
  70
  71                 if (!ancestors_only)
  72                         o->flags |= COMMON;
  73
  74                 if (!(o->flags & SEEN))
  75                         rev_list_push(commit, SEEN);
  76                 else {
  77                         struct commit_list *parents;
  78
  79                         if (!ancestors_only && !(o->flags & POPPED))
  80                                 non_common_revs--;
  81                         if (!o->parsed && !dont_parse)
  82                                 parse_commit(commit);
  83
  84                         for (parents = commit->parents;
  85                                         parents;
  86                                         parents = parents->next)
  87                                 mark_common(parents->item, 0, dont_parse);
  88                 }
  89         }
  90 }
  91
  92 /*
  93   Get the next rev to send, ignoring the common.
  94 */
  95
  96 static const unsigned char* get_rev(void)
  97 {
  98         struct commit *commit = NULL;
  99
 100         while (commit == NULL) {
 101                 unsigned int mark;
 102                 struct commit_list* parents;
 103
 104                 if (rev_list == NULL || non_common_revs == 0)
 105                         return NULL;
 106
 107                 commit = rev_list->item;
 108                 if (!(commit->object.parsed))
 109                         parse_commit(commit);
 110                 commit->object.flags |= POPPED;
 111                 if (!(commit->object.flags & COMMON))
 112                         non_common_revs--;
 113
 114                 parents = commit->parents;
 115
 116                 if (commit->object.flags & COMMON) {
 117                         /* do not send "have", and ignore ancestors */
 118                         commit = NULL;
 119                         mark = COMMON | SEEN;
 120                 } else if (commit->object.flags & COMMON_REF)
 121                         /* send "have", and ignore ancestors */
 122                         mark = COMMON | SEEN;
 123                 else
 124                         /* send "have", also for its ancestors */
 125                         mark = SEEN;
 126
 127                 while (parents) {
 128                         if (!(parents->item->object.flags & SEEN))
 129                                 rev_list_push(parents->item, mark);
 130                         if (mark & COMMON)
 131                                 mark_common(parents->item, 1, 0);
 132                         parents = parents->next;
 133                 }
 134
 135                 rev_list = rev_list->next;
 136         }
 137
 138         return commit->object.sha1;
 139 }
 140
 141 static int find_common(int fd[2], unsigned char *result_sha1,
 142                        struct ref *refs)
 143 {
 144         int fetching;
 145         int count = 0, flushes = 0, retval;
 146         const unsigned char *sha1;
 147         unsigned in_vain = 0;
 148         int got_continue = 0;
 149
 150         for_each_ref(rev_list_insert_ref, NULL);
 151
 152         fetching = 0;
 153         for ( ; refs ; refs = refs->next) {
 154                 unsigned char *remote = refs->old_sha1;
 155                 struct object *o;
 156
 157                 /*
 158                  * If that object is complete (i.e. it is an ancestor of a
 159                  * local ref), we tell them we have it but do not have to
 160                  * tell them about its ancestors, which they already know
 161                  * about.
 162                  *
 163                  * We use lookup_object here because we are only
 164                  * interested in the case we *know* the object is
 165                  * reachable and we have already scanned it.
 166                  */
 167                 if (((o = lookup_object(remote)) != NULL) &&
 168                                 (o->flags & COMPLETE)) {
 169                         continue;
 170                 }
 171
 172                 if (!fetching)
 173                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
 174                                      sha1_to_hex(remote),
 175                                      (multi_ack ? " multi_ack" : ""),
 176                                      (use_sideband == 2 ? " side-band-64k" : ""),
 177                                      (use_sideband == 1 ? " side-band" : ""),
 178                                      (use_thin_pack ? " thin-pack" : ""),
 179                                      " ofs-delta");
 180                 else
 181                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
 182                 fetching++;
 183         }
 184         if (is_repository_shallow())
 185                 write_shallow_commits(fd[1], 1);
 186         if (depth > 0)
 187                 packet_write(fd[1], "deepen %d", depth);
 188         packet_flush(fd[1]);
 189         if (!fetching)
 190                 return 1;
 191
 192         if (depth > 0) {
 193                 char line[1024];
 194                 unsigned char sha1[20];
 195                 int len;
 196
 197                 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
 198                         if (!strncmp("shallow ", line, 8)) {
 199                                 if (get_sha1_hex(line + 8, sha1))
 200                                         die("invalid shallow line: %s", line);
 201                                 register_shallow(sha1);
 202                                 continue;
 203                         }
 204                         if (!strncmp("unshallow ", line, 10)) {
 205                                 if (get_sha1_hex(line + 10, sha1))
 206                                         die("invalid unshallow line: %s", line);
 207                                 if (!lookup_object(sha1))
 208                                         die("object not found: %s", line);
 209                                 /* make sure that it is parsed as shallow */
 210                                 parse_object(sha1);
 211                                 if (unregister_shallow(sha1))
 212                                         die("no shallow found: %s", line);
 213                                 continue;
 214                         }
 215                         die("expected shallow/unshallow, got %s", line);
 216                 }
 217         }
 218
 219         flushes = 0;
 220         retval = -1;
 221         while ((sha1 = get_rev())) {
 222                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
 223                 if (verbose)
 224                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 225                 in_vain++;
 226                 if (!(31 & ++count)) {
 227                         int ack;
 228
 229                         packet_flush(fd[1]);
 230                         flushes++;
 231
 232                         /*
 233                          * We keep one window "ahead" of the other side, and
 234                          * will wait for an ACK only on the next one
 235                          */
 236                         if (count == 32)
 237                                 continue;
 238
 239                         do {
 240                                 ack = get_ack(fd[0], result_sha1);
 241                                 if (verbose && ack)
 242                                         fprintf(stderr, "got ack %d %s\n", ack,
 243                                                         sha1_to_hex(result_sha1));
 244                                 if (ack == 1) {
 245                                         flushes = 0;
 246                                         multi_ack = 0;
 247                                         retval = 0;
 248                                         goto done;
 249                                 } else if (ack == 2) {
 250                                         struct commit *commit =
 251                                                 lookup_commit(result_sha1);
 252                                         mark_common(commit, 0, 1);
 253                                         retval = 0;
 254                                         in_vain = 0;
 255                                         got_continue = 1;
 256                                 }
 257                         } while (ack);
 258                         flushes--;
 259                         if (got_continue && MAX_IN_VAIN < in_vain) {
 260                                 if (verbose)
 261                                         fprintf(stderr, "giving up\n");
 262                                 break; /* give up */
 263                         }
 264                 }
 265         }
 266 done:
 267         packet_write(fd[1], "done\n");
 268         if (verbose)
 269                 fprintf(stderr, "done\n");
 270         if (retval != 0) {
 271                 multi_ack = 0;
 272                 flushes++;
 273         }
 274         while (flushes || multi_ack) {
 275                 int ack = get_ack(fd[0], result_sha1);
 276                 if (ack) {
 277                         if (verbose)
 278                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 279                                         sha1_to_hex(result_sha1));
 280                         if (ack == 1)
 281                                 return 0;
 282                         multi_ack = 1;
 283                         continue;
 284                 }
 285                 flushes--;
 286         }
 287         return retval;
 288 }
 289
 290 static struct commit_list *complete;
 291
 292 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 293 {
 294         struct object *o = parse_object(sha1);
 295
 296         while (o && o->type == OBJ_TAG) {
 297                 struct tag *t = (struct tag *) o;
 298                 if (!t->tagged)
 299                         break; /* broken repository */
 300                 o->flags |= COMPLETE;
 301                 o = parse_object(t->tagged->sha1);
 302         }
 303         if (o && o->type == OBJ_COMMIT) {
 304                 struct commit *commit = (struct commit *)o;
 305                 commit->object.flags |= COMPLETE;
 306                 insert_by_date(commit, &complete);
 307         }
 308         return 0;
 309 }
 310
 311 static void mark_recent_complete_commits(unsigned long cutoff)
 312 {
 313         while (complete && cutoff <= complete->item->date) {
 314                 if (verbose)
 315                         fprintf(stderr, "Marking %s as complete\n",
 316                                 sha1_to_hex(complete->item->object.sha1));
 317                 pop_most_recent_commit(&complete, COMPLETE);
 318         }
 319 }
 320
 321 static void filter_refs(struct ref **refs, int nr_match, char **match)
 322 {
 323         struct ref **return_refs;
 324         struct ref *newlist = NULL;
 325         struct ref **newtail = &newlist;
 326         struct ref *ref, *next;
 327         struct ref *fastarray[32];
 328
 329         if (nr_match && !fetch_all) {
 330                 if (ARRAY_SIZE(fastarray) < nr_match)
 331                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 332                 else {
 333                         return_refs = fastarray;
 334                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 335                 }
 336         }
 337         else
 338                 return_refs = NULL;
 339
 340         for (ref = *refs; ref; ref = next) {
 341                 next = ref->next;
 342                 if (!memcmp(ref->name, "refs/", 5) &&
 343                     check_ref_format(ref->name + 5))
 344                         ; /* trash */
 345                 else if (fetch_all &&
 346                          (!depth || strncmp(ref->name, "refs/tags/", 10) )) {
 347                         *newtail = ref;
 348                         ref->next = NULL;
 349                         newtail = &ref->next;
 350                         continue;
 351                 }
 352                 else {
 353                         int order = path_match(ref->name, nr_match, match);
 354                         if (order) {
 355                                 return_refs[order-1] = ref;
 356                                 continue; /* we will link it later */
 357                         }
 358                 }
 359                 free(ref);
 360         }
 361
 362         if (!fetch_all) {
 363                 int i;
 364                 for (i = 0; i < nr_match; i++) {
 365                         ref = return_refs[i];
 366                         if (ref) {
 367                                 *newtail = ref;
 368                                 ref->next = NULL;
 369                                 newtail = &ref->next;
 370                         }
 371                 }
 372                 if (return_refs != fastarray)
 373                         free(return_refs);
 374         }
 375         *refs = newlist;
 376 }
 377
 378 static int everything_local(struct ref **refs, int nr_match, char **match)
 379 {
 380         struct ref *ref;
 381         int retval;
 382         unsigned long cutoff = 0;
 383
 384         track_object_refs = 0;
 385         save_commit_buffer = 0;
 386
 387         for (ref = *refs; ref; ref = ref->next) {
 388                 struct object *o;
 389
 390                 o = parse_object(ref->old_sha1);
 391                 if (!o)
 392                         continue;
 393
 394                 /* We already have it -- which may mean that we were
 395                  * in sync with the other side at some time after
 396                  * that (it is OK if we guess wrong here).
 397                  */
 398                 if (o->type == OBJ_COMMIT) {
 399                         struct commit *commit = (struct commit *)o;
 400                         if (!cutoff || cutoff < commit->date)
 401                                 cutoff = commit->date;
 402                 }
 403         }
 404
 405         if (!depth) {
 406                 for_each_ref(mark_complete, NULL);
 407                 if (cutoff)
 408                         mark_recent_complete_commits(cutoff);
 409         }
 410
 411         /*
 412          * Mark all complete remote refs as common refs.
 413          * Don't mark them common yet; the server has to be told so first.
 414          */
 415         for (ref = *refs; ref; ref = ref->next) {
 416                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 417                                              NULL, 0);
 418
 419                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 420                         continue;
 421
 422                 if (!(o->flags & SEEN)) {
 423                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 424
 425                         mark_common((struct commit *)o, 1, 1);
 426                 }
 427         }
 428
 429         filter_refs(refs, nr_match, match);
 430
 431         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 432                 const unsigned char *remote = ref->old_sha1;
 433                 unsigned char local[20];
 434                 struct object *o;
 435
 436                 o = lookup_object(remote);
 437                 if (!o || !(o->flags & COMPLETE)) {
 438                         retval = 0;
 439                         if (!verbose)
 440                                 continue;
 441                         fprintf(stderr,
 442                                 "want %s (%s)\n", sha1_to_hex(remote),
 443                                 ref->name);
 444                         continue;
 445                 }
 446
 447                 hashcpy(ref->new_sha1, local);
 448                 if (!verbose)
 449                         continue;
 450                 fprintf(stderr,
 451                         "already have %s (%s)\n", sha1_to_hex(remote),
 452                         ref->name);
 453         }
 454         return retval;
 455 }
 456
 457 static pid_t setup_sideband(int fd[2], int xd[2])
 458 {
 459         pid_t side_pid;
 460
 461         if (!use_sideband) {
 462                 fd[0] = xd[0];
 463                 fd[1] = xd[1];
 464                 return 0;
 465         }
 466         /* xd[] is talking with upload-pack; subprocess reads from
 467          * xd[0], spits out band#2 to stderr, and feeds us band#1
 468          * through our fd[0].
 469          */
 470         if (pipe(fd) < 0)
 471                 die("fetch-pack: unable to set up pipe");
 472         side_pid = fork();
 473         if (side_pid < 0)
 474                 die("fetch-pack: unable to fork off sideband demultiplexer");
 475         if (!side_pid) {
 476                 /* subprocess */
 477                 close(fd[0]);
 478                 if (xd[0] != xd[1])
 479                         close(xd[1]);
 480                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
 481                         exit(1);
 482                 exit(0);
 483         }
 484         close(xd[0]);
 485         close(fd[1]);
 486         fd[1] = xd[1];
 487         return side_pid;
 488 }
 489
 490 static int get_pack(int xd[2], const char **argv)
 491 {
 492         int status;
 493         pid_t pid, side_pid;
 494         int fd[2];
 495
 496         side_pid = setup_sideband(fd, xd);
 497         pid = fork();
 498         if (pid < 0)
 499                 die("fetch-pack: unable to fork off %s", argv[0]);
 500         if (!pid) {
 501                 dup2(fd[0], 0);
 502                 close(fd[0]);
 503                 close(fd[1]);
 504                 execv_git_cmd(argv);
 505                 die("%s exec failed", argv[0]);
 506         }
 507         close(fd[0]);
 508         close(fd[1]);
 509         while (waitpid(pid, &status, 0) < 0) {
 510                 if (errno != EINTR)
 511                         die("waiting for %s: %s", argv[0], strerror(errno));
 512         }
 513         if (WIFEXITED(status)) {
 514                 int code = WEXITSTATUS(status);
 515                 if (code)
 516                         die("%s died with error code %d", argv[0], code);
 517                 return 0;
 518         }
 519         if (WIFSIGNALED(status)) {
 520                 int sig = WTERMSIG(status);
 521                 die("%s died of signal %d", argv[0], sig);
 522         }
 523         die("%s died of unnatural causes %d", argv[0], status);
 524 }
 525
 526 static int explode_rx_pack(int xd[2])
 527 {
 528         const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
 529         return get_pack(xd, argv);
 530 }
 531
 532 static int keep_rx_pack(int xd[2])
 533 {
 534         const char *argv[6];
 535         char keep_arg[256];
 536         int n = 0;
 537
 538         argv[n++] = "index-pack";
 539         argv[n++] = "--stdin";
 540         if (!quiet)
 541                 argv[n++] = "-v";
 542         if (use_thin_pack)
 543                 argv[n++] = "--fix-thin";
 544         if (keep_pack > 1) {
 545                 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
 546                 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 547                         strcpy(keep_arg + s, "localhost");
 548                 argv[n++] = keep_arg;
 549         }
 550         argv[n] = NULL;
 551         return get_pack(xd, argv);
 552 }
 553
 554 static int fetch_pack(int fd[2], int nr_match, char **match)
 555 {
 556         struct ref *ref;
 557         unsigned char sha1[20];
 558         int status;
 559
 560         get_remote_heads(fd[0], &ref, 0, NULL, 0);
 561         if (is_repository_shallow() && !server_supports("shallow"))
 562                 die("Server does not support shallow clients");
 563         if (server_supports("multi_ack")) {
 564                 if (verbose)
 565                         fprintf(stderr, "Server supports multi_ack\n");
 566                 multi_ack = 1;
 567         }
 568         if (server_supports("side-band-64k")) {
 569                 if (verbose)
 570                         fprintf(stderr, "Server supports side-band-64k\n");
 571                 use_sideband = 2;
 572         }
 573         else if (server_supports("side-band")) {
 574                 if (verbose)
 575                         fprintf(stderr, "Server supports side-band\n");
 576                 use_sideband = 1;
 577         }
 578         if (!ref) {
 579                 packet_flush(fd[1]);
 580                 die("no matching remote head");
 581         }
 582         if (everything_local(&ref, nr_match, match)) {
 583                 packet_flush(fd[1]);
 584                 goto all_done;
 585         }
 586         if (find_common(fd, sha1, ref) < 0)
 587                 if (keep_pack != 1)
 588                         /* When cloning, it is not unusual to have
 589                          * no common commit.
 590                          */
 591                         fprintf(stderr, "warning: no common commits\n");
 592
 593         status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
 594         if (status)
 595                 die("git-fetch-pack: fetch failed.");
 596
 597  all_done:
 598         while (ref) {
 599                 printf("%s %s\n",
 600                        sha1_to_hex(ref->old_sha1), ref->name);
 601                 ref = ref->next;
 602         }
 603         return 0;
 604 }
 605
 606 int main(int argc, char **argv)
 607 {
 608         int i, ret, nr_heads;
 609         char *dest = NULL, **heads;
 610         int fd[2];
 611         pid_t pid;
 612         struct stat st;
 613         struct lock_file lock;
 614
 615         setup_git_directory();
 616
 617         nr_heads = 0;
 618         heads = NULL;
 619         for (i = 1; i < argc; i++) {
 620                 char *arg = argv[i];
 621
 622                 if (*arg == '-') {
 623                         if (!strncmp("--exec=", arg, 7)) {
 624                                 exec = arg + 7;
 625                                 continue;
 626                         }
 627                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 628                                 quiet = 1;
 629                                 continue;
 630                         }
 631                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 632                                 keep_pack++;
 633                                 continue;
 634                         }
 635                         if (!strcmp("--thin", arg)) {
 636                                 use_thin_pack = 1;
 637                                 continue;
 638                         }
 639                         if (!strcmp("--all", arg)) {
 640                                 fetch_all = 1;
 641                                 continue;
 642                         }
 643                         if (!strcmp("-v", arg)) {
 644                                 verbose = 1;
 645                                 continue;
 646                         }
 647                         if (!strncmp("--depth=", arg, 8)) {
 648                                 depth = strtol(arg + 8, NULL, 0);
 649                                 if (stat(git_path("shallow"), &st))
 650                                         st.st_mtime = 0;
 651                                 continue;
 652                         }
 653                         usage(fetch_pack_usage);
 654                 }
 655                 dest = arg;
 656                 heads = argv + i + 1;
 657                 nr_heads = argc - i - 1;
 658                 break;
 659         }
 660         if (!dest)
 661                 usage(fetch_pack_usage);
 662         pid = git_connect(fd, dest, exec);
 663         if (pid < 0)
 664                 return 1;
 665         ret = fetch_pack(fd, nr_heads, heads);
 666         close(fd[0]);
 667         close(fd[1]);
 668         ret |= finish_connect(pid);
 669
 670         if (!ret && nr_heads) {
 671                 /* If the heads to pull were given, we should have
 672                  * consumed all of them by matching the remote.
 673                  * Otherwise, 'git-fetch remote no-such-ref' would
 674                  * silently succeed without issuing an error.
 675                  */
 676                 for (i = 0; i < nr_heads; i++)
 677                         if (heads[i] && heads[i][0]) {
 678                                 error("no such remote ref %s", heads[i]);
 679                                 ret = 1;
 680                         }
 681         }
 682
 683         if (!ret && depth > 0) {
 684                 struct cache_time mtime;
 685                 char *shallow = git_path("shallow");
 686                 int fd;
 687
 688                 mtime.sec = st.st_mtime;
 689 #ifdef USE_NSEC
 690                 mtime.usec = st.st_mtim.usec;
 691 #endif
 692                 if (stat(shallow, &st)) {
 693                         if (mtime.sec)
 694                                 die("shallow file was removed during fetch");
 695                 } else if (st.st_mtime != mtime.sec
 696 #ifdef USE_NSEC
 697                                 || st.st_mtim.usec != mtime.usec
 698 #endif
 699                           )
 700                         die("shallow file was changed during fetch");
 701
 702                 fd = hold_lock_file_for_update(&lock, shallow, 1);
 703                 if (!write_shallow_commits(fd, 0)) {
 704                         unlink(shallow);
 705                         rollback_lock_file(&lock);
 706                 } else {
 707                         close(fd);
 708                         commit_lock_file(&lock);
 709                 }
 710         }
 711
 712         return !!ret;
 713 }