git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "refs.h"
   3 #include "pkt-line.h"
   4 #include "commit.h"
   5 #include "tag.h"
   6 #include "exec_cmd.h"
   7 #include "sideband.h"
   8 #include <sys/wait.h>
   9
  10 static int keep_pack;
  11 static int quiet;
  12 static int verbose;
  13 static int fetch_all;
  14 static int depth;
  15 static const char fetch_pack_usage[] =
  16 "git-fetch-pack [--all] [-q] [-v] [-k] [--thin] [--exec=upload-pack] [--depth=<n>] [host:]directory <refs>...";
  17 static const char *exec = "git-upload-pack";
  18
  19 #define COMPLETE        (1U << 0)
  20 #define COMMON          (1U << 1)
  21 #define COMMON_REF      (1U << 2)
  22 #define SEEN            (1U << 3)
  23 #define POPPED          (1U << 4)
  24
  25 /*
  26  * After sending this many "have"s if we do not get any new ACK , we
  27  * give up traversing our history.
  28  */
  29 #define MAX_IN_VAIN 256
  30
  31 static struct commit_list *rev_list;
  32 static int non_common_revs, multi_ack, use_thin_pack, use_sideband;
  33
  34 static void rev_list_push(struct commit *commit, int mark)
  35 {
  36         if (!(commit->object.flags & mark)) {
  37                 commit->object.flags |= mark;
  38
  39                 if (!(commit->object.parsed))
  40                         parse_commit(commit);
  41
  42                 insert_by_date(commit, &rev_list);
  43
  44                 if (!(commit->object.flags & COMMON))
  45                         non_common_revs++;
  46         }
  47 }
  48
  49 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
  50 {
  51         struct object *o = deref_tag(parse_object(sha1), path, 0);
  52
  53         if (o && o->type == OBJ_COMMIT)
  54                 rev_list_push((struct commit *)o, SEEN);
  55
  56         return 0;
  57 }
  58
  59 /*
  60    This function marks a rev and its ancestors as common.
  61    In some cases, it is desirable to mark only the ancestors (for example
  62    when only the server does not yet know that they are common).
  63 */
  64
  65 static void mark_common(struct commit *commit,
  66                 int ancestors_only, int dont_parse)
  67 {
  68         if (commit != NULL && !(commit->object.flags & COMMON)) {
  69                 struct object *o = (struct object *)commit;
  70
  71                 if (!ancestors_only)
  72                         o->flags |= COMMON;
  73
  74                 if (!(o->flags & SEEN))
  75                         rev_list_push(commit, SEEN);
  76                 else {
  77                         struct commit_list *parents;
  78
  79                         if (!ancestors_only && !(o->flags & POPPED))
  80                                 non_common_revs--;
  81                         if (!o->parsed && !dont_parse)
  82                                 parse_commit(commit);
  83
  84                         for (parents = commit->parents;
  85                                         parents;
  86                                         parents = parents->next)
  87                                 mark_common(parents->item, 0, dont_parse);
  88                 }
  89         }
  90 }
  91
  92 /*
  93   Get the next rev to send, ignoring the common.
  94 */
  95
  96 static const unsigned char* get_rev(void)
  97 {
  98         struct commit *commit = NULL;
  99
 100         while (commit == NULL) {
 101                 unsigned int mark;
 102                 struct commit_list* parents;
 103
 104                 if (rev_list == NULL || non_common_revs == 0)
 105                         return NULL;
 106
 107                 commit = rev_list->item;
 108                 if (!(commit->object.parsed))
 109                         parse_commit(commit);
 110                 commit->object.flags |= POPPED;
 111                 if (!(commit->object.flags & COMMON))
 112                         non_common_revs--;
 113
 114                 parents = commit->parents;
 115
 116                 if (commit->object.flags & COMMON) {
 117                         /* do not send "have", and ignore ancestors */
 118                         commit = NULL;
 119                         mark = COMMON | SEEN;
 120                 } else if (commit->object.flags & COMMON_REF)
 121                         /* send "have", and ignore ancestors */
 122                         mark = COMMON | SEEN;
 123                 else
 124                         /* send "have", also for its ancestors */
 125                         mark = SEEN;
 126
 127                 while (parents) {
 128                         if (!(parents->item->object.flags & SEEN))
 129                                 rev_list_push(parents->item, mark);
 130                         if (mark & COMMON)
 131                                 mark_common(parents->item, 1, 0);
 132                         parents = parents->next;
 133                 }
 134
 135                 rev_list = rev_list->next;
 136         }
 137
 138         return commit->object.sha1;
 139 }
 140
 141 static int find_common(int fd[2], unsigned char *result_sha1,
 142                        struct ref *refs)
 143 {
 144         int fetching;
 145         int count = 0, flushes = 0, retval;
 146         const unsigned char *sha1;
 147         unsigned in_vain = 0;
 148         int got_continue = 0;
 149
 150         for_each_ref(rev_list_insert_ref, NULL);
 151
 152         fetching = 0;
 153         for ( ; refs ; refs = refs->next) {
 154                 unsigned char *remote = refs->old_sha1;
 155                 struct object *o;
 156
 157                 /*
 158                  * If that object is complete (i.e. it is an ancestor of a
 159                  * local ref), we tell them we have it but do not have to
 160                  * tell them about its ancestors, which they already know
 161                  * about.
 162                  *
 163                  * We use lookup_object here because we are only
 164                  * interested in the case we *know* the object is
 165                  * reachable and we have already scanned it.
 166                  */
 167                 if (((o = lookup_object(remote)) != NULL) &&
 168                                 (o->flags & COMPLETE)) {
 169                         continue;
 170                 }
 171
 172                 if (!fetching)
 173                         packet_write(fd[1], "want %s%s%s%s%s%s\n",
 174                                      sha1_to_hex(remote),
 175                                      (multi_ack ? " multi_ack" : ""),
 176                                      (use_sideband == 2 ? " side-band-64k" : ""),
 177                                      (use_sideband == 1 ? " side-band" : ""),
 178                                      (use_thin_pack ? " thin-pack" : ""),
 179                                      " ofs-delta");
 180                 else
 181                         packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
 182                 fetching++;
 183         }
 184         if (is_repository_shallow())
 185                 write_shallow_commits(fd[1], 1);
 186         if (depth > 0)
 187                 packet_write(fd[1], "deepen %d", depth);
 188         packet_flush(fd[1]);
 189         if (!fetching)
 190                 return 1;
 191
 192         if (depth > 0) {
 193                 char line[1024];
 194                 unsigned char sha1[20];
 195                 int len;
 196
 197                 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
 198                         if (!strncmp("shallow ", line, 8)) {
 199                                 if (get_sha1_hex(line + 8, sha1))
 200                                         die("invalid shallow line: %s", line);
 201                                 /* no need making it shallow if we have it already */
 202                                 if (lookup_object(sha1))
 203                                         continue;
 204                                 register_shallow(sha1);
 205                                 continue;
 206                         }
 207                         if (!strncmp("unshallow ", line, 10)) {
 208                                 if (get_sha1_hex(line + 10, sha1))
 209                                         die("invalid unshallow line: %s", line);
 210                                 if (!lookup_object(sha1))
 211                                         die("object not found: %s", line);
 212                                 /* make sure that it is parsed as shallow */
 213                                 parse_object(sha1);
 214                                 if (unregister_shallow(sha1))
 215                                         die("no shallow found: %s", line);
 216                                 continue;
 217                         }
 218                         die("expected shallow/unshallow, got %s", line);
 219                 }
 220         }
 221
 222         flushes = 0;
 223         retval = -1;
 224         while ((sha1 = get_rev())) {
 225                 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
 226                 if (verbose)
 227                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 228                 in_vain++;
 229                 if (!(31 & ++count)) {
 230                         int ack;
 231
 232                         packet_flush(fd[1]);
 233                         flushes++;
 234
 235                         /*
 236                          * We keep one window "ahead" of the other side, and
 237                          * will wait for an ACK only on the next one
 238                          */
 239                         if (count == 32)
 240                                 continue;
 241
 242                         do {
 243                                 ack = get_ack(fd[0], result_sha1);
 244                                 if (verbose && ack)
 245                                         fprintf(stderr, "got ack %d %s\n", ack,
 246                                                         sha1_to_hex(result_sha1));
 247                                 if (ack == 1) {
 248                                         flushes = 0;
 249                                         multi_ack = 0;
 250                                         retval = 0;
 251                                         goto done;
 252                                 } else if (ack == 2) {
 253                                         struct commit *commit =
 254                                                 lookup_commit(result_sha1);
 255                                         mark_common(commit, 0, 1);
 256                                         retval = 0;
 257                                         in_vain = 0;
 258                                         got_continue = 1;
 259                                 }
 260                         } while (ack);
 261                         flushes--;
 262                         if (got_continue && MAX_IN_VAIN < in_vain) {
 263                                 if (verbose)
 264                                         fprintf(stderr, "giving up\n");
 265                                 break; /* give up */
 266                         }
 267                 }
 268         }
 269 done:
 270         packet_write(fd[1], "done\n");
 271         if (verbose)
 272                 fprintf(stderr, "done\n");
 273         if (retval != 0) {
 274                 multi_ack = 0;
 275                 flushes++;
 276         }
 277         while (flushes || multi_ack) {
 278                 int ack = get_ack(fd[0], result_sha1);
 279                 if (ack) {
 280                         if (verbose)
 281                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 282                                         sha1_to_hex(result_sha1));
 283                         if (ack == 1)
 284                                 return 0;
 285                         multi_ack = 1;
 286                         continue;
 287                 }
 288                 flushes--;
 289         }
 290         return retval;
 291 }
 292
 293 static struct commit_list *complete;
 294
 295 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
 296 {
 297         struct object *o = parse_object(sha1);
 298
 299         while (o && o->type == OBJ_TAG) {
 300                 struct tag *t = (struct tag *) o;
 301                 if (!t->tagged)
 302                         break; /* broken repository */
 303                 o->flags |= COMPLETE;
 304                 o = parse_object(t->tagged->sha1);
 305         }
 306         if (o && o->type == OBJ_COMMIT) {
 307                 struct commit *commit = (struct commit *)o;
 308                 commit->object.flags |= COMPLETE;
 309                 insert_by_date(commit, &complete);
 310         }
 311         return 0;
 312 }
 313
 314 static void mark_recent_complete_commits(unsigned long cutoff)
 315 {
 316         while (complete && cutoff <= complete->item->date) {
 317                 if (verbose)
 318                         fprintf(stderr, "Marking %s as complete\n",
 319                                 sha1_to_hex(complete->item->object.sha1));
 320                 pop_most_recent_commit(&complete, COMPLETE);
 321         }
 322 }
 323
 324 static void filter_refs(struct ref **refs, int nr_match, char **match)
 325 {
 326         struct ref **return_refs;
 327         struct ref *newlist = NULL;
 328         struct ref **newtail = &newlist;
 329         struct ref *ref, *next;
 330         struct ref *fastarray[32];
 331
 332         if (nr_match && !fetch_all) {
 333                 if (ARRAY_SIZE(fastarray) < nr_match)
 334                         return_refs = xcalloc(nr_match, sizeof(struct ref *));
 335                 else {
 336                         return_refs = fastarray;
 337                         memset(return_refs, 0, sizeof(struct ref *) * nr_match);
 338                 }
 339         }
 340         else
 341                 return_refs = NULL;
 342
 343         for (ref = *refs; ref; ref = next) {
 344                 next = ref->next;
 345                 if (!memcmp(ref->name, "refs/", 5) &&
 346                     check_ref_format(ref->name + 5))
 347                         ; /* trash */
 348                 else if (fetch_all) {
 349                         *newtail = ref;
 350                         ref->next = NULL;
 351                         newtail = &ref->next;
 352                         continue;
 353                 }
 354                 else {
 355                         int order = path_match(ref->name, nr_match, match);
 356                         if (order) {
 357                                 return_refs[order-1] = ref;
 358                                 continue; /* we will link it later */
 359                         }
 360                 }
 361                 free(ref);
 362         }
 363
 364         if (!fetch_all) {
 365                 int i;
 366                 for (i = 0; i < nr_match; i++) {
 367                         ref = return_refs[i];
 368                         if (ref) {
 369                                 *newtail = ref;
 370                                 ref->next = NULL;
 371                                 newtail = &ref->next;
 372                         }
 373                 }
 374                 if (return_refs != fastarray)
 375                         free(return_refs);
 376         }
 377         *refs = newlist;
 378 }
 379
 380 static int everything_local(struct ref **refs, int nr_match, char **match)
 381 {
 382         struct ref *ref;
 383         int retval;
 384         unsigned long cutoff = 0;
 385
 386         track_object_refs = 0;
 387         save_commit_buffer = 0;
 388
 389         for (ref = *refs; ref; ref = ref->next) {
 390                 struct object *o;
 391
 392                 o = parse_object(ref->old_sha1);
 393                 if (!o)
 394                         continue;
 395
 396                 /* We already have it -- which may mean that we were
 397                  * in sync with the other side at some time after
 398                  * that (it is OK if we guess wrong here).
 399                  */
 400                 if (o->type == OBJ_COMMIT) {
 401                         struct commit *commit = (struct commit *)o;
 402                         if (!cutoff || cutoff < commit->date)
 403                                 cutoff = commit->date;
 404                 }
 405         }
 406
 407         if (!depth) {
 408                 for_each_ref(mark_complete, NULL);
 409                 if (cutoff)
 410                         mark_recent_complete_commits(cutoff);
 411         }
 412
 413         /*
 414          * Mark all complete remote refs as common refs.
 415          * Don't mark them common yet; the server has to be told so first.
 416          */
 417         for (ref = *refs; ref; ref = ref->next) {
 418                 struct object *o = deref_tag(lookup_object(ref->old_sha1),
 419                                              NULL, 0);
 420
 421                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 422                         continue;
 423
 424                 if (!(o->flags & SEEN)) {
 425                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 426
 427                         mark_common((struct commit *)o, 1, 1);
 428                 }
 429         }
 430
 431         filter_refs(refs, nr_match, match);
 432
 433         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 434                 const unsigned char *remote = ref->old_sha1;
 435                 unsigned char local[20];
 436                 struct object *o;
 437
 438                 o = lookup_object(remote);
 439                 if (!o || !(o->flags & COMPLETE)) {
 440                         retval = 0;
 441                         if (!verbose)
 442                                 continue;
 443                         fprintf(stderr,
 444                                 "want %s (%s)\n", sha1_to_hex(remote),
 445                                 ref->name);
 446                         continue;
 447                 }
 448
 449                 hashcpy(ref->new_sha1, local);
 450                 if (!verbose)
 451                         continue;
 452                 fprintf(stderr,
 453                         "already have %s (%s)\n", sha1_to_hex(remote),
 454                         ref->name);
 455         }
 456         return retval;
 457 }
 458
 459 static pid_t setup_sideband(int fd[2], int xd[2])
 460 {
 461         pid_t side_pid;
 462
 463         if (!use_sideband) {
 464                 fd[0] = xd[0];
 465                 fd[1] = xd[1];
 466                 return 0;
 467         }
 468         /* xd[] is talking with upload-pack; subprocess reads from
 469          * xd[0], spits out band#2 to stderr, and feeds us band#1
 470          * through our fd[0].
 471          */
 472         if (pipe(fd) < 0)
 473                 die("fetch-pack: unable to set up pipe");
 474         side_pid = fork();
 475         if (side_pid < 0)
 476                 die("fetch-pack: unable to fork off sideband demultiplexer");
 477         if (!side_pid) {
 478                 /* subprocess */
 479                 close(fd[0]);
 480                 if (xd[0] != xd[1])
 481                         close(xd[1]);
 482                 if (recv_sideband("fetch-pack", xd[0], fd[1], 2))
 483                         exit(1);
 484                 exit(0);
 485         }
 486         close(xd[0]);
 487         close(fd[1]);
 488         fd[1] = xd[1];
 489         return side_pid;
 490 }
 491
 492 static int get_pack(int xd[2], const char **argv)
 493 {
 494         int status;
 495         pid_t pid, side_pid;
 496         int fd[2];
 497
 498         side_pid = setup_sideband(fd, xd);
 499         pid = fork();
 500         if (pid < 0)
 501                 die("fetch-pack: unable to fork off %s", argv[0]);
 502         if (!pid) {
 503                 dup2(fd[0], 0);
 504                 close(fd[0]);
 505                 close(fd[1]);
 506                 execv_git_cmd(argv);
 507                 die("%s exec failed", argv[0]);
 508         }
 509         close(fd[0]);
 510         close(fd[1]);
 511         while (waitpid(pid, &status, 0) < 0) {
 512                 if (errno != EINTR)
 513                         die("waiting for %s: %s", argv[0], strerror(errno));
 514         }
 515         if (WIFEXITED(status)) {
 516                 int code = WEXITSTATUS(status);
 517                 if (code)
 518                         die("%s died with error code %d", argv[0], code);
 519                 return 0;
 520         }
 521         if (WIFSIGNALED(status)) {
 522                 int sig = WTERMSIG(status);
 523                 die("%s died of signal %d", argv[0], sig);
 524         }
 525         die("%s died of unnatural causes %d", argv[0], status);
 526 }
 527
 528 static int explode_rx_pack(int xd[2])
 529 {
 530         const char *argv[3] = { "unpack-objects", quiet ? "-q" : NULL, NULL };
 531         return get_pack(xd, argv);
 532 }
 533
 534 static int keep_rx_pack(int xd[2])
 535 {
 536         const char *argv[6];
 537         char keep_arg[256];
 538         int n = 0;
 539
 540         argv[n++] = "index-pack";
 541         argv[n++] = "--stdin";
 542         if (!quiet)
 543                 argv[n++] = "-v";
 544         if (use_thin_pack)
 545                 argv[n++] = "--fix-thin";
 546         if (keep_pack > 1) {
 547                 int s = sprintf(keep_arg, "--keep=fetch-pack %i on ", getpid());
 548                 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
 549                         strcpy(keep_arg + s, "localhost");
 550                 argv[n++] = keep_arg;
 551         }
 552         argv[n] = NULL;
 553         return get_pack(xd, argv);
 554 }
 555
 556 static int fetch_pack(int fd[2], int nr_match, char **match)
 557 {
 558         struct ref *ref;
 559         unsigned char sha1[20];
 560         int status;
 561
 562         get_remote_heads(fd[0], &ref, 0, NULL, 0);
 563         if (is_repository_shallow() && !server_supports("shallow"))
 564                 die("Server does not support shallow clients");
 565         if (server_supports("multi_ack")) {
 566                 if (verbose)
 567                         fprintf(stderr, "Server supports multi_ack\n");
 568                 multi_ack = 1;
 569         }
 570         if (server_supports("side-band-64k")) {
 571                 if (verbose)
 572                         fprintf(stderr, "Server supports side-band-64k\n");
 573                 use_sideband = 2;
 574         }
 575         else if (server_supports("side-band")) {
 576                 if (verbose)
 577                         fprintf(stderr, "Server supports side-band\n");
 578                 use_sideband = 1;
 579         }
 580         if (!ref) {
 581                 packet_flush(fd[1]);
 582                 die("no matching remote head");
 583         }
 584         if (everything_local(&ref, nr_match, match)) {
 585                 packet_flush(fd[1]);
 586                 goto all_done;
 587         }
 588         if (find_common(fd, sha1, ref) < 0)
 589                 if (keep_pack != 1)
 590                         /* When cloning, it is not unusual to have
 591                          * no common commit.
 592                          */
 593                         fprintf(stderr, "warning: no common commits\n");
 594
 595         status = (keep_pack) ? keep_rx_pack(fd) : explode_rx_pack(fd);
 596         if (status)
 597                 die("git-fetch-pack: fetch failed.");
 598
 599  all_done:
 600         while (ref) {
 601                 printf("%s %s\n",
 602                        sha1_to_hex(ref->old_sha1), ref->name);
 603                 ref = ref->next;
 604         }
 605         return 0;
 606 }
 607
 608 int main(int argc, char **argv)
 609 {
 610         int i, ret, nr_heads;
 611         char *dest = NULL, **heads;
 612         int fd[2];
 613         pid_t pid;
 614         struct stat st;
 615         struct lock_file lock;
 616
 617         setup_git_directory();
 618
 619         nr_heads = 0;
 620         heads = NULL;
 621         for (i = 1; i < argc; i++) {
 622                 char *arg = argv[i];
 623
 624                 if (*arg == '-') {
 625                         if (!strncmp("--exec=", arg, 7)) {
 626                                 exec = arg + 7;
 627                                 continue;
 628                         }
 629                         if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
 630                                 quiet = 1;
 631                                 continue;
 632                         }
 633                         if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
 634                                 keep_pack++;
 635                                 continue;
 636                         }
 637                         if (!strcmp("--thin", arg)) {
 638                                 use_thin_pack = 1;
 639                                 continue;
 640                         }
 641                         if (!strcmp("--all", arg)) {
 642                                 fetch_all = 1;
 643                                 continue;
 644                         }
 645                         if (!strcmp("-v", arg)) {
 646                                 verbose = 1;
 647                                 continue;
 648                         }
 649                         if (!strncmp("--depth=", arg, 8)) {
 650                                 depth = strtol(arg + 8, NULL, 0);
 651                                 if (stat(git_path("shallow"), &st))
 652                                         st.st_mtime = 0;
 653                                 continue;
 654                         }
 655                         usage(fetch_pack_usage);
 656                 }
 657                 dest = arg;
 658                 heads = argv + i + 1;
 659                 nr_heads = argc - i - 1;
 660                 break;
 661         }
 662         if (!dest)
 663                 usage(fetch_pack_usage);
 664         pid = git_connect(fd, dest, exec);
 665         if (pid < 0)
 666                 return 1;
 667         ret = fetch_pack(fd, nr_heads, heads);
 668         close(fd[0]);
 669         close(fd[1]);
 670         ret |= finish_connect(pid);
 671
 672         if (!ret && nr_heads) {
 673                 /* If the heads to pull were given, we should have
 674                  * consumed all of them by matching the remote.
 675                  * Otherwise, 'git-fetch remote no-such-ref' would
 676                  * silently succeed without issuing an error.
 677                  */
 678                 for (i = 0; i < nr_heads; i++)
 679                         if (heads[i] && heads[i][0]) {
 680                                 error("no such remote ref %s", heads[i]);
 681                                 ret = 1;
 682                         }
 683         }
 684
 685         if (!ret && depth > 0) {
 686                 struct cache_time mtime;
 687                 char *shallow = git_path("shallow");
 688                 int fd;
 689
 690                 mtime.sec = st.st_mtime;
 691 #ifdef USE_NSEC
 692                 mtime.usec = st.st_mtim.usec;
 693 #endif
 694                 if (stat(shallow, &st)) {
 695                         if (mtime.sec)
 696                                 die("shallow file was removed during fetch");
 697                 } else if (st.st_mtime != mtime.sec
 698 #ifdef USE_NSEC
 699                                 || st.st_mtim.usec != mtime.usec
 700 #endif
 701                           )
 702                         die("shallow file was changed during fetch");
 703
 704                 fd = hold_lock_file_for_update(&lock, shallow, 1);
 705                 if (!write_shallow_commits(fd, 0)) {
 706                         unlink(lock.filename);
 707                         rollback_lock_file(&lock);
 708                 } else {
 709                         close(fd);
 710                         commit_lock_file(&lock);
 711                 }
 712         }
 713
 714         return !!ret;
 715 }