git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband;
  47 /* Allow specifying sha1 if it is a ref tip. */
  48 #define ALLOW_TIP_SHA1  01
  49 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  50 #define ALLOW_REACHABLE_SHA1    02
  51 static unsigned int allow_unadvertised_object_request;
  52
  53 __attribute__((format (printf, 2, 3)))
  54 static inline void print_verbose(const struct fetch_pack_args *args,
  55                                  const char *fmt, ...)
  56 {
  57         va_list params;
  58
  59         if (!args->verbose)
  60                 return;
  61
  62         va_start(params, fmt);
  63         vfprintf(stderr, fmt, params);
  64         va_end(params);
  65         fputc('\n', stderr);
  66 }
  67
  68 static void rev_list_push(struct commit *commit, int mark)
  69 {
  70         if (!(commit->object.flags & mark)) {
  71                 commit->object.flags |= mark;
  72
  73                 if (parse_commit(commit))
  74                         return;
  75
  76                 prio_queue_put(&rev_list, commit);
  77
  78                 if (!(commit->object.flags & COMMON))
  79                         non_common_revs++;
  80         }
  81 }
  82
  83 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  84 {
  85         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  86
  87         if (o && o->type == OBJ_COMMIT)
  88                 rev_list_push((struct commit *)o, SEEN);
  89
  90         return 0;
  91 }
  92
  93 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  94                                    int flag, void *cb_data)
  95 {
  96         return rev_list_insert_ref(refname, oid->hash);
  97 }
  98
  99 static int clear_marks(const char *refname, const struct object_id *oid,
 100                        int flag, void *cb_data)
 101 {
 102         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
 103
 104         if (o && o->type == OBJ_COMMIT)
 105                 clear_commit_marks((struct commit *)o,
 106                                    COMMON | COMMON_REF | SEEN | POPPED);
 107         return 0;
 108 }
 109
 110 /*
 111    This function marks a rev and its ancestors as common.
 112    In some cases, it is desirable to mark only the ancestors (for example
 113    when only the server does not yet know that they are common).
 114 */
 115
 116 static void mark_common(struct commit *commit,
 117                 int ancestors_only, int dont_parse)
 118 {
 119         if (commit != NULL && !(commit->object.flags & COMMON)) {
 120                 struct object *o = (struct object *)commit;
 121
 122                 if (!ancestors_only)
 123                         o->flags |= COMMON;
 124
 125                 if (!(o->flags & SEEN))
 126                         rev_list_push(commit, SEEN);
 127                 else {
 128                         struct commit_list *parents;
 129
 130                         if (!ancestors_only && !(o->flags & POPPED))
 131                                 non_common_revs--;
 132                         if (!o->parsed && !dont_parse)
 133                                 if (parse_commit(commit))
 134                                         return;
 135
 136                         for (parents = commit->parents;
 137                                         parents;
 138                                         parents = parents->next)
 139                                 mark_common(parents->item, 0, dont_parse);
 140                 }
 141         }
 142 }
 143
 144 /*
 145   Get the next rev to send, ignoring the common.
 146 */
 147
 148 static const unsigned char *get_rev(void)
 149 {
 150         struct commit *commit = NULL;
 151
 152         while (commit == NULL) {
 153                 unsigned int mark;
 154                 struct commit_list *parents;
 155
 156                 if (rev_list.nr == 0 || non_common_revs == 0)
 157                         return NULL;
 158
 159                 commit = prio_queue_get(&rev_list);
 160                 parse_commit(commit);
 161                 parents = commit->parents;
 162
 163                 commit->object.flags |= POPPED;
 164                 if (!(commit->object.flags & COMMON))
 165                         non_common_revs--;
 166
 167                 if (commit->object.flags & COMMON) {
 168                         /* do not send "have", and ignore ancestors */
 169                         commit = NULL;
 170                         mark = COMMON | SEEN;
 171                 } else if (commit->object.flags & COMMON_REF)
 172                         /* send "have", and ignore ancestors */
 173                         mark = COMMON | SEEN;
 174                 else
 175                         /* send "have", also for its ancestors */
 176                         mark = SEEN;
 177
 178                 while (parents) {
 179                         if (!(parents->item->object.flags & SEEN))
 180                                 rev_list_push(parents->item, mark);
 181                         if (mark & COMMON)
 182                                 mark_common(parents->item, 1, 0);
 183                         parents = parents->next;
 184                 }
 185         }
 186
 187         return commit->object.oid.hash;
 188 }
 189
 190 enum ack_type {
 191         NAK = 0,
 192         ACK,
 193         ACK_continue,
 194         ACK_common,
 195         ACK_ready
 196 };
 197
 198 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 199 {
 200         if (args->stateless_rpc && args->deepen) {
 201                 /* If we sent a depth we will get back "duplicate"
 202                  * shallow and unshallow commands every time there
 203                  * is a block of have lines exchanged.
 204                  */
 205                 char *line;
 206                 while ((line = packet_read_line(fd, NULL))) {
 207                         if (starts_with(line, "shallow "))
 208                                 continue;
 209                         if (starts_with(line, "unshallow "))
 210                                 continue;
 211                         die(_("git fetch-pack: expected shallow list"));
 212                 }
 213         }
 214 }
 215
 216 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 217 {
 218         int len;
 219         char *line = packet_read_line(fd, &len);
 220         const char *arg;
 221
 222         if (!len)
 223                 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
 224         if (!strcmp(line, "NAK"))
 225                 return NAK;
 226         if (skip_prefix(line, "ACK ", &arg)) {
 227                 if (!get_sha1_hex(arg, result_sha1)) {
 228                         arg += 40;
 229                         len -= arg - line;
 230                         if (len < 1)
 231                                 return ACK;
 232                         if (strstr(arg, "continue"))
 233                                 return ACK_continue;
 234                         if (strstr(arg, "common"))
 235                                 return ACK_common;
 236                         if (strstr(arg, "ready"))
 237                                 return ACK_ready;
 238                         return ACK;
 239                 }
 240         }
 241         die(_("git fetch_pack: expected ACK/NAK, got '%s'"), line);
 242 }
 243
 244 static void send_request(struct fetch_pack_args *args,
 245                          int fd, struct strbuf *buf)
 246 {
 247         if (args->stateless_rpc) {
 248                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 249                 packet_flush(fd);
 250         } else
 251                 write_or_die(fd, buf->buf, buf->len);
 252 }
 253
 254 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 255 {
 256         rev_list_insert_ref(NULL, ref->old_oid.hash);
 257 }
 258
 259 #define INITIAL_FLUSH 16
 260 #define PIPESAFE_FLUSH 32
 261 #define LARGE_FLUSH 1024
 262
 263 static int next_flush(struct fetch_pack_args *args, int count)
 264 {
 265         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 266
 267         if (count < flush_limit)
 268                 count <<= 1;
 269         else
 270                 count += flush_limit;
 271         return count;
 272 }
 273
 274 static int find_common(struct fetch_pack_args *args,
 275                        int fd[2], unsigned char *result_sha1,
 276                        struct ref *refs)
 277 {
 278         int fetching;
 279         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 280         const unsigned char *sha1;
 281         unsigned in_vain = 0;
 282         int got_continue = 0;
 283         int got_ready = 0;
 284         struct strbuf req_buf = STRBUF_INIT;
 285         size_t state_len = 0;
 286
 287         if (args->stateless_rpc && multi_ack == 1)
 288                 die(_("--stateless-rpc requires multi_ack_detailed"));
 289         if (marked)
 290                 for_each_ref(clear_marks, NULL);
 291         marked = 1;
 292
 293         for_each_ref(rev_list_insert_ref_oid, NULL);
 294         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 295
 296         fetching = 0;
 297         for ( ; refs ; refs = refs->next) {
 298                 unsigned char *remote = refs->old_oid.hash;
 299                 const char *remote_hex;
 300                 struct object *o;
 301
 302                 /*
 303                  * If that object is complete (i.e. it is an ancestor of a
 304                  * local ref), we tell them we have it but do not have to
 305                  * tell them about its ancestors, which they already know
 306                  * about.
 307                  *
 308                  * We use lookup_object here because we are only
 309                  * interested in the case we *know* the object is
 310                  * reachable and we have already scanned it.
 311                  */
 312                 if (((o = lookup_object(remote)) != NULL) &&
 313                                 (o->flags & COMPLETE)) {
 314                         continue;
 315                 }
 316
 317                 remote_hex = sha1_to_hex(remote);
 318                 if (!fetching) {
 319                         struct strbuf c = STRBUF_INIT;
 320                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 321                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 322                         if (no_done)            strbuf_addstr(&c, " no-done");
 323                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 324                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 325                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 326                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 327                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 328                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 329                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 330                                                             git_user_agent_sanitized());
 331                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 332                         strbuf_release(&c);
 333                 } else
 334                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 335                 fetching++;
 336         }
 337
 338         if (!fetching) {
 339                 strbuf_release(&req_buf);
 340                 packet_flush(fd[1]);
 341                 return 1;
 342         }
 343
 344         if (is_repository_shallow())
 345                 write_shallow_commits(&req_buf, 1, NULL);
 346         if (args->depth > 0)
 347                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 348         packet_buf_flush(&req_buf);
 349         state_len = req_buf.len;
 350
 351         if (args->deepen) {
 352                 char *line;
 353                 const char *arg;
 354                 unsigned char sha1[20];
 355
 356                 send_request(args, fd[1], &req_buf);
 357                 while ((line = packet_read_line(fd[0], NULL))) {
 358                         if (skip_prefix(line, "shallow ", &arg)) {
 359                                 if (get_sha1_hex(arg, sha1))
 360                                         die(_("invalid shallow line: %s"), line);
 361                                 register_shallow(sha1);
 362                                 continue;
 363                         }
 364                         if (skip_prefix(line, "unshallow ", &arg)) {
 365                                 if (get_sha1_hex(arg, sha1))
 366                                         die(_("invalid unshallow line: %s"), line);
 367                                 if (!lookup_object(sha1))
 368                                         die(_("object not found: %s"), line);
 369                                 /* make sure that it is parsed as shallow */
 370                                 if (!parse_object(sha1))
 371                                         die(_("error in object: %s"), line);
 372                                 if (unregister_shallow(sha1))
 373                                         die(_("no shallow found: %s"), line);
 374                                 continue;
 375                         }
 376                         die(_("expected shallow/unshallow, got %s"), line);
 377                 }
 378         } else if (!args->stateless_rpc)
 379                 send_request(args, fd[1], &req_buf);
 380
 381         if (!args->stateless_rpc) {
 382                 /* If we aren't using the stateless-rpc interface
 383                  * we don't need to retain the headers.
 384                  */
 385                 strbuf_setlen(&req_buf, 0);
 386                 state_len = 0;
 387         }
 388
 389         flushes = 0;
 390         retval = -1;
 391         while ((sha1 = get_rev())) {
 392                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 393                 print_verbose(args, "have %s", sha1_to_hex(sha1));
 394                 in_vain++;
 395                 if (flush_at <= ++count) {
 396                         int ack;
 397
 398                         packet_buf_flush(&req_buf);
 399                         send_request(args, fd[1], &req_buf);
 400                         strbuf_setlen(&req_buf, state_len);
 401                         flushes++;
 402                         flush_at = next_flush(args, count);
 403
 404                         /*
 405                          * We keep one window "ahead" of the other side, and
 406                          * will wait for an ACK only on the next one
 407                          */
 408                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 409                                 continue;
 410
 411                         consume_shallow_list(args, fd[0]);
 412                         do {
 413                                 ack = get_ack(fd[0], result_sha1);
 414                                 if (ack)
 415                                         print_verbose(args, _("got %s %d %s"), "ack",
 416                                                       ack, sha1_to_hex(result_sha1));
 417                                 switch (ack) {
 418                                 case ACK:
 419                                         flushes = 0;
 420                                         multi_ack = 0;
 421                                         retval = 0;
 422                                         goto done;
 423                                 case ACK_common:
 424                                 case ACK_ready:
 425                                 case ACK_continue: {
 426                                         struct commit *commit =
 427                                                 lookup_commit(result_sha1);
 428                                         if (!commit)
 429                                                 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
 430                                         if (args->stateless_rpc
 431                                          && ack == ACK_common
 432                                          && !(commit->object.flags & COMMON)) {
 433                                                 /* We need to replay the have for this object
 434                                                  * on the next RPC request so the peer knows
 435                                                  * it is in common with us.
 436                                                  */
 437                                                 const char *hex = sha1_to_hex(result_sha1);
 438                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 439                                                 state_len = req_buf.len;
 440                                         }
 441                                         mark_common(commit, 0, 1);
 442                                         retval = 0;
 443                                         in_vain = 0;
 444                                         got_continue = 1;
 445                                         if (ack == ACK_ready) {
 446                                                 clear_prio_queue(&rev_list);
 447                                                 got_ready = 1;
 448                                         }
 449                                         break;
 450                                         }
 451                                 }
 452                         } while (ack);
 453                         flushes--;
 454                         if (got_continue && MAX_IN_VAIN < in_vain) {
 455                                 print_verbose(args, _("giving up"));
 456                                 break; /* give up */
 457                         }
 458                 }
 459         }
 460 done:
 461         if (!got_ready || !no_done) {
 462                 packet_buf_write(&req_buf, "done\n");
 463                 send_request(args, fd[1], &req_buf);
 464         }
 465         print_verbose(args, _("done"));
 466         if (retval != 0) {
 467                 multi_ack = 0;
 468                 flushes++;
 469         }
 470         strbuf_release(&req_buf);
 471
 472         if (!got_ready || !no_done)
 473                 consume_shallow_list(args, fd[0]);
 474         while (flushes || multi_ack) {
 475                 int ack = get_ack(fd[0], result_sha1);
 476                 if (ack) {
 477                         print_verbose(args, _("got %s (%d) %s"), "ack",
 478                                       ack, sha1_to_hex(result_sha1));
 479                         if (ack == ACK)
 480                                 return 0;
 481                         multi_ack = 1;
 482                         continue;
 483                 }
 484                 flushes--;
 485         }
 486         /* it is no error to fetch into a completely empty repo */
 487         return count ? retval : 0;
 488 }
 489
 490 static struct commit_list *complete;
 491
 492 static int mark_complete(const unsigned char *sha1)
 493 {
 494         struct object *o = parse_object(sha1);
 495
 496         while (o && o->type == OBJ_TAG) {
 497                 struct tag *t = (struct tag *) o;
 498                 if (!t->tagged)
 499                         break; /* broken repository */
 500                 o->flags |= COMPLETE;
 501                 o = parse_object(t->tagged->oid.hash);
 502         }
 503         if (o && o->type == OBJ_COMMIT) {
 504                 struct commit *commit = (struct commit *)o;
 505                 if (!(commit->object.flags & COMPLETE)) {
 506                         commit->object.flags |= COMPLETE;
 507                         commit_list_insert(commit, &complete);
 508                 }
 509         }
 510         return 0;
 511 }
 512
 513 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 514                              int flag, void *cb_data)
 515 {
 516         return mark_complete(oid->hash);
 517 }
 518
 519 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 520                                          unsigned long cutoff)
 521 {
 522         while (complete && cutoff <= complete->item->date) {
 523                 print_verbose(args, _("Marking %s as complete"),
 524                               oid_to_hex(&complete->item->object.oid));
 525                 pop_most_recent_commit(&complete, COMPLETE);
 526         }
 527 }
 528
 529 static void filter_refs(struct fetch_pack_args *args,
 530                         struct ref **refs,
 531                         struct ref **sought, int nr_sought)
 532 {
 533         struct ref *newlist = NULL;
 534         struct ref **newtail = &newlist;
 535         struct ref *ref, *next;
 536         int i;
 537
 538         i = 0;
 539         for (ref = *refs; ref; ref = next) {
 540                 int keep = 0;
 541                 next = ref->next;
 542
 543                 if (starts_with(ref->name, "refs/") &&
 544                     check_refname_format(ref->name, 0))
 545                         ; /* trash */
 546                 else {
 547                         while (i < nr_sought) {
 548                                 int cmp = strcmp(ref->name, sought[i]->name);
 549                                 if (cmp < 0)
 550                                         break; /* definitely do not have it */
 551                                 else if (cmp == 0) {
 552                                         keep = 1; /* definitely have it */
 553                                         sought[i]->matched = 1;
 554                                 }
 555                                 i++;
 556                         }
 557                 }
 558
 559                 if (!keep && args->fetch_all &&
 560                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 561                         keep = 1;
 562
 563                 if (keep) {
 564                         *newtail = ref;
 565                         ref->next = NULL;
 566                         newtail = &ref->next;
 567                 } else {
 568                         free(ref);
 569                 }
 570         }
 571
 572         /* Append unmatched requests to the list */
 573         if ((allow_unadvertised_object_request &
 574             (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 575                 for (i = 0; i < nr_sought; i++) {
 576                         unsigned char sha1[20];
 577
 578                         ref = sought[i];
 579                         if (ref->matched)
 580                                 continue;
 581                         if (get_sha1_hex(ref->name, sha1) ||
 582                             ref->name[40] != '\0' ||
 583                             hashcmp(sha1, ref->old_oid.hash))
 584                                 continue;
 585
 586                         ref->matched = 1;
 587                         *newtail = copy_ref(ref);
 588                         newtail = &(*newtail)->next;
 589                 }
 590         }
 591         *refs = newlist;
 592 }
 593
 594 static void mark_alternate_complete(const struct ref *ref, void *unused)
 595 {
 596         mark_complete(ref->old_oid.hash);
 597 }
 598
 599 static int everything_local(struct fetch_pack_args *args,
 600                             struct ref **refs,
 601                             struct ref **sought, int nr_sought)
 602 {
 603         struct ref *ref;
 604         int retval;
 605         unsigned long cutoff = 0;
 606
 607         save_commit_buffer = 0;
 608
 609         for (ref = *refs; ref; ref = ref->next) {
 610                 struct object *o;
 611
 612                 if (!has_object_file(&ref->old_oid))
 613                         continue;
 614
 615                 o = parse_object(ref->old_oid.hash);
 616                 if (!o)
 617                         continue;
 618
 619                 /* We already have it -- which may mean that we were
 620                  * in sync with the other side at some time after
 621                  * that (it is OK if we guess wrong here).
 622                  */
 623                 if (o->type == OBJ_COMMIT) {
 624                         struct commit *commit = (struct commit *)o;
 625                         if (!cutoff || cutoff < commit->date)
 626                                 cutoff = commit->date;
 627                 }
 628         }
 629
 630         if (!args->deepen) {
 631                 for_each_ref(mark_complete_oid, NULL);
 632                 for_each_alternate_ref(mark_alternate_complete, NULL);
 633                 commit_list_sort_by_date(&complete);
 634                 if (cutoff)
 635                         mark_recent_complete_commits(args, cutoff);
 636         }
 637
 638         /*
 639          * Mark all complete remote refs as common refs.
 640          * Don't mark them common yet; the server has to be told so first.
 641          */
 642         for (ref = *refs; ref; ref = ref->next) {
 643                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 644                                              NULL, 0);
 645
 646                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 647                         continue;
 648
 649                 if (!(o->flags & SEEN)) {
 650                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 651
 652                         mark_common((struct commit *)o, 1, 1);
 653                 }
 654         }
 655
 656         filter_refs(args, refs, sought, nr_sought);
 657
 658         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 659                 const unsigned char *remote = ref->old_oid.hash;
 660                 struct object *o;
 661
 662                 o = lookup_object(remote);
 663                 if (!o || !(o->flags & COMPLETE)) {
 664                         retval = 0;
 665                         print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
 666                                       ref->name);
 667                         continue;
 668                 }
 669                 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
 670                               ref->name);
 671         }
 672         return retval;
 673 }
 674
 675 static int sideband_demux(int in, int out, void *data)
 676 {
 677         int *xd = data;
 678
 679         int ret = recv_sideband("fetch-pack", xd[0], out);
 680         close(out);
 681         return ret;
 682 }
 683
 684 static int get_pack(struct fetch_pack_args *args,
 685                     int xd[2], char **pack_lockfile)
 686 {
 687         struct async demux;
 688         int do_keep = args->keep_pack;
 689         const char *cmd_name;
 690         struct pack_header header;
 691         int pass_header = 0;
 692         struct child_process cmd = CHILD_PROCESS_INIT;
 693         int ret;
 694
 695         memset(&demux, 0, sizeof(demux));
 696         if (use_sideband) {
 697                 /* xd[] is talking with upload-pack; subprocess reads from
 698                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 699                  * through demux->out.
 700                  */
 701                 demux.proc = sideband_demux;
 702                 demux.data = xd;
 703                 demux.out = -1;
 704                 if (start_async(&demux))
 705                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 706         }
 707         else
 708                 demux.out = xd[0];
 709
 710         if (!args->keep_pack && unpack_limit) {
 711
 712                 if (read_pack_header(demux.out, &header))
 713                         die(_("protocol error: bad pack header"));
 714                 pass_header = 1;
 715                 if (ntohl(header.hdr_entries) < unpack_limit)
 716                         do_keep = 0;
 717                 else
 718                         do_keep = 1;
 719         }
 720
 721         if (alternate_shallow_file) {
 722                 argv_array_push(&cmd.args, "--shallow-file");
 723                 argv_array_push(&cmd.args, alternate_shallow_file);
 724         }
 725
 726         if (do_keep) {
 727                 if (pack_lockfile)
 728                         cmd.out = -1;
 729                 cmd_name = "index-pack";
 730                 argv_array_push(&cmd.args, cmd_name);
 731                 argv_array_push(&cmd.args, "--stdin");
 732                 if (!args->quiet && !args->no_progress)
 733                         argv_array_push(&cmd.args, "-v");
 734                 if (args->use_thin_pack)
 735                         argv_array_push(&cmd.args, "--fix-thin");
 736                 if (args->lock_pack || unpack_limit) {
 737                         char hostname[256];
 738                         if (gethostname(hostname, sizeof(hostname)))
 739                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 740                         argv_array_pushf(&cmd.args,
 741                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 742                                         (uintmax_t)getpid(), hostname);
 743                 }
 744                 if (args->check_self_contained_and_connected)
 745                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 746         }
 747         else {
 748                 cmd_name = "unpack-objects";
 749                 argv_array_push(&cmd.args, cmd_name);
 750                 if (args->quiet || args->no_progress)
 751                         argv_array_push(&cmd.args, "-q");
 752                 args->check_self_contained_and_connected = 0;
 753         }
 754
 755         if (pass_header)
 756                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 757                                  ntohl(header.hdr_version),
 758                                  ntohl(header.hdr_entries));
 759         if (fetch_fsck_objects >= 0
 760             ? fetch_fsck_objects
 761             : transfer_fsck_objects >= 0
 762             ? transfer_fsck_objects
 763             : 0)
 764                 argv_array_push(&cmd.args, "--strict");
 765
 766         cmd.in = demux.out;
 767         cmd.git_cmd = 1;
 768         if (start_command(&cmd))
 769                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 770         if (do_keep && pack_lockfile) {
 771                 *pack_lockfile = index_pack_lockfile(cmd.out);
 772                 close(cmd.out);
 773         }
 774
 775         if (!use_sideband)
 776                 /* Closed by start_command() */
 777                 xd[0] = -1;
 778
 779         ret = finish_command(&cmd);
 780         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 781                 args->self_contained_and_connected =
 782                         args->check_self_contained_and_connected &&
 783                         ret == 0;
 784         else
 785                 die(_("%s failed"), cmd_name);
 786         if (use_sideband && finish_async(&demux))
 787                 die(_("error in sideband demultiplexer"));
 788         return 0;
 789 }
 790
 791 static int cmp_ref_by_name(const void *a_, const void *b_)
 792 {
 793         const struct ref *a = *((const struct ref **)a_);
 794         const struct ref *b = *((const struct ref **)b_);
 795         return strcmp(a->name, b->name);
 796 }
 797
 798 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 799                                  int fd[2],
 800                                  const struct ref *orig_ref,
 801                                  struct ref **sought, int nr_sought,
 802                                  struct shallow_info *si,
 803                                  char **pack_lockfile)
 804 {
 805         struct ref *ref = copy_ref_list(orig_ref);
 806         unsigned char sha1[20];
 807         const char *agent_feature;
 808         int agent_len;
 809
 810         sort_ref_list(&ref, ref_compare_name);
 811         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 812
 813         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 814                 die(_("Server does not support shallow clients"));
 815         if (args->depth > 0)
 816                 args->deepen = 1;
 817         if (server_supports("multi_ack_detailed")) {
 818                 print_verbose(args, _("Server supports multi_ack_detailed"));
 819                 multi_ack = 2;
 820                 if (server_supports("no-done")) {
 821                         print_verbose(args, _("Server supports no-done"));
 822                         if (args->stateless_rpc)
 823                                 no_done = 1;
 824                 }
 825         }
 826         else if (server_supports("multi_ack")) {
 827                 print_verbose(args, _("Server supports multi_ack"));
 828                 multi_ack = 1;
 829         }
 830         if (server_supports("side-band-64k")) {
 831                 print_verbose(args, _("Server supports side-band-64k"));
 832                 use_sideband = 2;
 833         }
 834         else if (server_supports("side-band")) {
 835                 print_verbose(args, _("Server supports side-band"));
 836                 use_sideband = 1;
 837         }
 838         if (server_supports("allow-tip-sha1-in-want")) {
 839                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 840                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 841         }
 842         if (server_supports("allow-reachable-sha1-in-want")) {
 843                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 844                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 845         }
 846         if (!server_supports("thin-pack"))
 847                 args->use_thin_pack = 0;
 848         if (!server_supports("no-progress"))
 849                 args->no_progress = 0;
 850         if (!server_supports("include-tag"))
 851                 args->include_tag = 0;
 852         if (server_supports("ofs-delta"))
 853                 print_verbose(args, _("Server supports ofs-delta"));
 854         else
 855                 prefer_ofs_delta = 0;
 856
 857         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 858                 agent_supported = 1;
 859                 if (agent_len)
 860                         print_verbose(args, _("Server version is %.*s"),
 861                                       agent_len, agent_feature);
 862         }
 863
 864         if (everything_local(args, &ref, sought, nr_sought)) {
 865                 packet_flush(fd[1]);
 866                 goto all_done;
 867         }
 868         if (find_common(args, fd, sha1, ref) < 0)
 869                 if (!args->keep_pack)
 870                         /* When cloning, it is not unusual to have
 871                          * no common commit.
 872                          */
 873                         warning(_("no common commits"));
 874
 875         if (args->stateless_rpc)
 876                 packet_flush(fd[1]);
 877         if (args->deepen)
 878                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 879                                         NULL);
 880         else if (si->nr_ours || si->nr_theirs)
 881                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 882         else
 883                 alternate_shallow_file = NULL;
 884         if (get_pack(args, fd, pack_lockfile))
 885                 die(_("git fetch-pack: fetch failed."));
 886
 887  all_done:
 888         return ref;
 889 }
 890
 891 static void fetch_pack_config(void)
 892 {
 893         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 894         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 895         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 896         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 897         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 898
 899         git_config(git_default_config, NULL);
 900 }
 901
 902 static void fetch_pack_setup(void)
 903 {
 904         static int did_setup;
 905         if (did_setup)
 906                 return;
 907         fetch_pack_config();
 908         if (0 <= transfer_unpack_limit)
 909                 unpack_limit = transfer_unpack_limit;
 910         else if (0 <= fetch_unpack_limit)
 911                 unpack_limit = fetch_unpack_limit;
 912         did_setup = 1;
 913 }
 914
 915 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 916 {
 917         struct string_list names = STRING_LIST_INIT_NODUP;
 918         int src, dst;
 919
 920         for (src = dst = 0; src < nr; src++) {
 921                 struct string_list_item *item;
 922                 item = string_list_insert(&names, ref[src]->name);
 923                 if (item->util)
 924                         continue; /* already have it */
 925                 item->util = ref[src];
 926                 if (src != dst)
 927                         ref[dst] = ref[src];
 928                 dst++;
 929         }
 930         for (src = dst; src < nr; src++)
 931                 ref[src] = NULL;
 932         string_list_clear(&names, 0);
 933         return dst;
 934 }
 935
 936 static void update_shallow(struct fetch_pack_args *args,
 937                            struct ref **sought, int nr_sought,
 938                            struct shallow_info *si)
 939 {
 940         struct sha1_array ref = SHA1_ARRAY_INIT;
 941         int *status;
 942         int i;
 943
 944         if (args->deepen && alternate_shallow_file) {
 945                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 946                         unlink_or_warn(git_path_shallow());
 947                         rollback_lock_file(&shallow_lock);
 948                 } else
 949                         commit_lock_file(&shallow_lock);
 950                 return;
 951         }
 952
 953         if (!si->shallow || !si->shallow->nr)
 954                 return;
 955
 956         if (args->cloning) {
 957                 /*
 958                  * remote is shallow, but this is a clone, there are
 959                  * no objects in repo to worry about. Accept any
 960                  * shallow points that exist in the pack (iow in repo
 961                  * after get_pack() and reprepare_packed_git())
 962                  */
 963                 struct sha1_array extra = SHA1_ARRAY_INIT;
 964                 unsigned char (*sha1)[20] = si->shallow->sha1;
 965                 for (i = 0; i < si->shallow->nr; i++)
 966                         if (has_sha1_file(sha1[i]))
 967                                 sha1_array_append(&extra, sha1[i]);
 968                 if (extra.nr) {
 969                         setup_alternate_shallow(&shallow_lock,
 970                                                 &alternate_shallow_file,
 971                                                 &extra);
 972                         commit_lock_file(&shallow_lock);
 973                 }
 974                 sha1_array_clear(&extra);
 975                 return;
 976         }
 977
 978         if (!si->nr_ours && !si->nr_theirs)
 979                 return;
 980
 981         remove_nonexistent_theirs_shallow(si);
 982         if (!si->nr_ours && !si->nr_theirs)
 983                 return;
 984         for (i = 0; i < nr_sought; i++)
 985                 sha1_array_append(&ref, sought[i]->old_oid.hash);
 986         si->ref = &ref;
 987
 988         if (args->update_shallow) {
 989                 /*
 990                  * remote is also shallow, .git/shallow may be updated
 991                  * so all refs can be accepted. Make sure we only add
 992                  * shallow roots that are actually reachable from new
 993                  * refs.
 994                  */
 995                 struct sha1_array extra = SHA1_ARRAY_INIT;
 996                 unsigned char (*sha1)[20] = si->shallow->sha1;
 997                 assign_shallow_commits_to_refs(si, NULL, NULL);
 998                 if (!si->nr_ours && !si->nr_theirs) {
 999                         sha1_array_clear(&ref);
1000                         return;
1001                 }
1002                 for (i = 0; i < si->nr_ours; i++)
1003                         sha1_array_append(&extra, sha1[si->ours[i]]);
1004                 for (i = 0; i < si->nr_theirs; i++)
1005                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1006                 setup_alternate_shallow(&shallow_lock,
1007                                         &alternate_shallow_file,
1008                                         &extra);
1009                 commit_lock_file(&shallow_lock);
1010                 sha1_array_clear(&extra);
1011                 sha1_array_clear(&ref);
1012                 return;
1013         }
1014
1015         /*
1016          * remote is also shallow, check what ref is safe to update
1017          * without updating .git/shallow
1018          */
1019         status = xcalloc(nr_sought, sizeof(*status));
1020         assign_shallow_commits_to_refs(si, NULL, status);
1021         if (si->nr_ours || si->nr_theirs) {
1022                 for (i = 0; i < nr_sought; i++)
1023                         if (status[i])
1024                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1025         }
1026         free(status);
1027         sha1_array_clear(&ref);
1028 }
1029
1030 struct ref *fetch_pack(struct fetch_pack_args *args,
1031                        int fd[], struct child_process *conn,
1032                        const struct ref *ref,
1033                        const char *dest,
1034                        struct ref **sought, int nr_sought,
1035                        struct sha1_array *shallow,
1036                        char **pack_lockfile)
1037 {
1038         struct ref *ref_cpy;
1039         struct shallow_info si;
1040
1041         fetch_pack_setup();
1042         if (nr_sought)
1043                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1044
1045         if (!ref) {
1046                 packet_flush(fd[1]);
1047                 die(_("no matching remote head"));
1048         }
1049         prepare_shallow_info(&si, shallow);
1050         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1051                                 &si, pack_lockfile);
1052         reprepare_packed_git();
1053         update_shallow(args, sought, nr_sought, &si);
1054         clear_shallow_info(&si);
1055         return ref_cpy;
1056 }