git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int deepen_since_ok;
  25 static int deepen_not_ok;
  26 static int fetch_fsck_objects = -1;
  27 static int transfer_fsck_objects = -1;
  28 static int agent_supported;
  29 static struct lock_file shallow_lock;
  30 static const char *alternate_shallow_file;
  31
  32 /* Remember to update object flag allocation in object.h */
  33 #define COMPLETE        (1U << 0)
  34 #define COMMON          (1U << 1)
  35 #define COMMON_REF      (1U << 2)
  36 #define SEEN            (1U << 3)
  37 #define POPPED          (1U << 4)
  38
  39 static int marked;
  40
  41 /*
  42  * After sending this many "have"s if we do not get any new ACK , we
  43  * give up traversing our history.
  44  */
  45 #define MAX_IN_VAIN 256
  46
  47 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  48 static int non_common_revs, multi_ack, use_sideband;
  49 /* Allow specifying sha1 if it is a ref tip. */
  50 #define ALLOW_TIP_SHA1  01
  51 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  52 #define ALLOW_REACHABLE_SHA1    02
  53 static unsigned int allow_unadvertised_object_request;
  54
  55 __attribute__((format (printf, 2, 3)))
  56 static inline void print_verbose(const struct fetch_pack_args *args,
  57                                  const char *fmt, ...)
  58 {
  59         va_list params;
  60
  61         if (!args->verbose)
  62                 return;
  63
  64         va_start(params, fmt);
  65         vfprintf(stderr, fmt, params);
  66         va_end(params);
  67         fputc('\n', stderr);
  68 }
  69
  70 static void rev_list_push(struct commit *commit, int mark)
  71 {
  72         if (!(commit->object.flags & mark)) {
  73                 commit->object.flags |= mark;
  74
  75                 if (parse_commit(commit))
  76                         return;
  77
  78                 prio_queue_put(&rev_list, commit);
  79
  80                 if (!(commit->object.flags & COMMON))
  81                         non_common_revs++;
  82         }
  83 }
  84
  85 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  86 {
  87         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  88
  89         if (o && o->type == OBJ_COMMIT)
  90                 rev_list_push((struct commit *)o, SEEN);
  91
  92         return 0;
  93 }
  94
  95 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  96                                    int flag, void *cb_data)
  97 {
  98         return rev_list_insert_ref(refname, oid->hash);
  99 }
 100
 101 static int clear_marks(const char *refname, const struct object_id *oid,
 102                        int flag, void *cb_data)
 103 {
 104         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
 105
 106         if (o && o->type == OBJ_COMMIT)
 107                 clear_commit_marks((struct commit *)o,
 108                                    COMMON | COMMON_REF | SEEN | POPPED);
 109         return 0;
 110 }
 111
 112 /*
 113    This function marks a rev and its ancestors as common.
 114    In some cases, it is desirable to mark only the ancestors (for example
 115    when only the server does not yet know that they are common).
 116 */
 117
 118 static void mark_common(struct commit *commit,
 119                 int ancestors_only, int dont_parse)
 120 {
 121         if (commit != NULL && !(commit->object.flags & COMMON)) {
 122                 struct object *o = (struct object *)commit;
 123
 124                 if (!ancestors_only)
 125                         o->flags |= COMMON;
 126
 127                 if (!(o->flags & SEEN))
 128                         rev_list_push(commit, SEEN);
 129                 else {
 130                         struct commit_list *parents;
 131
 132                         if (!ancestors_only && !(o->flags & POPPED))
 133                                 non_common_revs--;
 134                         if (!o->parsed && !dont_parse)
 135                                 if (parse_commit(commit))
 136                                         return;
 137
 138                         for (parents = commit->parents;
 139                                         parents;
 140                                         parents = parents->next)
 141                                 mark_common(parents->item, 0, dont_parse);
 142                 }
 143         }
 144 }
 145
 146 /*
 147   Get the next rev to send, ignoring the common.
 148 */
 149
 150 static const unsigned char *get_rev(void)
 151 {
 152         struct commit *commit = NULL;
 153
 154         while (commit == NULL) {
 155                 unsigned int mark;
 156                 struct commit_list *parents;
 157
 158                 if (rev_list.nr == 0 || non_common_revs == 0)
 159                         return NULL;
 160
 161                 commit = prio_queue_get(&rev_list);
 162                 parse_commit(commit);
 163                 parents = commit->parents;
 164
 165                 commit->object.flags |= POPPED;
 166                 if (!(commit->object.flags & COMMON))
 167                         non_common_revs--;
 168
 169                 if (commit->object.flags & COMMON) {
 170                         /* do not send "have", and ignore ancestors */
 171                         commit = NULL;
 172                         mark = COMMON | SEEN;
 173                 } else if (commit->object.flags & COMMON_REF)
 174                         /* send "have", and ignore ancestors */
 175                         mark = COMMON | SEEN;
 176                 else
 177                         /* send "have", also for its ancestors */
 178                         mark = SEEN;
 179
 180                 while (parents) {
 181                         if (!(parents->item->object.flags & SEEN))
 182                                 rev_list_push(parents->item, mark);
 183                         if (mark & COMMON)
 184                                 mark_common(parents->item, 1, 0);
 185                         parents = parents->next;
 186                 }
 187         }
 188
 189         return commit->object.oid.hash;
 190 }
 191
 192 enum ack_type {
 193         NAK = 0,
 194         ACK,
 195         ACK_continue,
 196         ACK_common,
 197         ACK_ready
 198 };
 199
 200 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 201 {
 202         if (args->stateless_rpc && args->deepen) {
 203                 /* If we sent a depth we will get back "duplicate"
 204                  * shallow and unshallow commands every time there
 205                  * is a block of have lines exchanged.
 206                  */
 207                 char *line;
 208                 while ((line = packet_read_line(fd, NULL))) {
 209                         if (starts_with(line, "shallow "))
 210                                 continue;
 211                         if (starts_with(line, "unshallow "))
 212                                 continue;
 213                         die(_("git fetch-pack: expected shallow list"));
 214                 }
 215         }
 216 }
 217
 218 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 219 {
 220         int len;
 221         char *line = packet_read_line(fd, &len);
 222         const char *arg;
 223
 224         if (!len)
 225                 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
 226         if (!strcmp(line, "NAK"))
 227                 return NAK;
 228         if (skip_prefix(line, "ACK ", &arg)) {
 229                 if (!get_sha1_hex(arg, result_sha1)) {
 230                         arg += 40;
 231                         len -= arg - line;
 232                         if (len < 1)
 233                                 return ACK;
 234                         if (strstr(arg, "continue"))
 235                                 return ACK_continue;
 236                         if (strstr(arg, "common"))
 237                                 return ACK_common;
 238                         if (strstr(arg, "ready"))
 239                                 return ACK_ready;
 240                         return ACK;
 241                 }
 242         }
 243         die(_("git fetch_pack: expected ACK/NAK, got '%s'"), line);
 244 }
 245
 246 static void send_request(struct fetch_pack_args *args,
 247                          int fd, struct strbuf *buf)
 248 {
 249         if (args->stateless_rpc) {
 250                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 251                 packet_flush(fd);
 252         } else
 253                 write_or_die(fd, buf->buf, buf->len);
 254 }
 255
 256 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 257 {
 258         rev_list_insert_ref(NULL, ref->old_oid.hash);
 259 }
 260
 261 #define INITIAL_FLUSH 16
 262 #define PIPESAFE_FLUSH 32
 263 #define LARGE_FLUSH 1024
 264
 265 static int next_flush(struct fetch_pack_args *args, int count)
 266 {
 267         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 268
 269         if (count < flush_limit)
 270                 count <<= 1;
 271         else
 272                 count += flush_limit;
 273         return count;
 274 }
 275
 276 static int find_common(struct fetch_pack_args *args,
 277                        int fd[2], unsigned char *result_sha1,
 278                        struct ref *refs)
 279 {
 280         int fetching;
 281         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 282         const unsigned char *sha1;
 283         unsigned in_vain = 0;
 284         int got_continue = 0;
 285         int got_ready = 0;
 286         struct strbuf req_buf = STRBUF_INIT;
 287         size_t state_len = 0;
 288
 289         if (args->stateless_rpc && multi_ack == 1)
 290                 die(_("--stateless-rpc requires multi_ack_detailed"));
 291         if (marked)
 292                 for_each_ref(clear_marks, NULL);
 293         marked = 1;
 294
 295         for_each_ref(rev_list_insert_ref_oid, NULL);
 296         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 297
 298         fetching = 0;
 299         for ( ; refs ; refs = refs->next) {
 300                 unsigned char *remote = refs->old_oid.hash;
 301                 const char *remote_hex;
 302                 struct object *o;
 303
 304                 /*
 305                  * If that object is complete (i.e. it is an ancestor of a
 306                  * local ref), we tell them we have it but do not have to
 307                  * tell them about its ancestors, which they already know
 308                  * about.
 309                  *
 310                  * We use lookup_object here because we are only
 311                  * interested in the case we *know* the object is
 312                  * reachable and we have already scanned it.
 313                  */
 314                 if (((o = lookup_object(remote)) != NULL) &&
 315                                 (o->flags & COMPLETE)) {
 316                         continue;
 317                 }
 318
 319                 remote_hex = sha1_to_hex(remote);
 320                 if (!fetching) {
 321                         struct strbuf c = STRBUF_INIT;
 322                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 323                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 324                         if (no_done)            strbuf_addstr(&c, " no-done");
 325                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 326                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 327                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 328                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 329                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 330                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 331                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 332                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 333                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 334                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 335                                                             git_user_agent_sanitized());
 336                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 337                         strbuf_release(&c);
 338                 } else
 339                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 340                 fetching++;
 341         }
 342
 343         if (!fetching) {
 344                 strbuf_release(&req_buf);
 345                 packet_flush(fd[1]);
 346                 return 1;
 347         }
 348
 349         if (is_repository_shallow())
 350                 write_shallow_commits(&req_buf, 1, NULL);
 351         if (args->depth > 0)
 352                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 353         if (args->deepen_since) {
 354                 unsigned long max_age = approxidate(args->deepen_since);
 355                 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
 356         }
 357         if (args->deepen_not) {
 358                 int i;
 359                 for (i = 0; i < args->deepen_not->nr; i++) {
 360                         struct string_list_item *s = args->deepen_not->items + i;
 361                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 362                 }
 363         }
 364         packet_buf_flush(&req_buf);
 365         state_len = req_buf.len;
 366
 367         if (args->deepen) {
 368                 char *line;
 369                 const char *arg;
 370                 unsigned char sha1[20];
 371
 372                 send_request(args, fd[1], &req_buf);
 373                 while ((line = packet_read_line(fd[0], NULL))) {
 374                         if (skip_prefix(line, "shallow ", &arg)) {
 375                                 if (get_sha1_hex(arg, sha1))
 376                                         die(_("invalid shallow line: %s"), line);
 377                                 register_shallow(sha1);
 378                                 continue;
 379                         }
 380                         if (skip_prefix(line, "unshallow ", &arg)) {
 381                                 if (get_sha1_hex(arg, sha1))
 382                                         die(_("invalid unshallow line: %s"), line);
 383                                 if (!lookup_object(sha1))
 384                                         die(_("object not found: %s"), line);
 385                                 /* make sure that it is parsed as shallow */
 386                                 if (!parse_object(sha1))
 387                                         die(_("error in object: %s"), line);
 388                                 if (unregister_shallow(sha1))
 389                                         die(_("no shallow found: %s"), line);
 390                                 continue;
 391                         }
 392                         die(_("expected shallow/unshallow, got %s"), line);
 393                 }
 394         } else if (!args->stateless_rpc)
 395                 send_request(args, fd[1], &req_buf);
 396
 397         if (!args->stateless_rpc) {
 398                 /* If we aren't using the stateless-rpc interface
 399                  * we don't need to retain the headers.
 400                  */
 401                 strbuf_setlen(&req_buf, 0);
 402                 state_len = 0;
 403         }
 404
 405         flushes = 0;
 406         retval = -1;
 407         while ((sha1 = get_rev())) {
 408                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 409                 print_verbose(args, "have %s", sha1_to_hex(sha1));
 410                 in_vain++;
 411                 if (flush_at <= ++count) {
 412                         int ack;
 413
 414                         packet_buf_flush(&req_buf);
 415                         send_request(args, fd[1], &req_buf);
 416                         strbuf_setlen(&req_buf, state_len);
 417                         flushes++;
 418                         flush_at = next_flush(args, count);
 419
 420                         /*
 421                          * We keep one window "ahead" of the other side, and
 422                          * will wait for an ACK only on the next one
 423                          */
 424                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 425                                 continue;
 426
 427                         consume_shallow_list(args, fd[0]);
 428                         do {
 429                                 ack = get_ack(fd[0], result_sha1);
 430                                 if (ack)
 431                                         print_verbose(args, _("got %s %d %s"), "ack",
 432                                                       ack, sha1_to_hex(result_sha1));
 433                                 switch (ack) {
 434                                 case ACK:
 435                                         flushes = 0;
 436                                         multi_ack = 0;
 437                                         retval = 0;
 438                                         goto done;
 439                                 case ACK_common:
 440                                 case ACK_ready:
 441                                 case ACK_continue: {
 442                                         struct commit *commit =
 443                                                 lookup_commit(result_sha1);
 444                                         if (!commit)
 445                                                 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
 446                                         if (args->stateless_rpc
 447                                          && ack == ACK_common
 448                                          && !(commit->object.flags & COMMON)) {
 449                                                 /* We need to replay the have for this object
 450                                                  * on the next RPC request so the peer knows
 451                                                  * it is in common with us.
 452                                                  */
 453                                                 const char *hex = sha1_to_hex(result_sha1);
 454                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 455                                                 state_len = req_buf.len;
 456                                         }
 457                                         mark_common(commit, 0, 1);
 458                                         retval = 0;
 459                                         in_vain = 0;
 460                                         got_continue = 1;
 461                                         if (ack == ACK_ready) {
 462                                                 clear_prio_queue(&rev_list);
 463                                                 got_ready = 1;
 464                                         }
 465                                         break;
 466                                         }
 467                                 }
 468                         } while (ack);
 469                         flushes--;
 470                         if (got_continue && MAX_IN_VAIN < in_vain) {
 471                                 print_verbose(args, _("giving up"));
 472                                 break; /* give up */
 473                         }
 474                 }
 475         }
 476 done:
 477         if (!got_ready || !no_done) {
 478                 packet_buf_write(&req_buf, "done\n");
 479                 send_request(args, fd[1], &req_buf);
 480         }
 481         print_verbose(args, _("done"));
 482         if (retval != 0) {
 483                 multi_ack = 0;
 484                 flushes++;
 485         }
 486         strbuf_release(&req_buf);
 487
 488         if (!got_ready || !no_done)
 489                 consume_shallow_list(args, fd[0]);
 490         while (flushes || multi_ack) {
 491                 int ack = get_ack(fd[0], result_sha1);
 492                 if (ack) {
 493                         print_verbose(args, _("got %s (%d) %s"), "ack",
 494                                       ack, sha1_to_hex(result_sha1));
 495                         if (ack == ACK)
 496                                 return 0;
 497                         multi_ack = 1;
 498                         continue;
 499                 }
 500                 flushes--;
 501         }
 502         /* it is no error to fetch into a completely empty repo */
 503         return count ? retval : 0;
 504 }
 505
 506 static struct commit_list *complete;
 507
 508 static int mark_complete(const unsigned char *sha1)
 509 {
 510         struct object *o = parse_object(sha1);
 511
 512         while (o && o->type == OBJ_TAG) {
 513                 struct tag *t = (struct tag *) o;
 514                 if (!t->tagged)
 515                         break; /* broken repository */
 516                 o->flags |= COMPLETE;
 517                 o = parse_object(t->tagged->oid.hash);
 518         }
 519         if (o && o->type == OBJ_COMMIT) {
 520                 struct commit *commit = (struct commit *)o;
 521                 if (!(commit->object.flags & COMPLETE)) {
 522                         commit->object.flags |= COMPLETE;
 523                         commit_list_insert(commit, &complete);
 524                 }
 525         }
 526         return 0;
 527 }
 528
 529 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 530                              int flag, void *cb_data)
 531 {
 532         return mark_complete(oid->hash);
 533 }
 534
 535 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 536                                          unsigned long cutoff)
 537 {
 538         while (complete && cutoff <= complete->item->date) {
 539                 print_verbose(args, _("Marking %s as complete"),
 540                               oid_to_hex(&complete->item->object.oid));
 541                 pop_most_recent_commit(&complete, COMPLETE);
 542         }
 543 }
 544
 545 static void filter_refs(struct fetch_pack_args *args,
 546                         struct ref **refs,
 547                         struct ref **sought, int nr_sought)
 548 {
 549         struct ref *newlist = NULL;
 550         struct ref **newtail = &newlist;
 551         struct ref *ref, *next;
 552         int i;
 553
 554         i = 0;
 555         for (ref = *refs; ref; ref = next) {
 556                 int keep = 0;
 557                 next = ref->next;
 558
 559                 if (starts_with(ref->name, "refs/") &&
 560                     check_refname_format(ref->name, 0))
 561                         ; /* trash */
 562                 else {
 563                         while (i < nr_sought) {
 564                                 int cmp = strcmp(ref->name, sought[i]->name);
 565                                 if (cmp < 0)
 566                                         break; /* definitely do not have it */
 567                                 else if (cmp == 0) {
 568                                         keep = 1; /* definitely have it */
 569                                         sought[i]->matched = 1;
 570                                 }
 571                                 i++;
 572                         }
 573                 }
 574
 575                 if (!keep && args->fetch_all &&
 576                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 577                         keep = 1;
 578
 579                 if (keep) {
 580                         *newtail = ref;
 581                         ref->next = NULL;
 582                         newtail = &ref->next;
 583                 } else {
 584                         free(ref);
 585                 }
 586         }
 587
 588         /* Append unmatched requests to the list */
 589         if ((allow_unadvertised_object_request &
 590             (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 591                 for (i = 0; i < nr_sought; i++) {
 592                         unsigned char sha1[20];
 593
 594                         ref = sought[i];
 595                         if (ref->matched)
 596                                 continue;
 597                         if (get_sha1_hex(ref->name, sha1) ||
 598                             ref->name[40] != '\0' ||
 599                             hashcmp(sha1, ref->old_oid.hash))
 600                                 continue;
 601
 602                         ref->matched = 1;
 603                         *newtail = copy_ref(ref);
 604                         newtail = &(*newtail)->next;
 605                 }
 606         }
 607         *refs = newlist;
 608 }
 609
 610 static void mark_alternate_complete(const struct ref *ref, void *unused)
 611 {
 612         mark_complete(ref->old_oid.hash);
 613 }
 614
 615 static int everything_local(struct fetch_pack_args *args,
 616                             struct ref **refs,
 617                             struct ref **sought, int nr_sought)
 618 {
 619         struct ref *ref;
 620         int retval;
 621         unsigned long cutoff = 0;
 622
 623         save_commit_buffer = 0;
 624
 625         for (ref = *refs; ref; ref = ref->next) {
 626                 struct object *o;
 627
 628                 if (!has_object_file(&ref->old_oid))
 629                         continue;
 630
 631                 o = parse_object(ref->old_oid.hash);
 632                 if (!o)
 633                         continue;
 634
 635                 /* We already have it -- which may mean that we were
 636                  * in sync with the other side at some time after
 637                  * that (it is OK if we guess wrong here).
 638                  */
 639                 if (o->type == OBJ_COMMIT) {
 640                         struct commit *commit = (struct commit *)o;
 641                         if (!cutoff || cutoff < commit->date)
 642                                 cutoff = commit->date;
 643                 }
 644         }
 645
 646         if (!args->deepen) {
 647                 for_each_ref(mark_complete_oid, NULL);
 648                 for_each_alternate_ref(mark_alternate_complete, NULL);
 649                 commit_list_sort_by_date(&complete);
 650                 if (cutoff)
 651                         mark_recent_complete_commits(args, cutoff);
 652         }
 653
 654         /*
 655          * Mark all complete remote refs as common refs.
 656          * Don't mark them common yet; the server has to be told so first.
 657          */
 658         for (ref = *refs; ref; ref = ref->next) {
 659                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 660                                              NULL, 0);
 661
 662                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 663                         continue;
 664
 665                 if (!(o->flags & SEEN)) {
 666                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 667
 668                         mark_common((struct commit *)o, 1, 1);
 669                 }
 670         }
 671
 672         filter_refs(args, refs, sought, nr_sought);
 673
 674         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 675                 const unsigned char *remote = ref->old_oid.hash;
 676                 struct object *o;
 677
 678                 o = lookup_object(remote);
 679                 if (!o || !(o->flags & COMPLETE)) {
 680                         retval = 0;
 681                         print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
 682                                       ref->name);
 683                         continue;
 684                 }
 685                 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
 686                               ref->name);
 687         }
 688         return retval;
 689 }
 690
 691 static int sideband_demux(int in, int out, void *data)
 692 {
 693         int *xd = data;
 694
 695         int ret = recv_sideband("fetch-pack", xd[0], out);
 696         close(out);
 697         return ret;
 698 }
 699
 700 static int get_pack(struct fetch_pack_args *args,
 701                     int xd[2], char **pack_lockfile)
 702 {
 703         struct async demux;
 704         int do_keep = args->keep_pack;
 705         const char *cmd_name;
 706         struct pack_header header;
 707         int pass_header = 0;
 708         struct child_process cmd = CHILD_PROCESS_INIT;
 709         int ret;
 710
 711         memset(&demux, 0, sizeof(demux));
 712         if (use_sideband) {
 713                 /* xd[] is talking with upload-pack; subprocess reads from
 714                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 715                  * through demux->out.
 716                  */
 717                 demux.proc = sideband_demux;
 718                 demux.data = xd;
 719                 demux.out = -1;
 720                 if (start_async(&demux))
 721                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 722         }
 723         else
 724                 demux.out = xd[0];
 725
 726         if (!args->keep_pack && unpack_limit) {
 727
 728                 if (read_pack_header(demux.out, &header))
 729                         die(_("protocol error: bad pack header"));
 730                 pass_header = 1;
 731                 if (ntohl(header.hdr_entries) < unpack_limit)
 732                         do_keep = 0;
 733                 else
 734                         do_keep = 1;
 735         }
 736
 737         if (alternate_shallow_file) {
 738                 argv_array_push(&cmd.args, "--shallow-file");
 739                 argv_array_push(&cmd.args, alternate_shallow_file);
 740         }
 741
 742         if (do_keep) {
 743                 if (pack_lockfile)
 744                         cmd.out = -1;
 745                 cmd_name = "index-pack";
 746                 argv_array_push(&cmd.args, cmd_name);
 747                 argv_array_push(&cmd.args, "--stdin");
 748                 if (!args->quiet && !args->no_progress)
 749                         argv_array_push(&cmd.args, "-v");
 750                 if (args->use_thin_pack)
 751                         argv_array_push(&cmd.args, "--fix-thin");
 752                 if (args->lock_pack || unpack_limit) {
 753                         char hostname[256];
 754                         if (gethostname(hostname, sizeof(hostname)))
 755                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 756                         argv_array_pushf(&cmd.args,
 757                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 758                                         (uintmax_t)getpid(), hostname);
 759                 }
 760                 if (args->check_self_contained_and_connected)
 761                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 762         }
 763         else {
 764                 cmd_name = "unpack-objects";
 765                 argv_array_push(&cmd.args, cmd_name);
 766                 if (args->quiet || args->no_progress)
 767                         argv_array_push(&cmd.args, "-q");
 768                 args->check_self_contained_and_connected = 0;
 769         }
 770
 771         if (pass_header)
 772                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 773                                  ntohl(header.hdr_version),
 774                                  ntohl(header.hdr_entries));
 775         if (fetch_fsck_objects >= 0
 776             ? fetch_fsck_objects
 777             : transfer_fsck_objects >= 0
 778             ? transfer_fsck_objects
 779             : 0)
 780                 argv_array_push(&cmd.args, "--strict");
 781
 782         cmd.in = demux.out;
 783         cmd.git_cmd = 1;
 784         if (start_command(&cmd))
 785                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 786         if (do_keep && pack_lockfile) {
 787                 *pack_lockfile = index_pack_lockfile(cmd.out);
 788                 close(cmd.out);
 789         }
 790
 791         if (!use_sideband)
 792                 /* Closed by start_command() */
 793                 xd[0] = -1;
 794
 795         ret = finish_command(&cmd);
 796         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 797                 args->self_contained_and_connected =
 798                         args->check_self_contained_and_connected &&
 799                         ret == 0;
 800         else
 801                 die(_("%s failed"), cmd_name);
 802         if (use_sideband && finish_async(&demux))
 803                 die(_("error in sideband demultiplexer"));
 804         return 0;
 805 }
 806
 807 static int cmp_ref_by_name(const void *a_, const void *b_)
 808 {
 809         const struct ref *a = *((const struct ref **)a_);
 810         const struct ref *b = *((const struct ref **)b_);
 811         return strcmp(a->name, b->name);
 812 }
 813
 814 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 815                                  int fd[2],
 816                                  const struct ref *orig_ref,
 817                                  struct ref **sought, int nr_sought,
 818                                  struct shallow_info *si,
 819                                  char **pack_lockfile)
 820 {
 821         struct ref *ref = copy_ref_list(orig_ref);
 822         unsigned char sha1[20];
 823         const char *agent_feature;
 824         int agent_len;
 825
 826         sort_ref_list(&ref, ref_compare_name);
 827         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 828
 829         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 830                 die(_("Server does not support shallow clients"));
 831         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 832                 args->deepen = 1;
 833         if (server_supports("multi_ack_detailed")) {
 834                 print_verbose(args, _("Server supports multi_ack_detailed"));
 835                 multi_ack = 2;
 836                 if (server_supports("no-done")) {
 837                         print_verbose(args, _("Server supports no-done"));
 838                         if (args->stateless_rpc)
 839                                 no_done = 1;
 840                 }
 841         }
 842         else if (server_supports("multi_ack")) {
 843                 print_verbose(args, _("Server supports multi_ack"));
 844                 multi_ack = 1;
 845         }
 846         if (server_supports("side-band-64k")) {
 847                 print_verbose(args, _("Server supports side-band-64k"));
 848                 use_sideband = 2;
 849         }
 850         else if (server_supports("side-band")) {
 851                 print_verbose(args, _("Server supports side-band"));
 852                 use_sideband = 1;
 853         }
 854         if (server_supports("allow-tip-sha1-in-want")) {
 855                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 856                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 857         }
 858         if (server_supports("allow-reachable-sha1-in-want")) {
 859                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 860                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 861         }
 862         if (!server_supports("thin-pack"))
 863                 args->use_thin_pack = 0;
 864         if (!server_supports("no-progress"))
 865                 args->no_progress = 0;
 866         if (!server_supports("include-tag"))
 867                 args->include_tag = 0;
 868         if (server_supports("ofs-delta"))
 869                 print_verbose(args, _("Server supports ofs-delta"));
 870         else
 871                 prefer_ofs_delta = 0;
 872
 873         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 874                 agent_supported = 1;
 875                 if (agent_len)
 876                         print_verbose(args, _("Server version is %.*s"),
 877                                       agent_len, agent_feature);
 878         }
 879         if (server_supports("deepen-since"))
 880                 deepen_since_ok = 1;
 881         else if (args->deepen_since)
 882                 die(_("Server does not support --shallow-since"));
 883         if (server_supports("deepen-not"))
 884                 deepen_not_ok = 1;
 885         else if (args->deepen_not)
 886                 die(_("Server does not support --shallow-exclude"));
 887         if (!server_supports("deepen-relative") && args->deepen_relative)
 888                 die(_("Server does not support --deepen"));
 889
 890         if (everything_local(args, &ref, sought, nr_sought)) {
 891                 packet_flush(fd[1]);
 892                 goto all_done;
 893         }
 894         if (find_common(args, fd, sha1, ref) < 0)
 895                 if (!args->keep_pack)
 896                         /* When cloning, it is not unusual to have
 897                          * no common commit.
 898                          */
 899                         warning(_("no common commits"));
 900
 901         if (args->stateless_rpc)
 902                 packet_flush(fd[1]);
 903         if (args->deepen)
 904                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 905                                         NULL);
 906         else if (si->nr_ours || si->nr_theirs)
 907                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 908         else
 909                 alternate_shallow_file = NULL;
 910         if (get_pack(args, fd, pack_lockfile))
 911                 die(_("git fetch-pack: fetch failed."));
 912
 913  all_done:
 914         return ref;
 915 }
 916
 917 static void fetch_pack_config(void)
 918 {
 919         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 920         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 921         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 922         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 923         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 924
 925         git_config(git_default_config, NULL);
 926 }
 927
 928 static void fetch_pack_setup(void)
 929 {
 930         static int did_setup;
 931         if (did_setup)
 932                 return;
 933         fetch_pack_config();
 934         if (0 <= transfer_unpack_limit)
 935                 unpack_limit = transfer_unpack_limit;
 936         else if (0 <= fetch_unpack_limit)
 937                 unpack_limit = fetch_unpack_limit;
 938         did_setup = 1;
 939 }
 940
 941 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 942 {
 943         struct string_list names = STRING_LIST_INIT_NODUP;
 944         int src, dst;
 945
 946         for (src = dst = 0; src < nr; src++) {
 947                 struct string_list_item *item;
 948                 item = string_list_insert(&names, ref[src]->name);
 949                 if (item->util)
 950                         continue; /* already have it */
 951                 item->util = ref[src];
 952                 if (src != dst)
 953                         ref[dst] = ref[src];
 954                 dst++;
 955         }
 956         for (src = dst; src < nr; src++)
 957                 ref[src] = NULL;
 958         string_list_clear(&names, 0);
 959         return dst;
 960 }
 961
 962 static void update_shallow(struct fetch_pack_args *args,
 963                            struct ref **sought, int nr_sought,
 964                            struct shallow_info *si)
 965 {
 966         struct sha1_array ref = SHA1_ARRAY_INIT;
 967         int *status;
 968         int i;
 969
 970         if (args->deepen && alternate_shallow_file) {
 971                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 972                         unlink_or_warn(git_path_shallow());
 973                         rollback_lock_file(&shallow_lock);
 974                 } else
 975                         commit_lock_file(&shallow_lock);
 976                 return;
 977         }
 978
 979         if (!si->shallow || !si->shallow->nr)
 980                 return;
 981
 982         if (args->cloning) {
 983                 /*
 984                  * remote is shallow, but this is a clone, there are
 985                  * no objects in repo to worry about. Accept any
 986                  * shallow points that exist in the pack (iow in repo
 987                  * after get_pack() and reprepare_packed_git())
 988                  */
 989                 struct sha1_array extra = SHA1_ARRAY_INIT;
 990                 unsigned char (*sha1)[20] = si->shallow->sha1;
 991                 for (i = 0; i < si->shallow->nr; i++)
 992                         if (has_sha1_file(sha1[i]))
 993                                 sha1_array_append(&extra, sha1[i]);
 994                 if (extra.nr) {
 995                         setup_alternate_shallow(&shallow_lock,
 996                                                 &alternate_shallow_file,
 997                                                 &extra);
 998                         commit_lock_file(&shallow_lock);
 999                 }
1000                 sha1_array_clear(&extra);
1001                 return;
1002         }
1003
1004         if (!si->nr_ours && !si->nr_theirs)
1005                 return;
1006
1007         remove_nonexistent_theirs_shallow(si);
1008         if (!si->nr_ours && !si->nr_theirs)
1009                 return;
1010         for (i = 0; i < nr_sought; i++)
1011                 sha1_array_append(&ref, sought[i]->old_oid.hash);
1012         si->ref = &ref;
1013
1014         if (args->update_shallow) {
1015                 /*
1016                  * remote is also shallow, .git/shallow may be updated
1017                  * so all refs can be accepted. Make sure we only add
1018                  * shallow roots that are actually reachable from new
1019                  * refs.
1020                  */
1021                 struct sha1_array extra = SHA1_ARRAY_INIT;
1022                 unsigned char (*sha1)[20] = si->shallow->sha1;
1023                 assign_shallow_commits_to_refs(si, NULL, NULL);
1024                 if (!si->nr_ours && !si->nr_theirs) {
1025                         sha1_array_clear(&ref);
1026                         return;
1027                 }
1028                 for (i = 0; i < si->nr_ours; i++)
1029                         sha1_array_append(&extra, sha1[si->ours[i]]);
1030                 for (i = 0; i < si->nr_theirs; i++)
1031                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1032                 setup_alternate_shallow(&shallow_lock,
1033                                         &alternate_shallow_file,
1034                                         &extra);
1035                 commit_lock_file(&shallow_lock);
1036                 sha1_array_clear(&extra);
1037                 sha1_array_clear(&ref);
1038                 return;
1039         }
1040
1041         /*
1042          * remote is also shallow, check what ref is safe to update
1043          * without updating .git/shallow
1044          */
1045         status = xcalloc(nr_sought, sizeof(*status));
1046         assign_shallow_commits_to_refs(si, NULL, status);
1047         if (si->nr_ours || si->nr_theirs) {
1048                 for (i = 0; i < nr_sought; i++)
1049                         if (status[i])
1050                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1051         }
1052         free(status);
1053         sha1_array_clear(&ref);
1054 }
1055
1056 struct ref *fetch_pack(struct fetch_pack_args *args,
1057                        int fd[], struct child_process *conn,
1058                        const struct ref *ref,
1059                        const char *dest,
1060                        struct ref **sought, int nr_sought,
1061                        struct sha1_array *shallow,
1062                        char **pack_lockfile)
1063 {
1064         struct ref *ref_cpy;
1065         struct shallow_info si;
1066
1067         fetch_pack_setup();
1068         if (nr_sought)
1069                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1070
1071         if (!ref) {
1072                 packet_flush(fd[1]);
1073                 die(_("no matching remote head"));
1074         }
1075         prepare_shallow_info(&si, shallow);
1076         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1077                                 &si, pack_lockfile);
1078         reprepare_packed_git();
1079         update_shallow(args, sought, nr_sought, &si);
1080         clear_shallow_info(&si);
1081         return ref_cpy;
1082 }