git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int fetch_fsck_objects = -1;
  25 static int transfer_fsck_objects = -1;
  26 static int agent_supported;
  27 static struct lock_file shallow_lock;
  28 static const char *alternate_shallow_file;
  29
  30 /* Remember to update object flag allocation in object.h */
  31 #define COMPLETE        (1U << 0)
  32 #define COMMON          (1U << 1)
  33 #define COMMON_REF      (1U << 2)
  34 #define SEEN            (1U << 3)
  35 #define POPPED          (1U << 4)
  36
  37 static int marked;
  38
  39 /*
  40  * After sending this many "have"s if we do not get any new ACK , we
  41  * give up traversing our history.
  42  */
  43 #define MAX_IN_VAIN 256
  44
  45 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  46 static int non_common_revs, multi_ack, use_sideband;
  47 /* Allow specifying sha1 if it is a ref tip. */
  48 #define ALLOW_TIP_SHA1  01
  49 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  50 #define ALLOW_REACHABLE_SHA1    02
  51 static unsigned int allow_unadvertised_object_request;
  52
  53 static void rev_list_push(struct commit *commit, int mark)
  54 {
  55         if (!(commit->object.flags & mark)) {
  56                 commit->object.flags |= mark;
  57
  58                 if (parse_commit(commit))
  59                         return;
  60
  61                 prio_queue_put(&rev_list, commit);
  62
  63                 if (!(commit->object.flags & COMMON))
  64                         non_common_revs++;
  65         }
  66 }
  67
  68 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  69 {
  70         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  71
  72         if (o && o->type == OBJ_COMMIT)
  73                 rev_list_push((struct commit *)o, SEEN);
  74
  75         return 0;
  76 }
  77
  78 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  79                                    int flag, void *cb_data)
  80 {
  81         return rev_list_insert_ref(refname, oid->hash);
  82 }
  83
  84 static int clear_marks(const char *refname, const struct object_id *oid,
  85                        int flag, void *cb_data)
  86 {
  87         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
  88
  89         if (o && o->type == OBJ_COMMIT)
  90                 clear_commit_marks((struct commit *)o,
  91                                    COMMON | COMMON_REF | SEEN | POPPED);
  92         return 0;
  93 }
  94
  95 /*
  96    This function marks a rev and its ancestors as common.
  97    In some cases, it is desirable to mark only the ancestors (for example
  98    when only the server does not yet know that they are common).
  99 */
 100
 101 static void mark_common(struct commit *commit,
 102                 int ancestors_only, int dont_parse)
 103 {
 104         if (commit != NULL && !(commit->object.flags & COMMON)) {
 105                 struct object *o = (struct object *)commit;
 106
 107                 if (!ancestors_only)
 108                         o->flags |= COMMON;
 109
 110                 if (!(o->flags & SEEN))
 111                         rev_list_push(commit, SEEN);
 112                 else {
 113                         struct commit_list *parents;
 114
 115                         if (!ancestors_only && !(o->flags & POPPED))
 116                                 non_common_revs--;
 117                         if (!o->parsed && !dont_parse)
 118                                 if (parse_commit(commit))
 119                                         return;
 120
 121                         for (parents = commit->parents;
 122                                         parents;
 123                                         parents = parents->next)
 124                                 mark_common(parents->item, 0, dont_parse);
 125                 }
 126         }
 127 }
 128
 129 /*
 130   Get the next rev to send, ignoring the common.
 131 */
 132
 133 static const unsigned char *get_rev(void)
 134 {
 135         struct commit *commit = NULL;
 136
 137         while (commit == NULL) {
 138                 unsigned int mark;
 139                 struct commit_list *parents;
 140
 141                 if (rev_list.nr == 0 || non_common_revs == 0)
 142                         return NULL;
 143
 144                 commit = prio_queue_get(&rev_list);
 145                 parse_commit(commit);
 146                 parents = commit->parents;
 147
 148                 commit->object.flags |= POPPED;
 149                 if (!(commit->object.flags & COMMON))
 150                         non_common_revs--;
 151
 152                 if (commit->object.flags & COMMON) {
 153                         /* do not send "have", and ignore ancestors */
 154                         commit = NULL;
 155                         mark = COMMON | SEEN;
 156                 } else if (commit->object.flags & COMMON_REF)
 157                         /* send "have", and ignore ancestors */
 158                         mark = COMMON | SEEN;
 159                 else
 160                         /* send "have", also for its ancestors */
 161                         mark = SEEN;
 162
 163                 while (parents) {
 164                         if (!(parents->item->object.flags & SEEN))
 165                                 rev_list_push(parents->item, mark);
 166                         if (mark & COMMON)
 167                                 mark_common(parents->item, 1, 0);
 168                         parents = parents->next;
 169                 }
 170         }
 171
 172         return commit->object.oid.hash;
 173 }
 174
 175 enum ack_type {
 176         NAK = 0,
 177         ACK,
 178         ACK_continue,
 179         ACK_common,
 180         ACK_ready
 181 };
 182
 183 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 184 {
 185         if (args->stateless_rpc && args->depth > 0) {
 186                 /* If we sent a depth we will get back "duplicate"
 187                  * shallow and unshallow commands every time there
 188                  * is a block of have lines exchanged.
 189                  */
 190                 char *line;
 191                 while ((line = packet_read_line(fd, NULL))) {
 192                         if (starts_with(line, "shallow "))
 193                                 continue;
 194                         if (starts_with(line, "unshallow "))
 195                                 continue;
 196                         die("git fetch-pack: expected shallow list");
 197                 }
 198         }
 199 }
 200
 201 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 202 {
 203         int len;
 204         char *line = packet_read_line(fd, &len);
 205         const char *arg;
 206
 207         if (!len)
 208                 die("git fetch-pack: expected ACK/NAK, got EOF");
 209         if (!strcmp(line, "NAK"))
 210                 return NAK;
 211         if (skip_prefix(line, "ACK ", &arg)) {
 212                 if (!get_sha1_hex(arg, result_sha1)) {
 213                         arg += 40;
 214                         len -= arg - line;
 215                         if (len < 1)
 216                                 return ACK;
 217                         if (strstr(arg, "continue"))
 218                                 return ACK_continue;
 219                         if (strstr(arg, "common"))
 220                                 return ACK_common;
 221                         if (strstr(arg, "ready"))
 222                                 return ACK_ready;
 223                         return ACK;
 224                 }
 225         }
 226         die("git fetch_pack: expected ACK/NAK, got '%s'", line);
 227 }
 228
 229 static void send_request(struct fetch_pack_args *args,
 230                          int fd, struct strbuf *buf)
 231 {
 232         if (args->stateless_rpc) {
 233                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 234                 packet_flush(fd);
 235         } else
 236                 write_or_die(fd, buf->buf, buf->len);
 237 }
 238
 239 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 240 {
 241         rev_list_insert_ref(NULL, ref->old_oid.hash);
 242 }
 243
 244 #define INITIAL_FLUSH 16
 245 #define PIPESAFE_FLUSH 32
 246 #define LARGE_FLUSH 16384
 247
 248 static int next_flush(struct fetch_pack_args *args, int count)
 249 {
 250         if (args->stateless_rpc) {
 251                 if (count < LARGE_FLUSH)
 252                         count <<= 1;
 253                 else
 254                         count = count * 11 / 10;
 255         } else {
 256                 if (count < PIPESAFE_FLUSH)
 257                         count <<= 1;
 258                 else
 259                         count += PIPESAFE_FLUSH;
 260         }
 261         return count;
 262 }
 263
 264 static int find_common(struct fetch_pack_args *args,
 265                        int fd[2], unsigned char *result_sha1,
 266                        struct ref *refs)
 267 {
 268         int fetching;
 269         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 270         const unsigned char *sha1;
 271         unsigned in_vain = 0;
 272         int got_continue = 0;
 273         int got_ready = 0;
 274         struct strbuf req_buf = STRBUF_INIT;
 275         size_t state_len = 0;
 276
 277         if (args->stateless_rpc && multi_ack == 1)
 278                 die("--stateless-rpc requires multi_ack_detailed");
 279         if (marked)
 280                 for_each_ref(clear_marks, NULL);
 281         marked = 1;
 282
 283         for_each_ref(rev_list_insert_ref_oid, NULL);
 284         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 285
 286         fetching = 0;
 287         for ( ; refs ; refs = refs->next) {
 288                 unsigned char *remote = refs->old_oid.hash;
 289                 const char *remote_hex;
 290                 struct object *o;
 291
 292                 /*
 293                  * If that object is complete (i.e. it is an ancestor of a
 294                  * local ref), we tell them we have it but do not have to
 295                  * tell them about its ancestors, which they already know
 296                  * about.
 297                  *
 298                  * We use lookup_object here because we are only
 299                  * interested in the case we *know* the object is
 300                  * reachable and we have already scanned it.
 301                  */
 302                 if (((o = lookup_object(remote)) != NULL) &&
 303                                 (o->flags & COMPLETE)) {
 304                         continue;
 305                 }
 306
 307                 remote_hex = sha1_to_hex(remote);
 308                 if (!fetching) {
 309                         struct strbuf c = STRBUF_INIT;
 310                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 311                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 312                         if (no_done)            strbuf_addstr(&c, " no-done");
 313                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 314                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 315                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 316                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 317                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 318                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 319                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 320                                                             git_user_agent_sanitized());
 321                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 322                         strbuf_release(&c);
 323                 } else
 324                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 325                 fetching++;
 326         }
 327
 328         if (!fetching) {
 329                 strbuf_release(&req_buf);
 330                 packet_flush(fd[1]);
 331                 return 1;
 332         }
 333
 334         if (is_repository_shallow())
 335                 write_shallow_commits(&req_buf, 1, NULL);
 336         if (args->depth > 0)
 337                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 338         packet_buf_flush(&req_buf);
 339         state_len = req_buf.len;
 340
 341         if (args->depth > 0) {
 342                 char *line;
 343                 const char *arg;
 344                 unsigned char sha1[20];
 345
 346                 send_request(args, fd[1], &req_buf);
 347                 while ((line = packet_read_line(fd[0], NULL))) {
 348                         if (skip_prefix(line, "shallow ", &arg)) {
 349                                 if (get_sha1_hex(arg, sha1))
 350                                         die("invalid shallow line: %s", line);
 351                                 register_shallow(sha1);
 352                                 continue;
 353                         }
 354                         if (skip_prefix(line, "unshallow ", &arg)) {
 355                                 if (get_sha1_hex(arg, sha1))
 356                                         die("invalid unshallow line: %s", line);
 357                                 if (!lookup_object(sha1))
 358                                         die("object not found: %s", line);
 359                                 /* make sure that it is parsed as shallow */
 360                                 if (!parse_object(sha1))
 361                                         die("error in object: %s", line);
 362                                 if (unregister_shallow(sha1))
 363                                         die("no shallow found: %s", line);
 364                                 continue;
 365                         }
 366                         die("expected shallow/unshallow, got %s", line);
 367                 }
 368         } else if (!args->stateless_rpc)
 369                 send_request(args, fd[1], &req_buf);
 370
 371         if (!args->stateless_rpc) {
 372                 /* If we aren't using the stateless-rpc interface
 373                  * we don't need to retain the headers.
 374                  */
 375                 strbuf_setlen(&req_buf, 0);
 376                 state_len = 0;
 377         }
 378
 379         flushes = 0;
 380         retval = -1;
 381         while ((sha1 = get_rev())) {
 382                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 383                 if (args->verbose)
 384                         fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
 385                 in_vain++;
 386                 if (flush_at <= ++count) {
 387                         int ack;
 388
 389                         packet_buf_flush(&req_buf);
 390                         send_request(args, fd[1], &req_buf);
 391                         strbuf_setlen(&req_buf, state_len);
 392                         flushes++;
 393                         flush_at = next_flush(args, count);
 394
 395                         /*
 396                          * We keep one window "ahead" of the other side, and
 397                          * will wait for an ACK only on the next one
 398                          */
 399                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 400                                 continue;
 401
 402                         consume_shallow_list(args, fd[0]);
 403                         do {
 404                                 ack = get_ack(fd[0], result_sha1);
 405                                 if (args->verbose && ack)
 406                                         fprintf(stderr, "got ack %d %s\n", ack,
 407                                                         sha1_to_hex(result_sha1));
 408                                 switch (ack) {
 409                                 case ACK:
 410                                         flushes = 0;
 411                                         multi_ack = 0;
 412                                         retval = 0;
 413                                         goto done;
 414                                 case ACK_common:
 415                                 case ACK_ready:
 416                                 case ACK_continue: {
 417                                         struct commit *commit =
 418                                                 lookup_commit(result_sha1);
 419                                         if (!commit)
 420                                                 die("invalid commit %s", sha1_to_hex(result_sha1));
 421                                         if (args->stateless_rpc
 422                                          && ack == ACK_common
 423                                          && !(commit->object.flags & COMMON)) {
 424                                                 /* We need to replay the have for this object
 425                                                  * on the next RPC request so the peer knows
 426                                                  * it is in common with us.
 427                                                  */
 428                                                 const char *hex = sha1_to_hex(result_sha1);
 429                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 430                                                 state_len = req_buf.len;
 431                                                 /*
 432                                                  * Reset in_vain because an ack
 433                                                  * for this commit has not been
 434                                                  * seen.
 435                                                  */
 436                                                 in_vain = 0;
 437                                         } else if (!args->stateless_rpc
 438                                                    || ack != ACK_common)
 439                                                 in_vain = 0;
 440                                         mark_common(commit, 0, 1);
 441                                         retval = 0;
 442                                         got_continue = 1;
 443                                         if (ack == ACK_ready) {
 444                                                 clear_prio_queue(&rev_list);
 445                                                 got_ready = 1;
 446                                         }
 447                                         break;
 448                                         }
 449                                 }
 450                         } while (ack);
 451                         flushes--;
 452                         if (got_continue && MAX_IN_VAIN < in_vain) {
 453                                 if (args->verbose)
 454                                         fprintf(stderr, "giving up\n");
 455                                 break; /* give up */
 456                         }
 457                 }
 458         }
 459 done:
 460         if (!got_ready || !no_done) {
 461                 packet_buf_write(&req_buf, "done\n");
 462                 send_request(args, fd[1], &req_buf);
 463         }
 464         if (args->verbose)
 465                 fprintf(stderr, "done\n");
 466         if (retval != 0) {
 467                 multi_ack = 0;
 468                 flushes++;
 469         }
 470         strbuf_release(&req_buf);
 471
 472         if (!got_ready || !no_done)
 473                 consume_shallow_list(args, fd[0]);
 474         while (flushes || multi_ack) {
 475                 int ack = get_ack(fd[0], result_sha1);
 476                 if (ack) {
 477                         if (args->verbose)
 478                                 fprintf(stderr, "got ack (%d) %s\n", ack,
 479                                         sha1_to_hex(result_sha1));
 480                         if (ack == ACK)
 481                                 return 0;
 482                         multi_ack = 1;
 483                         continue;
 484                 }
 485                 flushes--;
 486         }
 487         /* it is no error to fetch into a completely empty repo */
 488         return count ? retval : 0;
 489 }
 490
 491 static struct commit_list *complete;
 492
 493 static int mark_complete(const unsigned char *sha1)
 494 {
 495         struct object *o = parse_object(sha1);
 496
 497         while (o && o->type == OBJ_TAG) {
 498                 struct tag *t = (struct tag *) o;
 499                 if (!t->tagged)
 500                         break; /* broken repository */
 501                 o->flags |= COMPLETE;
 502                 o = parse_object(t->tagged->oid.hash);
 503         }
 504         if (o && o->type == OBJ_COMMIT) {
 505                 struct commit *commit = (struct commit *)o;
 506                 if (!(commit->object.flags & COMPLETE)) {
 507                         commit->object.flags |= COMPLETE;
 508                         commit_list_insert(commit, &complete);
 509                 }
 510         }
 511         return 0;
 512 }
 513
 514 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 515                              int flag, void *cb_data)
 516 {
 517         return mark_complete(oid->hash);
 518 }
 519
 520 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 521                                          unsigned long cutoff)
 522 {
 523         while (complete && cutoff <= complete->item->date) {
 524                 if (args->verbose)
 525                         fprintf(stderr, "Marking %s as complete\n",
 526                                 oid_to_hex(&complete->item->object.oid));
 527                 pop_most_recent_commit(&complete, COMPLETE);
 528         }
 529 }
 530
 531 static void filter_refs(struct fetch_pack_args *args,
 532                         struct ref **refs,
 533                         struct ref **sought, int nr_sought)
 534 {
 535         struct ref *newlist = NULL;
 536         struct ref **newtail = &newlist;
 537         struct ref *ref, *next;
 538         int i;
 539
 540         i = 0;
 541         for (ref = *refs; ref; ref = next) {
 542                 int keep = 0;
 543                 next = ref->next;
 544
 545                 if (starts_with(ref->name, "refs/") &&
 546                     check_refname_format(ref->name, 0))
 547                         ; /* trash */
 548                 else {
 549                         while (i < nr_sought) {
 550                                 int cmp = strcmp(ref->name, sought[i]->name);
 551                                 if (cmp < 0)
 552                                         break; /* definitely do not have it */
 553                                 else if (cmp == 0) {
 554                                         keep = 1; /* definitely have it */
 555                                         sought[i]->matched = 1;
 556                                 }
 557                                 i++;
 558                         }
 559                 }
 560
 561                 if (!keep && args->fetch_all &&
 562                     (!args->depth || !starts_with(ref->name, "refs/tags/")))
 563                         keep = 1;
 564
 565                 if (keep) {
 566                         *newtail = ref;
 567                         ref->next = NULL;
 568                         newtail = &ref->next;
 569                 } else {
 570                         free(ref);
 571                 }
 572         }
 573
 574         /* Append unmatched requests to the list */
 575         if ((allow_unadvertised_object_request &
 576             (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 577                 for (i = 0; i < nr_sought; i++) {
 578                         unsigned char sha1[20];
 579
 580                         ref = sought[i];
 581                         if (ref->matched)
 582                                 continue;
 583                         if (get_sha1_hex(ref->name, sha1) ||
 584                             ref->name[40] != '\0' ||
 585                             hashcmp(sha1, ref->old_oid.hash))
 586                                 continue;
 587
 588                         ref->matched = 1;
 589                         *newtail = copy_ref(ref);
 590                         newtail = &(*newtail)->next;
 591                 }
 592         }
 593         *refs = newlist;
 594 }
 595
 596 static void mark_alternate_complete(const struct ref *ref, void *unused)
 597 {
 598         mark_complete(ref->old_oid.hash);
 599 }
 600
 601 static int everything_local(struct fetch_pack_args *args,
 602                             struct ref **refs,
 603                             struct ref **sought, int nr_sought)
 604 {
 605         struct ref *ref;
 606         int retval;
 607         unsigned long cutoff = 0;
 608
 609         save_commit_buffer = 0;
 610
 611         for (ref = *refs; ref; ref = ref->next) {
 612                 struct object *o;
 613
 614                 if (!has_object_file(&ref->old_oid))
 615                         continue;
 616
 617                 o = parse_object(ref->old_oid.hash);
 618                 if (!o)
 619                         continue;
 620
 621                 /* We already have it -- which may mean that we were
 622                  * in sync with the other side at some time after
 623                  * that (it is OK if we guess wrong here).
 624                  */
 625                 if (o->type == OBJ_COMMIT) {
 626                         struct commit *commit = (struct commit *)o;
 627                         if (!cutoff || cutoff < commit->date)
 628                                 cutoff = commit->date;
 629                 }
 630         }
 631
 632         if (!args->depth) {
 633                 for_each_ref(mark_complete_oid, NULL);
 634                 for_each_alternate_ref(mark_alternate_complete, NULL);
 635                 commit_list_sort_by_date(&complete);
 636                 if (cutoff)
 637                         mark_recent_complete_commits(args, cutoff);
 638         }
 639
 640         /*
 641          * Mark all complete remote refs as common refs.
 642          * Don't mark them common yet; the server has to be told so first.
 643          */
 644         for (ref = *refs; ref; ref = ref->next) {
 645                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 646                                              NULL, 0);
 647
 648                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 649                         continue;
 650
 651                 if (!(o->flags & SEEN)) {
 652                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 653
 654                         mark_common((struct commit *)o, 1, 1);
 655                 }
 656         }
 657
 658         filter_refs(args, refs, sought, nr_sought);
 659
 660         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 661                 const unsigned char *remote = ref->old_oid.hash;
 662                 struct object *o;
 663
 664                 o = lookup_object(remote);
 665                 if (!o || !(o->flags & COMPLETE)) {
 666                         retval = 0;
 667                         if (!args->verbose)
 668                                 continue;
 669                         fprintf(stderr,
 670                                 "want %s (%s)\n", sha1_to_hex(remote),
 671                                 ref->name);
 672                         continue;
 673                 }
 674                 if (!args->verbose)
 675                         continue;
 676                 fprintf(stderr,
 677                         "already have %s (%s)\n", sha1_to_hex(remote),
 678                         ref->name);
 679         }
 680         return retval;
 681 }
 682
 683 static int sideband_demux(int in, int out, void *data)
 684 {
 685         int *xd = data;
 686         int ret;
 687
 688         ret = recv_sideband("fetch-pack", xd[0], out);
 689         close(out);
 690         return ret;
 691 }
 692
 693 static int get_pack(struct fetch_pack_args *args,
 694                     int xd[2], char **pack_lockfile)
 695 {
 696         struct async demux;
 697         int do_keep = args->keep_pack;
 698         const char *cmd_name;
 699         struct pack_header header;
 700         int pass_header = 0;
 701         struct child_process cmd = CHILD_PROCESS_INIT;
 702         int ret;
 703
 704         memset(&demux, 0, sizeof(demux));
 705         if (use_sideband) {
 706                 /* xd[] is talking with upload-pack; subprocess reads from
 707                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 708                  * through demux->out.
 709                  */
 710                 demux.proc = sideband_demux;
 711                 demux.data = xd;
 712                 demux.out = -1;
 713                 demux.isolate_sigpipe = 1;
 714                 if (start_async(&demux))
 715                         die("fetch-pack: unable to fork off sideband"
 716                             " demultiplexer");
 717         }
 718         else
 719                 demux.out = xd[0];
 720
 721         if (!args->keep_pack && unpack_limit) {
 722
 723                 if (read_pack_header(demux.out, &header))
 724                         die("protocol error: bad pack header");
 725                 pass_header = 1;
 726                 if (ntohl(header.hdr_entries) < unpack_limit)
 727                         do_keep = 0;
 728                 else
 729                         do_keep = 1;
 730         }
 731
 732         if (alternate_shallow_file) {
 733                 argv_array_push(&cmd.args, "--shallow-file");
 734                 argv_array_push(&cmd.args, alternate_shallow_file);
 735         }
 736
 737         if (do_keep) {
 738                 if (pack_lockfile)
 739                         cmd.out = -1;
 740                 cmd_name = "index-pack";
 741                 argv_array_push(&cmd.args, cmd_name);
 742                 argv_array_push(&cmd.args, "--stdin");
 743                 if (!args->quiet && !args->no_progress)
 744                         argv_array_push(&cmd.args, "-v");
 745                 if (args->use_thin_pack)
 746                         argv_array_push(&cmd.args, "--fix-thin");
 747                 if (args->lock_pack || unpack_limit) {
 748                         char hostname[256];
 749                         if (gethostname(hostname, sizeof(hostname)))
 750                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 751                         argv_array_pushf(&cmd.args,
 752                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 753                                         (uintmax_t)getpid(), hostname);
 754                 }
 755                 if (args->check_self_contained_and_connected)
 756                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 757         }
 758         else {
 759                 cmd_name = "unpack-objects";
 760                 argv_array_push(&cmd.args, cmd_name);
 761                 if (args->quiet || args->no_progress)
 762                         argv_array_push(&cmd.args, "-q");
 763                 args->check_self_contained_and_connected = 0;
 764         }
 765
 766         if (pass_header)
 767                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 768                                  ntohl(header.hdr_version),
 769                                  ntohl(header.hdr_entries));
 770         if (fetch_fsck_objects >= 0
 771             ? fetch_fsck_objects
 772             : transfer_fsck_objects >= 0
 773             ? transfer_fsck_objects
 774             : 0)
 775                 argv_array_push(&cmd.args, "--strict");
 776
 777         cmd.in = demux.out;
 778         cmd.git_cmd = 1;
 779         if (start_command(&cmd))
 780                 die("fetch-pack: unable to fork off %s", cmd_name);
 781         if (do_keep && pack_lockfile) {
 782                 *pack_lockfile = index_pack_lockfile(cmd.out);
 783                 close(cmd.out);
 784         }
 785
 786         if (!use_sideband)
 787                 /* Closed by start_command() */
 788                 xd[0] = -1;
 789
 790         ret = finish_command(&cmd);
 791         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 792                 args->self_contained_and_connected =
 793                         args->check_self_contained_and_connected &&
 794                         ret == 0;
 795         else
 796                 die("%s failed", cmd_name);
 797         if (use_sideband && finish_async(&demux))
 798                 die("error in sideband demultiplexer");
 799         return 0;
 800 }
 801
 802 static int cmp_ref_by_name(const void *a_, const void *b_)
 803 {
 804         const struct ref *a = *((const struct ref **)a_);
 805         const struct ref *b = *((const struct ref **)b_);
 806         return strcmp(a->name, b->name);
 807 }
 808
 809 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 810                                  int fd[2],
 811                                  const struct ref *orig_ref,
 812                                  struct ref **sought, int nr_sought,
 813                                  struct shallow_info *si,
 814                                  char **pack_lockfile)
 815 {
 816         struct ref *ref = copy_ref_list(orig_ref);
 817         unsigned char sha1[20];
 818         const char *agent_feature;
 819         int agent_len;
 820
 821         sort_ref_list(&ref, ref_compare_name);
 822         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 823
 824         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 825                 die("Server does not support shallow clients");
 826         if (server_supports("multi_ack_detailed")) {
 827                 if (args->verbose)
 828                         fprintf(stderr, "Server supports multi_ack_detailed\n");
 829                 multi_ack = 2;
 830                 if (server_supports("no-done")) {
 831                         if (args->verbose)
 832                                 fprintf(stderr, "Server supports no-done\n");
 833                         if (args->stateless_rpc)
 834                                 no_done = 1;
 835                 }
 836         }
 837         else if (server_supports("multi_ack")) {
 838                 if (args->verbose)
 839                         fprintf(stderr, "Server supports multi_ack\n");
 840                 multi_ack = 1;
 841         }
 842         if (server_supports("side-band-64k")) {
 843                 if (args->verbose)
 844                         fprintf(stderr, "Server supports side-band-64k\n");
 845                 use_sideband = 2;
 846         }
 847         else if (server_supports("side-band")) {
 848                 if (args->verbose)
 849                         fprintf(stderr, "Server supports side-band\n");
 850                 use_sideband = 1;
 851         }
 852         if (server_supports("allow-tip-sha1-in-want")) {
 853                 if (args->verbose)
 854                         fprintf(stderr, "Server supports allow-tip-sha1-in-want\n");
 855                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 856         }
 857         if (server_supports("allow-reachable-sha1-in-want")) {
 858                 if (args->verbose)
 859                         fprintf(stderr, "Server supports allow-reachable-sha1-in-want\n");
 860                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 861         }
 862         if (!server_supports("thin-pack"))
 863                 args->use_thin_pack = 0;
 864         if (!server_supports("no-progress"))
 865                 args->no_progress = 0;
 866         if (!server_supports("include-tag"))
 867                 args->include_tag = 0;
 868         if (server_supports("ofs-delta")) {
 869                 if (args->verbose)
 870                         fprintf(stderr, "Server supports ofs-delta\n");
 871         } else
 872                 prefer_ofs_delta = 0;
 873
 874         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 875                 agent_supported = 1;
 876                 if (args->verbose && agent_len)
 877                         fprintf(stderr, "Server version is %.*s\n",
 878                                 agent_len, agent_feature);
 879         }
 880
 881         if (everything_local(args, &ref, sought, nr_sought)) {
 882                 packet_flush(fd[1]);
 883                 goto all_done;
 884         }
 885         if (find_common(args, fd, sha1, ref) < 0)
 886                 if (!args->keep_pack)
 887                         /* When cloning, it is not unusual to have
 888                          * no common commit.
 889                          */
 890                         warning("no common commits");
 891
 892         if (args->stateless_rpc)
 893                 packet_flush(fd[1]);
 894         if (args->depth > 0)
 895                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 896                                         NULL);
 897         else if (si->nr_ours || si->nr_theirs)
 898                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 899         else
 900                 alternate_shallow_file = NULL;
 901         if (get_pack(args, fd, pack_lockfile))
 902                 die("git fetch-pack: fetch failed.");
 903
 904  all_done:
 905         return ref;
 906 }
 907
 908 static void fetch_pack_config(void)
 909 {
 910         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 911         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 912         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 913         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 914         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 915
 916         git_config(git_default_config, NULL);
 917 }
 918
 919 static void fetch_pack_setup(void)
 920 {
 921         static int did_setup;
 922         if (did_setup)
 923                 return;
 924         fetch_pack_config();
 925         if (0 <= transfer_unpack_limit)
 926                 unpack_limit = transfer_unpack_limit;
 927         else if (0 <= fetch_unpack_limit)
 928                 unpack_limit = fetch_unpack_limit;
 929         did_setup = 1;
 930 }
 931
 932 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 933 {
 934         struct string_list names = STRING_LIST_INIT_NODUP;
 935         int src, dst;
 936
 937         for (src = dst = 0; src < nr; src++) {
 938                 struct string_list_item *item;
 939                 item = string_list_insert(&names, ref[src]->name);
 940                 if (item->util)
 941                         continue; /* already have it */
 942                 item->util = ref[src];
 943                 if (src != dst)
 944                         ref[dst] = ref[src];
 945                 dst++;
 946         }
 947         for (src = dst; src < nr; src++)
 948                 ref[src] = NULL;
 949         string_list_clear(&names, 0);
 950         return dst;
 951 }
 952
 953 static void update_shallow(struct fetch_pack_args *args,
 954                            struct ref **sought, int nr_sought,
 955                            struct shallow_info *si)
 956 {
 957         struct sha1_array ref = SHA1_ARRAY_INIT;
 958         int *status;
 959         int i;
 960
 961         if (args->depth > 0 && alternate_shallow_file) {
 962                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 963                         unlink_or_warn(git_path_shallow());
 964                         rollback_lock_file(&shallow_lock);
 965                 } else
 966                         commit_lock_file(&shallow_lock);
 967                 return;
 968         }
 969
 970         if (!si->shallow || !si->shallow->nr)
 971                 return;
 972
 973         if (args->cloning) {
 974                 /*
 975                  * remote is shallow, but this is a clone, there are
 976                  * no objects in repo to worry about. Accept any
 977                  * shallow points that exist in the pack (iow in repo
 978                  * after get_pack() and reprepare_packed_git())
 979                  */
 980                 struct sha1_array extra = SHA1_ARRAY_INIT;
 981                 unsigned char (*sha1)[20] = si->shallow->sha1;
 982                 for (i = 0; i < si->shallow->nr; i++)
 983                         if (has_sha1_file(sha1[i]))
 984                                 sha1_array_append(&extra, sha1[i]);
 985                 if (extra.nr) {
 986                         setup_alternate_shallow(&shallow_lock,
 987                                                 &alternate_shallow_file,
 988                                                 &extra);
 989                         commit_lock_file(&shallow_lock);
 990                 }
 991                 sha1_array_clear(&extra);
 992                 return;
 993         }
 994
 995         if (!si->nr_ours && !si->nr_theirs)
 996                 return;
 997
 998         remove_nonexistent_theirs_shallow(si);
 999         if (!si->nr_ours && !si->nr_theirs)
1000                 return;
1001         for (i = 0; i < nr_sought; i++)
1002                 sha1_array_append(&ref, sought[i]->old_oid.hash);
1003         si->ref = &ref;
1004
1005         if (args->update_shallow) {
1006                 /*
1007                  * remote is also shallow, .git/shallow may be updated
1008                  * so all refs can be accepted. Make sure we only add
1009                  * shallow roots that are actually reachable from new
1010                  * refs.
1011                  */
1012                 struct sha1_array extra = SHA1_ARRAY_INIT;
1013                 unsigned char (*sha1)[20] = si->shallow->sha1;
1014                 assign_shallow_commits_to_refs(si, NULL, NULL);
1015                 if (!si->nr_ours && !si->nr_theirs) {
1016                         sha1_array_clear(&ref);
1017                         return;
1018                 }
1019                 for (i = 0; i < si->nr_ours; i++)
1020                         sha1_array_append(&extra, sha1[si->ours[i]]);
1021                 for (i = 0; i < si->nr_theirs; i++)
1022                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1023                 setup_alternate_shallow(&shallow_lock,
1024                                         &alternate_shallow_file,
1025                                         &extra);
1026                 commit_lock_file(&shallow_lock);
1027                 sha1_array_clear(&extra);
1028                 sha1_array_clear(&ref);
1029                 return;
1030         }
1031
1032         /*
1033          * remote is also shallow, check what ref is safe to update
1034          * without updating .git/shallow
1035          */
1036         status = xcalloc(nr_sought, sizeof(*status));
1037         assign_shallow_commits_to_refs(si, NULL, status);
1038         if (si->nr_ours || si->nr_theirs) {
1039                 for (i = 0; i < nr_sought; i++)
1040                         if (status[i])
1041                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1042         }
1043         free(status);
1044         sha1_array_clear(&ref);
1045 }
1046
1047 struct ref *fetch_pack(struct fetch_pack_args *args,
1048                        int fd[], struct child_process *conn,
1049                        const struct ref *ref,
1050                        const char *dest,
1051                        struct ref **sought, int nr_sought,
1052                        struct sha1_array *shallow,
1053                        char **pack_lockfile)
1054 {
1055         struct ref *ref_cpy;
1056         struct shallow_info si;
1057
1058         fetch_pack_setup();
1059         if (nr_sought)
1060                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1061
1062         if (!ref) {
1063                 packet_flush(fd[1]);
1064                 die("no matching remote head");
1065         }
1066         prepare_shallow_info(&si, shallow);
1067         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1068                                 &si, pack_lockfile);
1069         reprepare_packed_git();
1070         update_shallow(args, sought, nr_sought, &si);
1071         clear_shallow_info(&si);
1072         return ref_cpy;
1073 }