git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int deepen_since_ok;
  25 static int deepen_not_ok;
  26 static int fetch_fsck_objects = -1;
  27 static int transfer_fsck_objects = -1;
  28 static int agent_supported;
  29 static struct lock_file shallow_lock;
  30 static const char *alternate_shallow_file;
  31
  32 /* Remember to update object flag allocation in object.h */
  33 #define COMPLETE        (1U << 0)
  34 #define COMMON          (1U << 1)
  35 #define COMMON_REF      (1U << 2)
  36 #define SEEN            (1U << 3)
  37 #define POPPED          (1U << 4)
  38
  39 static int marked;
  40
  41 /*
  42  * After sending this many "have"s if we do not get any new ACK , we
  43  * give up traversing our history.
  44  */
  45 #define MAX_IN_VAIN 256
  46
  47 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  48 static int non_common_revs, multi_ack, use_sideband;
  49 /* Allow specifying sha1 if it is a ref tip. */
  50 #define ALLOW_TIP_SHA1  01
  51 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  52 #define ALLOW_REACHABLE_SHA1    02
  53 static unsigned int allow_unadvertised_object_request;
  54
  55 __attribute__((format (printf, 2, 3)))
  56 static inline void print_verbose(const struct fetch_pack_args *args,
  57                                  const char *fmt, ...)
  58 {
  59         va_list params;
  60
  61         if (!args->verbose)
  62                 return;
  63
  64         va_start(params, fmt);
  65         vfprintf(stderr, fmt, params);
  66         va_end(params);
  67         fputc('\n', stderr);
  68 }
  69
  70 static void rev_list_push(struct commit *commit, int mark)
  71 {
  72         if (!(commit->object.flags & mark)) {
  73                 commit->object.flags |= mark;
  74
  75                 if (parse_commit(commit))
  76                         return;
  77
  78                 prio_queue_put(&rev_list, commit);
  79
  80                 if (!(commit->object.flags & COMMON))
  81                         non_common_revs++;
  82         }
  83 }
  84
  85 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  86 {
  87         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  88
  89         if (o && o->type == OBJ_COMMIT)
  90                 rev_list_push((struct commit *)o, SEEN);
  91
  92         return 0;
  93 }
  94
  95 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  96                                    int flag, void *cb_data)
  97 {
  98         return rev_list_insert_ref(refname, oid->hash);
  99 }
 100
 101 static int clear_marks(const char *refname, const struct object_id *oid,
 102                        int flag, void *cb_data)
 103 {
 104         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
 105
 106         if (o && o->type == OBJ_COMMIT)
 107                 clear_commit_marks((struct commit *)o,
 108                                    COMMON | COMMON_REF | SEEN | POPPED);
 109         return 0;
 110 }
 111
 112 /*
 113    This function marks a rev and its ancestors as common.
 114    In some cases, it is desirable to mark only the ancestors (for example
 115    when only the server does not yet know that they are common).
 116 */
 117
 118 static void mark_common(struct commit *commit,
 119                 int ancestors_only, int dont_parse)
 120 {
 121         if (commit != NULL && !(commit->object.flags & COMMON)) {
 122                 struct object *o = (struct object *)commit;
 123
 124                 if (!ancestors_only)
 125                         o->flags |= COMMON;
 126
 127                 if (!(o->flags & SEEN))
 128                         rev_list_push(commit, SEEN);
 129                 else {
 130                         struct commit_list *parents;
 131
 132                         if (!ancestors_only && !(o->flags & POPPED))
 133                                 non_common_revs--;
 134                         if (!o->parsed && !dont_parse)
 135                                 if (parse_commit(commit))
 136                                         return;
 137
 138                         for (parents = commit->parents;
 139                                         parents;
 140                                         parents = parents->next)
 141                                 mark_common(parents->item, 0, dont_parse);
 142                 }
 143         }
 144 }
 145
 146 /*
 147   Get the next rev to send, ignoring the common.
 148 */
 149
 150 static const unsigned char *get_rev(void)
 151 {
 152         struct commit *commit = NULL;
 153
 154         while (commit == NULL) {
 155                 unsigned int mark;
 156                 struct commit_list *parents;
 157
 158                 if (rev_list.nr == 0 || non_common_revs == 0)
 159                         return NULL;
 160
 161                 commit = prio_queue_get(&rev_list);
 162                 parse_commit(commit);
 163                 parents = commit->parents;
 164
 165                 commit->object.flags |= POPPED;
 166                 if (!(commit->object.flags & COMMON))
 167                         non_common_revs--;
 168
 169                 if (commit->object.flags & COMMON) {
 170                         /* do not send "have", and ignore ancestors */
 171                         commit = NULL;
 172                         mark = COMMON | SEEN;
 173                 } else if (commit->object.flags & COMMON_REF)
 174                         /* send "have", and ignore ancestors */
 175                         mark = COMMON | SEEN;
 176                 else
 177                         /* send "have", also for its ancestors */
 178                         mark = SEEN;
 179
 180                 while (parents) {
 181                         if (!(parents->item->object.flags & SEEN))
 182                                 rev_list_push(parents->item, mark);
 183                         if (mark & COMMON)
 184                                 mark_common(parents->item, 1, 0);
 185                         parents = parents->next;
 186                 }
 187         }
 188
 189         return commit->object.oid.hash;
 190 }
 191
 192 enum ack_type {
 193         NAK = 0,
 194         ACK,
 195         ACK_continue,
 196         ACK_common,
 197         ACK_ready
 198 };
 199
 200 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 201 {
 202         if (args->stateless_rpc && args->deepen) {
 203                 /* If we sent a depth we will get back "duplicate"
 204                  * shallow and unshallow commands every time there
 205                  * is a block of have lines exchanged.
 206                  */
 207                 char *line;
 208                 while ((line = packet_read_line(fd, NULL))) {
 209                         if (starts_with(line, "shallow "))
 210                                 continue;
 211                         if (starts_with(line, "unshallow "))
 212                                 continue;
 213                         die(_("git fetch-pack: expected shallow list"));
 214                 }
 215         }
 216 }
 217
 218 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 219 {
 220         int len;
 221         char *line = packet_read_line(fd, &len);
 222         const char *arg;
 223
 224         if (!len)
 225                 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
 226         if (!strcmp(line, "NAK"))
 227                 return NAK;
 228         if (skip_prefix(line, "ACK ", &arg)) {
 229                 if (!get_sha1_hex(arg, result_sha1)) {
 230                         arg += 40;
 231                         len -= arg - line;
 232                         if (len < 1)
 233                                 return ACK;
 234                         if (strstr(arg, "continue"))
 235                                 return ACK_continue;
 236                         if (strstr(arg, "common"))
 237                                 return ACK_common;
 238                         if (strstr(arg, "ready"))
 239                                 return ACK_ready;
 240                         return ACK;
 241                 }
 242         }
 243         die(_("git fetch_pack: expected ACK/NAK, got '%s'"), line);
 244 }
 245
 246 static void send_request(struct fetch_pack_args *args,
 247                          int fd, struct strbuf *buf)
 248 {
 249         if (args->stateless_rpc) {
 250                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 251                 packet_flush(fd);
 252         } else
 253                 write_or_die(fd, buf->buf, buf->len);
 254 }
 255
 256 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 257 {
 258         rev_list_insert_ref(NULL, ref->old_oid.hash);
 259 }
 260
 261 #define INITIAL_FLUSH 16
 262 #define PIPESAFE_FLUSH 32
 263 #define LARGE_FLUSH 1024
 264
 265 static int next_flush(struct fetch_pack_args *args, int count)
 266 {
 267         int flush_limit = args->stateless_rpc ? LARGE_FLUSH : PIPESAFE_FLUSH;
 268
 269         if (count < flush_limit)
 270                 count <<= 1;
 271         else
 272                 count += flush_limit;
 273         return count;
 274 }
 275
 276 static int find_common(struct fetch_pack_args *args,
 277                        int fd[2], unsigned char *result_sha1,
 278                        struct ref *refs)
 279 {
 280         int fetching;
 281         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 282         const unsigned char *sha1;
 283         unsigned in_vain = 0;
 284         int got_continue = 0;
 285         int got_ready = 0;
 286         struct strbuf req_buf = STRBUF_INIT;
 287         size_t state_len = 0;
 288
 289         if (args->stateless_rpc && multi_ack == 1)
 290                 die(_("--stateless-rpc requires multi_ack_detailed"));
 291         if (marked)
 292                 for_each_ref(clear_marks, NULL);
 293         marked = 1;
 294
 295         for_each_ref(rev_list_insert_ref_oid, NULL);
 296         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 297
 298         fetching = 0;
 299         for ( ; refs ; refs = refs->next) {
 300                 unsigned char *remote = refs->old_oid.hash;
 301                 const char *remote_hex;
 302                 struct object *o;
 303
 304                 /*
 305                  * If that object is complete (i.e. it is an ancestor of a
 306                  * local ref), we tell them we have it but do not have to
 307                  * tell them about its ancestors, which they already know
 308                  * about.
 309                  *
 310                  * We use lookup_object here because we are only
 311                  * interested in the case we *know* the object is
 312                  * reachable and we have already scanned it.
 313                  */
 314                 if (((o = lookup_object(remote)) != NULL) &&
 315                                 (o->flags & COMPLETE)) {
 316                         continue;
 317                 }
 318
 319                 remote_hex = sha1_to_hex(remote);
 320                 if (!fetching) {
 321                         struct strbuf c = STRBUF_INIT;
 322                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 323                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 324                         if (no_done)            strbuf_addstr(&c, " no-done");
 325                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 326                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 327                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 328                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 329                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 330                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 331                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 332                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 333                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 334                                                             git_user_agent_sanitized());
 335                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 336                         strbuf_release(&c);
 337                 } else
 338                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 339                 fetching++;
 340         }
 341
 342         if (!fetching) {
 343                 strbuf_release(&req_buf);
 344                 packet_flush(fd[1]);
 345                 return 1;
 346         }
 347
 348         if (is_repository_shallow())
 349                 write_shallow_commits(&req_buf, 1, NULL);
 350         if (args->depth > 0)
 351                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 352         if (args->deepen_since) {
 353                 unsigned long max_age = approxidate(args->deepen_since);
 354                 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
 355         }
 356         if (args->deepen_not) {
 357                 int i;
 358                 for (i = 0; i < args->deepen_not->nr; i++) {
 359                         struct string_list_item *s = args->deepen_not->items + i;
 360                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 361                 }
 362         }
 363         packet_buf_flush(&req_buf);
 364         state_len = req_buf.len;
 365
 366         if (args->deepen) {
 367                 char *line;
 368                 const char *arg;
 369                 unsigned char sha1[20];
 370
 371                 send_request(args, fd[1], &req_buf);
 372                 while ((line = packet_read_line(fd[0], NULL))) {
 373                         if (skip_prefix(line, "shallow ", &arg)) {
 374                                 if (get_sha1_hex(arg, sha1))
 375                                         die(_("invalid shallow line: %s"), line);
 376                                 register_shallow(sha1);
 377                                 continue;
 378                         }
 379                         if (skip_prefix(line, "unshallow ", &arg)) {
 380                                 if (get_sha1_hex(arg, sha1))
 381                                         die(_("invalid unshallow line: %s"), line);
 382                                 if (!lookup_object(sha1))
 383                                         die(_("object not found: %s"), line);
 384                                 /* make sure that it is parsed as shallow */
 385                                 if (!parse_object(sha1))
 386                                         die(_("error in object: %s"), line);
 387                                 if (unregister_shallow(sha1))
 388                                         die(_("no shallow found: %s"), line);
 389                                 continue;
 390                         }
 391                         die(_("expected shallow/unshallow, got %s"), line);
 392                 }
 393         } else if (!args->stateless_rpc)
 394                 send_request(args, fd[1], &req_buf);
 395
 396         if (!args->stateless_rpc) {
 397                 /* If we aren't using the stateless-rpc interface
 398                  * we don't need to retain the headers.
 399                  */
 400                 strbuf_setlen(&req_buf, 0);
 401                 state_len = 0;
 402         }
 403
 404         flushes = 0;
 405         retval = -1;
 406         while ((sha1 = get_rev())) {
 407                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 408                 print_verbose(args, "have %s", sha1_to_hex(sha1));
 409                 in_vain++;
 410                 if (flush_at <= ++count) {
 411                         int ack;
 412
 413                         packet_buf_flush(&req_buf);
 414                         send_request(args, fd[1], &req_buf);
 415                         strbuf_setlen(&req_buf, state_len);
 416                         flushes++;
 417                         flush_at = next_flush(args, count);
 418
 419                         /*
 420                          * We keep one window "ahead" of the other side, and
 421                          * will wait for an ACK only on the next one
 422                          */
 423                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 424                                 continue;
 425
 426                         consume_shallow_list(args, fd[0]);
 427                         do {
 428                                 ack = get_ack(fd[0], result_sha1);
 429                                 if (ack)
 430                                         print_verbose(args, _("got %s %d %s"), "ack",
 431                                                       ack, sha1_to_hex(result_sha1));
 432                                 switch (ack) {
 433                                 case ACK:
 434                                         flushes = 0;
 435                                         multi_ack = 0;
 436                                         retval = 0;
 437                                         goto done;
 438                                 case ACK_common:
 439                                 case ACK_ready:
 440                                 case ACK_continue: {
 441                                         struct commit *commit =
 442                                                 lookup_commit(result_sha1);
 443                                         if (!commit)
 444                                                 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
 445                                         if (args->stateless_rpc
 446                                          && ack == ACK_common
 447                                          && !(commit->object.flags & COMMON)) {
 448                                                 /* We need to replay the have for this object
 449                                                  * on the next RPC request so the peer knows
 450                                                  * it is in common with us.
 451                                                  */
 452                                                 const char *hex = sha1_to_hex(result_sha1);
 453                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 454                                                 state_len = req_buf.len;
 455                                         }
 456                                         mark_common(commit, 0, 1);
 457                                         retval = 0;
 458                                         in_vain = 0;
 459                                         got_continue = 1;
 460                                         if (ack == ACK_ready) {
 461                                                 clear_prio_queue(&rev_list);
 462                                                 got_ready = 1;
 463                                         }
 464                                         break;
 465                                         }
 466                                 }
 467                         } while (ack);
 468                         flushes--;
 469                         if (got_continue && MAX_IN_VAIN < in_vain) {
 470                                 print_verbose(args, _("giving up"));
 471                                 break; /* give up */
 472                         }
 473                 }
 474         }
 475 done:
 476         if (!got_ready || !no_done) {
 477                 packet_buf_write(&req_buf, "done\n");
 478                 send_request(args, fd[1], &req_buf);
 479         }
 480         print_verbose(args, _("done"));
 481         if (retval != 0) {
 482                 multi_ack = 0;
 483                 flushes++;
 484         }
 485         strbuf_release(&req_buf);
 486
 487         if (!got_ready || !no_done)
 488                 consume_shallow_list(args, fd[0]);
 489         while (flushes || multi_ack) {
 490                 int ack = get_ack(fd[0], result_sha1);
 491                 if (ack) {
 492                         print_verbose(args, _("got %s (%d) %s"), "ack",
 493                                       ack, sha1_to_hex(result_sha1));
 494                         if (ack == ACK)
 495                                 return 0;
 496                         multi_ack = 1;
 497                         continue;
 498                 }
 499                 flushes--;
 500         }
 501         /* it is no error to fetch into a completely empty repo */
 502         return count ? retval : 0;
 503 }
 504
 505 static struct commit_list *complete;
 506
 507 static int mark_complete(const unsigned char *sha1)
 508 {
 509         struct object *o = parse_object(sha1);
 510
 511         while (o && o->type == OBJ_TAG) {
 512                 struct tag *t = (struct tag *) o;
 513                 if (!t->tagged)
 514                         break; /* broken repository */
 515                 o->flags |= COMPLETE;
 516                 o = parse_object(t->tagged->oid.hash);
 517         }
 518         if (o && o->type == OBJ_COMMIT) {
 519                 struct commit *commit = (struct commit *)o;
 520                 if (!(commit->object.flags & COMPLETE)) {
 521                         commit->object.flags |= COMPLETE;
 522                         commit_list_insert(commit, &complete);
 523                 }
 524         }
 525         return 0;
 526 }
 527
 528 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 529                              int flag, void *cb_data)
 530 {
 531         return mark_complete(oid->hash);
 532 }
 533
 534 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 535                                          unsigned long cutoff)
 536 {
 537         while (complete && cutoff <= complete->item->date) {
 538                 print_verbose(args, _("Marking %s as complete"),
 539                               oid_to_hex(&complete->item->object.oid));
 540                 pop_most_recent_commit(&complete, COMPLETE);
 541         }
 542 }
 543
 544 static void filter_refs(struct fetch_pack_args *args,
 545                         struct ref **refs,
 546                         struct ref **sought, int nr_sought)
 547 {
 548         struct ref *newlist = NULL;
 549         struct ref **newtail = &newlist;
 550         struct ref *ref, *next;
 551         int i;
 552
 553         i = 0;
 554         for (ref = *refs; ref; ref = next) {
 555                 int keep = 0;
 556                 next = ref->next;
 557
 558                 if (starts_with(ref->name, "refs/") &&
 559                     check_refname_format(ref->name, 0))
 560                         ; /* trash */
 561                 else {
 562                         while (i < nr_sought) {
 563                                 int cmp = strcmp(ref->name, sought[i]->name);
 564                                 if (cmp < 0)
 565                                         break; /* definitely do not have it */
 566                                 else if (cmp == 0) {
 567                                         keep = 1; /* definitely have it */
 568                                         sought[i]->matched = 1;
 569                                 }
 570                                 i++;
 571                         }
 572                 }
 573
 574                 if (!keep && args->fetch_all &&
 575                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 576                         keep = 1;
 577
 578                 if (keep) {
 579                         *newtail = ref;
 580                         ref->next = NULL;
 581                         newtail = &ref->next;
 582                 } else {
 583                         free(ref);
 584                 }
 585         }
 586
 587         /* Append unmatched requests to the list */
 588         if ((allow_unadvertised_object_request &
 589             (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 590                 for (i = 0; i < nr_sought; i++) {
 591                         unsigned char sha1[20];
 592
 593                         ref = sought[i];
 594                         if (ref->matched)
 595                                 continue;
 596                         if (get_sha1_hex(ref->name, sha1) ||
 597                             ref->name[40] != '\0' ||
 598                             hashcmp(sha1, ref->old_oid.hash))
 599                                 continue;
 600
 601                         ref->matched = 1;
 602                         *newtail = copy_ref(ref);
 603                         newtail = &(*newtail)->next;
 604                 }
 605         }
 606         *refs = newlist;
 607 }
 608
 609 static void mark_alternate_complete(const struct ref *ref, void *unused)
 610 {
 611         mark_complete(ref->old_oid.hash);
 612 }
 613
 614 static int everything_local(struct fetch_pack_args *args,
 615                             struct ref **refs,
 616                             struct ref **sought, int nr_sought)
 617 {
 618         struct ref *ref;
 619         int retval;
 620         unsigned long cutoff = 0;
 621
 622         save_commit_buffer = 0;
 623
 624         for (ref = *refs; ref; ref = ref->next) {
 625                 struct object *o;
 626
 627                 if (!has_object_file(&ref->old_oid))
 628                         continue;
 629
 630                 o = parse_object(ref->old_oid.hash);
 631                 if (!o)
 632                         continue;
 633
 634                 /* We already have it -- which may mean that we were
 635                  * in sync with the other side at some time after
 636                  * that (it is OK if we guess wrong here).
 637                  */
 638                 if (o->type == OBJ_COMMIT) {
 639                         struct commit *commit = (struct commit *)o;
 640                         if (!cutoff || cutoff < commit->date)
 641                                 cutoff = commit->date;
 642                 }
 643         }
 644
 645         if (!args->deepen) {
 646                 for_each_ref(mark_complete_oid, NULL);
 647                 for_each_alternate_ref(mark_alternate_complete, NULL);
 648                 commit_list_sort_by_date(&complete);
 649                 if (cutoff)
 650                         mark_recent_complete_commits(args, cutoff);
 651         }
 652
 653         /*
 654          * Mark all complete remote refs as common refs.
 655          * Don't mark them common yet; the server has to be told so first.
 656          */
 657         for (ref = *refs; ref; ref = ref->next) {
 658                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 659                                              NULL, 0);
 660
 661                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 662                         continue;
 663
 664                 if (!(o->flags & SEEN)) {
 665                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 666
 667                         mark_common((struct commit *)o, 1, 1);
 668                 }
 669         }
 670
 671         filter_refs(args, refs, sought, nr_sought);
 672
 673         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 674                 const unsigned char *remote = ref->old_oid.hash;
 675                 struct object *o;
 676
 677                 o = lookup_object(remote);
 678                 if (!o || !(o->flags & COMPLETE)) {
 679                         retval = 0;
 680                         print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
 681                                       ref->name);
 682                         continue;
 683                 }
 684                 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
 685                               ref->name);
 686         }
 687         return retval;
 688 }
 689
 690 static int sideband_demux(int in, int out, void *data)
 691 {
 692         int *xd = data;
 693
 694         int ret = recv_sideband("fetch-pack", xd[0], out);
 695         close(out);
 696         return ret;
 697 }
 698
 699 static int get_pack(struct fetch_pack_args *args,
 700                     int xd[2], char **pack_lockfile)
 701 {
 702         struct async demux;
 703         int do_keep = args->keep_pack;
 704         const char *cmd_name;
 705         struct pack_header header;
 706         int pass_header = 0;
 707         struct child_process cmd = CHILD_PROCESS_INIT;
 708         int ret;
 709
 710         memset(&demux, 0, sizeof(demux));
 711         if (use_sideband) {
 712                 /* xd[] is talking with upload-pack; subprocess reads from
 713                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 714                  * through demux->out.
 715                  */
 716                 demux.proc = sideband_demux;
 717                 demux.data = xd;
 718                 demux.out = -1;
 719                 if (start_async(&demux))
 720                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 721         }
 722         else
 723                 demux.out = xd[0];
 724
 725         if (!args->keep_pack && unpack_limit) {
 726
 727                 if (read_pack_header(demux.out, &header))
 728                         die(_("protocol error: bad pack header"));
 729                 pass_header = 1;
 730                 if (ntohl(header.hdr_entries) < unpack_limit)
 731                         do_keep = 0;
 732                 else
 733                         do_keep = 1;
 734         }
 735
 736         if (alternate_shallow_file) {
 737                 argv_array_push(&cmd.args, "--shallow-file");
 738                 argv_array_push(&cmd.args, alternate_shallow_file);
 739         }
 740
 741         if (do_keep) {
 742                 if (pack_lockfile)
 743                         cmd.out = -1;
 744                 cmd_name = "index-pack";
 745                 argv_array_push(&cmd.args, cmd_name);
 746                 argv_array_push(&cmd.args, "--stdin");
 747                 if (!args->quiet && !args->no_progress)
 748                         argv_array_push(&cmd.args, "-v");
 749                 if (args->use_thin_pack)
 750                         argv_array_push(&cmd.args, "--fix-thin");
 751                 if (args->lock_pack || unpack_limit) {
 752                         char hostname[256];
 753                         if (gethostname(hostname, sizeof(hostname)))
 754                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 755                         argv_array_pushf(&cmd.args,
 756                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 757                                         (uintmax_t)getpid(), hostname);
 758                 }
 759                 if (args->check_self_contained_and_connected)
 760                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 761         }
 762         else {
 763                 cmd_name = "unpack-objects";
 764                 argv_array_push(&cmd.args, cmd_name);
 765                 if (args->quiet || args->no_progress)
 766                         argv_array_push(&cmd.args, "-q");
 767                 args->check_self_contained_and_connected = 0;
 768         }
 769
 770         if (pass_header)
 771                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 772                                  ntohl(header.hdr_version),
 773                                  ntohl(header.hdr_entries));
 774         if (fetch_fsck_objects >= 0
 775             ? fetch_fsck_objects
 776             : transfer_fsck_objects >= 0
 777             ? transfer_fsck_objects
 778             : 0)
 779                 argv_array_push(&cmd.args, "--strict");
 780
 781         cmd.in = demux.out;
 782         cmd.git_cmd = 1;
 783         if (start_command(&cmd))
 784                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 785         if (do_keep && pack_lockfile) {
 786                 *pack_lockfile = index_pack_lockfile(cmd.out);
 787                 close(cmd.out);
 788         }
 789
 790         if (!use_sideband)
 791                 /* Closed by start_command() */
 792                 xd[0] = -1;
 793
 794         ret = finish_command(&cmd);
 795         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 796                 args->self_contained_and_connected =
 797                         args->check_self_contained_and_connected &&
 798                         ret == 0;
 799         else
 800                 die(_("%s failed"), cmd_name);
 801         if (use_sideband && finish_async(&demux))
 802                 die(_("error in sideband demultiplexer"));
 803         return 0;
 804 }
 805
 806 static int cmp_ref_by_name(const void *a_, const void *b_)
 807 {
 808         const struct ref *a = *((const struct ref **)a_);
 809         const struct ref *b = *((const struct ref **)b_);
 810         return strcmp(a->name, b->name);
 811 }
 812
 813 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 814                                  int fd[2],
 815                                  const struct ref *orig_ref,
 816                                  struct ref **sought, int nr_sought,
 817                                  struct shallow_info *si,
 818                                  char **pack_lockfile)
 819 {
 820         struct ref *ref = copy_ref_list(orig_ref);
 821         unsigned char sha1[20];
 822         const char *agent_feature;
 823         int agent_len;
 824
 825         sort_ref_list(&ref, ref_compare_name);
 826         qsort(sought, nr_sought, sizeof(*sought), cmp_ref_by_name);
 827
 828         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 829                 die(_("Server does not support shallow clients"));
 830         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 831                 args->deepen = 1;
 832         if (server_supports("multi_ack_detailed")) {
 833                 print_verbose(args, _("Server supports multi_ack_detailed"));
 834                 multi_ack = 2;
 835                 if (server_supports("no-done")) {
 836                         print_verbose(args, _("Server supports no-done"));
 837                         if (args->stateless_rpc)
 838                                 no_done = 1;
 839                 }
 840         }
 841         else if (server_supports("multi_ack")) {
 842                 print_verbose(args, _("Server supports multi_ack"));
 843                 multi_ack = 1;
 844         }
 845         if (server_supports("side-band-64k")) {
 846                 print_verbose(args, _("Server supports side-band-64k"));
 847                 use_sideband = 2;
 848         }
 849         else if (server_supports("side-band")) {
 850                 print_verbose(args, _("Server supports side-band"));
 851                 use_sideband = 1;
 852         }
 853         if (server_supports("allow-tip-sha1-in-want")) {
 854                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 855                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 856         }
 857         if (server_supports("allow-reachable-sha1-in-want")) {
 858                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 859                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 860         }
 861         if (!server_supports("thin-pack"))
 862                 args->use_thin_pack = 0;
 863         if (!server_supports("no-progress"))
 864                 args->no_progress = 0;
 865         if (!server_supports("include-tag"))
 866                 args->include_tag = 0;
 867         if (server_supports("ofs-delta"))
 868                 print_verbose(args, _("Server supports ofs-delta"));
 869         else
 870                 prefer_ofs_delta = 0;
 871
 872         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 873                 agent_supported = 1;
 874                 if (agent_len)
 875                         print_verbose(args, _("Server version is %.*s"),
 876                                       agent_len, agent_feature);
 877         }
 878         if (server_supports("deepen-since"))
 879                 deepen_since_ok = 1;
 880         else if (args->deepen_since)
 881                 die(_("Server does not support --shallow-since"));
 882         if (server_supports("deepen-not"))
 883                 deepen_not_ok = 1;
 884         else if (args->deepen_not)
 885                 die(_("Server does not support --shallow-exclude"));
 886
 887         if (everything_local(args, &ref, sought, nr_sought)) {
 888                 packet_flush(fd[1]);
 889                 goto all_done;
 890         }
 891         if (find_common(args, fd, sha1, ref) < 0)
 892                 if (!args->keep_pack)
 893                         /* When cloning, it is not unusual to have
 894                          * no common commit.
 895                          */
 896                         warning(_("no common commits"));
 897
 898         if (args->stateless_rpc)
 899                 packet_flush(fd[1]);
 900         if (args->deepen)
 901                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 902                                         NULL);
 903         else if (si->nr_ours || si->nr_theirs)
 904                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 905         else
 906                 alternate_shallow_file = NULL;
 907         if (get_pack(args, fd, pack_lockfile))
 908                 die(_("git fetch-pack: fetch failed."));
 909
 910  all_done:
 911         return ref;
 912 }
 913
 914 static void fetch_pack_config(void)
 915 {
 916         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 917         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 918         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 919         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 920         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 921
 922         git_config(git_default_config, NULL);
 923 }
 924
 925 static void fetch_pack_setup(void)
 926 {
 927         static int did_setup;
 928         if (did_setup)
 929                 return;
 930         fetch_pack_config();
 931         if (0 <= transfer_unpack_limit)
 932                 unpack_limit = transfer_unpack_limit;
 933         else if (0 <= fetch_unpack_limit)
 934                 unpack_limit = fetch_unpack_limit;
 935         did_setup = 1;
 936 }
 937
 938 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 939 {
 940         struct string_list names = STRING_LIST_INIT_NODUP;
 941         int src, dst;
 942
 943         for (src = dst = 0; src < nr; src++) {
 944                 struct string_list_item *item;
 945                 item = string_list_insert(&names, ref[src]->name);
 946                 if (item->util)
 947                         continue; /* already have it */
 948                 item->util = ref[src];
 949                 if (src != dst)
 950                         ref[dst] = ref[src];
 951                 dst++;
 952         }
 953         for (src = dst; src < nr; src++)
 954                 ref[src] = NULL;
 955         string_list_clear(&names, 0);
 956         return dst;
 957 }
 958
 959 static void update_shallow(struct fetch_pack_args *args,
 960                            struct ref **sought, int nr_sought,
 961                            struct shallow_info *si)
 962 {
 963         struct sha1_array ref = SHA1_ARRAY_INIT;
 964         int *status;
 965         int i;
 966
 967         if (args->deepen && alternate_shallow_file) {
 968                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 969                         unlink_or_warn(git_path_shallow());
 970                         rollback_lock_file(&shallow_lock);
 971                 } else
 972                         commit_lock_file(&shallow_lock);
 973                 return;
 974         }
 975
 976         if (!si->shallow || !si->shallow->nr)
 977                 return;
 978
 979         if (args->cloning) {
 980                 /*
 981                  * remote is shallow, but this is a clone, there are
 982                  * no objects in repo to worry about. Accept any
 983                  * shallow points that exist in the pack (iow in repo
 984                  * after get_pack() and reprepare_packed_git())
 985                  */
 986                 struct sha1_array extra = SHA1_ARRAY_INIT;
 987                 unsigned char (*sha1)[20] = si->shallow->sha1;
 988                 for (i = 0; i < si->shallow->nr; i++)
 989                         if (has_sha1_file(sha1[i]))
 990                                 sha1_array_append(&extra, sha1[i]);
 991                 if (extra.nr) {
 992                         setup_alternate_shallow(&shallow_lock,
 993                                                 &alternate_shallow_file,
 994                                                 &extra);
 995                         commit_lock_file(&shallow_lock);
 996                 }
 997                 sha1_array_clear(&extra);
 998                 return;
 999         }
1000
1001         if (!si->nr_ours && !si->nr_theirs)
1002                 return;
1003
1004         remove_nonexistent_theirs_shallow(si);
1005         if (!si->nr_ours && !si->nr_theirs)
1006                 return;
1007         for (i = 0; i < nr_sought; i++)
1008                 sha1_array_append(&ref, sought[i]->old_oid.hash);
1009         si->ref = &ref;
1010
1011         if (args->update_shallow) {
1012                 /*
1013                  * remote is also shallow, .git/shallow may be updated
1014                  * so all refs can be accepted. Make sure we only add
1015                  * shallow roots that are actually reachable from new
1016                  * refs.
1017                  */
1018                 struct sha1_array extra = SHA1_ARRAY_INIT;
1019                 unsigned char (*sha1)[20] = si->shallow->sha1;
1020                 assign_shallow_commits_to_refs(si, NULL, NULL);
1021                 if (!si->nr_ours && !si->nr_theirs) {
1022                         sha1_array_clear(&ref);
1023                         return;
1024                 }
1025                 for (i = 0; i < si->nr_ours; i++)
1026                         sha1_array_append(&extra, sha1[si->ours[i]]);
1027                 for (i = 0; i < si->nr_theirs; i++)
1028                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1029                 setup_alternate_shallow(&shallow_lock,
1030                                         &alternate_shallow_file,
1031                                         &extra);
1032                 commit_lock_file(&shallow_lock);
1033                 sha1_array_clear(&extra);
1034                 sha1_array_clear(&ref);
1035                 return;
1036         }
1037
1038         /*
1039          * remote is also shallow, check what ref is safe to update
1040          * without updating .git/shallow
1041          */
1042         status = xcalloc(nr_sought, sizeof(*status));
1043         assign_shallow_commits_to_refs(si, NULL, status);
1044         if (si->nr_ours || si->nr_theirs) {
1045                 for (i = 0; i < nr_sought; i++)
1046                         if (status[i])
1047                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1048         }
1049         free(status);
1050         sha1_array_clear(&ref);
1051 }
1052
1053 struct ref *fetch_pack(struct fetch_pack_args *args,
1054                        int fd[], struct child_process *conn,
1055                        const struct ref *ref,
1056                        const char *dest,
1057                        struct ref **sought, int nr_sought,
1058                        struct sha1_array *shallow,
1059                        char **pack_lockfile)
1060 {
1061         struct ref *ref_cpy;
1062         struct shallow_info si;
1063
1064         fetch_pack_setup();
1065         if (nr_sought)
1066                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1067
1068         if (!ref) {
1069                 packet_flush(fd[1]);
1070                 die(_("no matching remote head"));
1071         }
1072         prepare_shallow_info(&si, shallow);
1073         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1074                                 &si, pack_lockfile);
1075         reprepare_packed_git();
1076         update_shallow(args, sought, nr_sought, &si);
1077         clear_shallow_info(&si);
1078         return ref_cpy;
1079 }