git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18
  19 static int transfer_unpack_limit = -1;
  20 static int fetch_unpack_limit = -1;
  21 static int unpack_limit = 100;
  22 static int prefer_ofs_delta = 1;
  23 static int no_done;
  24 static int deepen_since_ok;
  25 static int deepen_not_ok;
  26 static int fetch_fsck_objects = -1;
  27 static int transfer_fsck_objects = -1;
  28 static int agent_supported;
  29 static struct lock_file shallow_lock;
  30 static const char *alternate_shallow_file;
  31
  32 /* Remember to update object flag allocation in object.h */
  33 #define COMPLETE        (1U << 0)
  34 #define COMMON          (1U << 1)
  35 #define COMMON_REF      (1U << 2)
  36 #define SEEN            (1U << 3)
  37 #define POPPED          (1U << 4)
  38
  39 static int marked;
  40
  41 /*
  42  * After sending this many "have"s if we do not get any new ACK , we
  43  * give up traversing our history.
  44  */
  45 #define MAX_IN_VAIN 256
  46
  47 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  48 static int non_common_revs, multi_ack, use_sideband;
  49 /* Allow specifying sha1 if it is a ref tip. */
  50 #define ALLOW_TIP_SHA1  01
  51 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  52 #define ALLOW_REACHABLE_SHA1    02
  53 static unsigned int allow_unadvertised_object_request;
  54
  55 __attribute__((format (printf, 2, 3)))
  56 static inline void print_verbose(const struct fetch_pack_args *args,
  57                                  const char *fmt, ...)
  58 {
  59         va_list params;
  60
  61         if (!args->verbose)
  62                 return;
  63
  64         va_start(params, fmt);
  65         vfprintf(stderr, fmt, params);
  66         va_end(params);
  67         fputc('\n', stderr);
  68 }
  69
  70 static void rev_list_push(struct commit *commit, int mark)
  71 {
  72         if (!(commit->object.flags & mark)) {
  73                 commit->object.flags |= mark;
  74
  75                 if (parse_commit(commit))
  76                         return;
  77
  78                 prio_queue_put(&rev_list, commit);
  79
  80                 if (!(commit->object.flags & COMMON))
  81                         non_common_revs++;
  82         }
  83 }
  84
  85 static int rev_list_insert_ref(const char *refname, const unsigned char *sha1)
  86 {
  87         struct object *o = deref_tag(parse_object(sha1), refname, 0);
  88
  89         if (o && o->type == OBJ_COMMIT)
  90                 rev_list_push((struct commit *)o, SEEN);
  91
  92         return 0;
  93 }
  94
  95 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
  96                                    int flag, void *cb_data)
  97 {
  98         return rev_list_insert_ref(refname, oid->hash);
  99 }
 100
 101 static int clear_marks(const char *refname, const struct object_id *oid,
 102                        int flag, void *cb_data)
 103 {
 104         struct object *o = deref_tag(parse_object(oid->hash), refname, 0);
 105
 106         if (o && o->type == OBJ_COMMIT)
 107                 clear_commit_marks((struct commit *)o,
 108                                    COMMON | COMMON_REF | SEEN | POPPED);
 109         return 0;
 110 }
 111
 112 /*
 113    This function marks a rev and its ancestors as common.
 114    In some cases, it is desirable to mark only the ancestors (for example
 115    when only the server does not yet know that they are common).
 116 */
 117
 118 static void mark_common(struct commit *commit,
 119                 int ancestors_only, int dont_parse)
 120 {
 121         if (commit != NULL && !(commit->object.flags & COMMON)) {
 122                 struct object *o = (struct object *)commit;
 123
 124                 if (!ancestors_only)
 125                         o->flags |= COMMON;
 126
 127                 if (!(o->flags & SEEN))
 128                         rev_list_push(commit, SEEN);
 129                 else {
 130                         struct commit_list *parents;
 131
 132                         if (!ancestors_only && !(o->flags & POPPED))
 133                                 non_common_revs--;
 134                         if (!o->parsed && !dont_parse)
 135                                 if (parse_commit(commit))
 136                                         return;
 137
 138                         for (parents = commit->parents;
 139                                         parents;
 140                                         parents = parents->next)
 141                                 mark_common(parents->item, 0, dont_parse);
 142                 }
 143         }
 144 }
 145
 146 /*
 147   Get the next rev to send, ignoring the common.
 148 */
 149
 150 static const unsigned char *get_rev(void)
 151 {
 152         struct commit *commit = NULL;
 153
 154         while (commit == NULL) {
 155                 unsigned int mark;
 156                 struct commit_list *parents;
 157
 158                 if (rev_list.nr == 0 || non_common_revs == 0)
 159                         return NULL;
 160
 161                 commit = prio_queue_get(&rev_list);
 162                 parse_commit(commit);
 163                 parents = commit->parents;
 164
 165                 commit->object.flags |= POPPED;
 166                 if (!(commit->object.flags & COMMON))
 167                         non_common_revs--;
 168
 169                 if (commit->object.flags & COMMON) {
 170                         /* do not send "have", and ignore ancestors */
 171                         commit = NULL;
 172                         mark = COMMON | SEEN;
 173                 } else if (commit->object.flags & COMMON_REF)
 174                         /* send "have", and ignore ancestors */
 175                         mark = COMMON | SEEN;
 176                 else
 177                         /* send "have", also for its ancestors */
 178                         mark = SEEN;
 179
 180                 while (parents) {
 181                         if (!(parents->item->object.flags & SEEN))
 182                                 rev_list_push(parents->item, mark);
 183                         if (mark & COMMON)
 184                                 mark_common(parents->item, 1, 0);
 185                         parents = parents->next;
 186                 }
 187         }
 188
 189         return commit->object.oid.hash;
 190 }
 191
 192 enum ack_type {
 193         NAK = 0,
 194         ACK,
 195         ACK_continue,
 196         ACK_common,
 197         ACK_ready
 198 };
 199
 200 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 201 {
 202         if (args->stateless_rpc && args->deepen) {
 203                 /* If we sent a depth we will get back "duplicate"
 204                  * shallow and unshallow commands every time there
 205                  * is a block of have lines exchanged.
 206                  */
 207                 char *line;
 208                 while ((line = packet_read_line(fd, NULL))) {
 209                         if (starts_with(line, "shallow "))
 210                                 continue;
 211                         if (starts_with(line, "unshallow "))
 212                                 continue;
 213                         die(_("git fetch-pack: expected shallow list"));
 214                 }
 215         }
 216 }
 217
 218 static enum ack_type get_ack(int fd, unsigned char *result_sha1)
 219 {
 220         int len;
 221         char *line = packet_read_line(fd, &len);
 222         const char *arg;
 223
 224         if (!len)
 225                 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
 226         if (!strcmp(line, "NAK"))
 227                 return NAK;
 228         if (skip_prefix(line, "ACK ", &arg)) {
 229                 if (!get_sha1_hex(arg, result_sha1)) {
 230                         arg += 40;
 231                         len -= arg - line;
 232                         if (len < 1)
 233                                 return ACK;
 234                         if (strstr(arg, "continue"))
 235                                 return ACK_continue;
 236                         if (strstr(arg, "common"))
 237                                 return ACK_common;
 238                         if (strstr(arg, "ready"))
 239                                 return ACK_ready;
 240                         return ACK;
 241                 }
 242         }
 243         if (skip_prefix(line, "ERR ", &arg))
 244                 die(_("remote error: %s"), arg);
 245         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 246 }
 247
 248 static void send_request(struct fetch_pack_args *args,
 249                          int fd, struct strbuf *buf)
 250 {
 251         if (args->stateless_rpc) {
 252                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 253                 packet_flush(fd);
 254         } else
 255                 write_or_die(fd, buf->buf, buf->len);
 256 }
 257
 258 static void insert_one_alternate_ref(const struct ref *ref, void *unused)
 259 {
 260         rev_list_insert_ref(NULL, ref->old_oid.hash);
 261 }
 262
 263 #define INITIAL_FLUSH 16
 264 #define PIPESAFE_FLUSH 32
 265 #define LARGE_FLUSH 16384
 266
 267 static int next_flush(struct fetch_pack_args *args, int count)
 268 {
 269         if (args->stateless_rpc) {
 270                 if (count < LARGE_FLUSH)
 271                         count <<= 1;
 272                 else
 273                         count = count * 11 / 10;
 274         } else {
 275                 if (count < PIPESAFE_FLUSH)
 276                         count <<= 1;
 277                 else
 278                         count += PIPESAFE_FLUSH;
 279         }
 280         return count;
 281 }
 282
 283 static int find_common(struct fetch_pack_args *args,
 284                        int fd[2], unsigned char *result_sha1,
 285                        struct ref *refs)
 286 {
 287         int fetching;
 288         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 289         const unsigned char *sha1;
 290         unsigned in_vain = 0;
 291         int got_continue = 0;
 292         int got_ready = 0;
 293         struct strbuf req_buf = STRBUF_INIT;
 294         size_t state_len = 0;
 295
 296         if (args->stateless_rpc && multi_ack == 1)
 297                 die(_("--stateless-rpc requires multi_ack_detailed"));
 298         if (marked)
 299                 for_each_ref(clear_marks, NULL);
 300         marked = 1;
 301
 302         for_each_ref(rev_list_insert_ref_oid, NULL);
 303         for_each_alternate_ref(insert_one_alternate_ref, NULL);
 304
 305         fetching = 0;
 306         for ( ; refs ; refs = refs->next) {
 307                 unsigned char *remote = refs->old_oid.hash;
 308                 const char *remote_hex;
 309                 struct object *o;
 310
 311                 /*
 312                  * If that object is complete (i.e. it is an ancestor of a
 313                  * local ref), we tell them we have it but do not have to
 314                  * tell them about its ancestors, which they already know
 315                  * about.
 316                  *
 317                  * We use lookup_object here because we are only
 318                  * interested in the case we *know* the object is
 319                  * reachable and we have already scanned it.
 320                  */
 321                 if (((o = lookup_object(remote)) != NULL) &&
 322                                 (o->flags & COMPLETE)) {
 323                         continue;
 324                 }
 325
 326                 remote_hex = sha1_to_hex(remote);
 327                 if (!fetching) {
 328                         struct strbuf c = STRBUF_INIT;
 329                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 330                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 331                         if (no_done)            strbuf_addstr(&c, " no-done");
 332                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 333                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 334                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 335                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 336                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 337                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 338                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 339                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 340                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 341                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 342                                                             git_user_agent_sanitized());
 343                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 344                         strbuf_release(&c);
 345                 } else
 346                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 347                 fetching++;
 348         }
 349
 350         if (!fetching) {
 351                 strbuf_release(&req_buf);
 352                 packet_flush(fd[1]);
 353                 return 1;
 354         }
 355
 356         if (is_repository_shallow())
 357                 write_shallow_commits(&req_buf, 1, NULL);
 358         if (args->depth > 0)
 359                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 360         if (args->deepen_since) {
 361                 unsigned long max_age = approxidate(args->deepen_since);
 362                 packet_buf_write(&req_buf, "deepen-since %lu", max_age);
 363         }
 364         if (args->deepen_not) {
 365                 int i;
 366                 for (i = 0; i < args->deepen_not->nr; i++) {
 367                         struct string_list_item *s = args->deepen_not->items + i;
 368                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 369                 }
 370         }
 371         packet_buf_flush(&req_buf);
 372         state_len = req_buf.len;
 373
 374         if (args->deepen) {
 375                 char *line;
 376                 const char *arg;
 377                 unsigned char sha1[20];
 378
 379                 send_request(args, fd[1], &req_buf);
 380                 while ((line = packet_read_line(fd[0], NULL))) {
 381                         if (skip_prefix(line, "shallow ", &arg)) {
 382                                 if (get_sha1_hex(arg, sha1))
 383                                         die(_("invalid shallow line: %s"), line);
 384                                 register_shallow(sha1);
 385                                 continue;
 386                         }
 387                         if (skip_prefix(line, "unshallow ", &arg)) {
 388                                 if (get_sha1_hex(arg, sha1))
 389                                         die(_("invalid unshallow line: %s"), line);
 390                                 if (!lookup_object(sha1))
 391                                         die(_("object not found: %s"), line);
 392                                 /* make sure that it is parsed as shallow */
 393                                 if (!parse_object(sha1))
 394                                         die(_("error in object: %s"), line);
 395                                 if (unregister_shallow(sha1))
 396                                         die(_("no shallow found: %s"), line);
 397                                 continue;
 398                         }
 399                         die(_("expected shallow/unshallow, got %s"), line);
 400                 }
 401         } else if (!args->stateless_rpc)
 402                 send_request(args, fd[1], &req_buf);
 403
 404         if (!args->stateless_rpc) {
 405                 /* If we aren't using the stateless-rpc interface
 406                  * we don't need to retain the headers.
 407                  */
 408                 strbuf_setlen(&req_buf, 0);
 409                 state_len = 0;
 410         }
 411
 412         flushes = 0;
 413         retval = -1;
 414         while ((sha1 = get_rev())) {
 415                 packet_buf_write(&req_buf, "have %s\n", sha1_to_hex(sha1));
 416                 print_verbose(args, "have %s", sha1_to_hex(sha1));
 417                 in_vain++;
 418                 if (flush_at <= ++count) {
 419                         int ack;
 420
 421                         packet_buf_flush(&req_buf);
 422                         send_request(args, fd[1], &req_buf);
 423                         strbuf_setlen(&req_buf, state_len);
 424                         flushes++;
 425                         flush_at = next_flush(args, count);
 426
 427                         /*
 428                          * We keep one window "ahead" of the other side, and
 429                          * will wait for an ACK only on the next one
 430                          */
 431                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 432                                 continue;
 433
 434                         consume_shallow_list(args, fd[0]);
 435                         do {
 436                                 ack = get_ack(fd[0], result_sha1);
 437                                 if (ack)
 438                                         print_verbose(args, _("got %s %d %s"), "ack",
 439                                                       ack, sha1_to_hex(result_sha1));
 440                                 switch (ack) {
 441                                 case ACK:
 442                                         flushes = 0;
 443                                         multi_ack = 0;
 444                                         retval = 0;
 445                                         goto done;
 446                                 case ACK_common:
 447                                 case ACK_ready:
 448                                 case ACK_continue: {
 449                                         struct commit *commit =
 450                                                 lookup_commit(result_sha1);
 451                                         if (!commit)
 452                                                 die(_("invalid commit %s"), sha1_to_hex(result_sha1));
 453                                         if (args->stateless_rpc
 454                                          && ack == ACK_common
 455                                          && !(commit->object.flags & COMMON)) {
 456                                                 /* We need to replay the have for this object
 457                                                  * on the next RPC request so the peer knows
 458                                                  * it is in common with us.
 459                                                  */
 460                                                 const char *hex = sha1_to_hex(result_sha1);
 461                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 462                                                 state_len = req_buf.len;
 463                                                 /*
 464                                                  * Reset in_vain because an ack
 465                                                  * for this commit has not been
 466                                                  * seen.
 467                                                  */
 468                                                 in_vain = 0;
 469                                         } else if (!args->stateless_rpc
 470                                                    || ack != ACK_common)
 471                                                 in_vain = 0;
 472                                         mark_common(commit, 0, 1);
 473                                         retval = 0;
 474                                         got_continue = 1;
 475                                         if (ack == ACK_ready) {
 476                                                 clear_prio_queue(&rev_list);
 477                                                 got_ready = 1;
 478                                         }
 479                                         break;
 480                                         }
 481                                 }
 482                         } while (ack);
 483                         flushes--;
 484                         if (got_continue && MAX_IN_VAIN < in_vain) {
 485                                 print_verbose(args, _("giving up"));
 486                                 break; /* give up */
 487                         }
 488                 }
 489         }
 490 done:
 491         if (!got_ready || !no_done) {
 492                 packet_buf_write(&req_buf, "done\n");
 493                 send_request(args, fd[1], &req_buf);
 494         }
 495         print_verbose(args, _("done"));
 496         if (retval != 0) {
 497                 multi_ack = 0;
 498                 flushes++;
 499         }
 500         strbuf_release(&req_buf);
 501
 502         if (!got_ready || !no_done)
 503                 consume_shallow_list(args, fd[0]);
 504         while (flushes || multi_ack) {
 505                 int ack = get_ack(fd[0], result_sha1);
 506                 if (ack) {
 507                         print_verbose(args, _("got %s (%d) %s"), "ack",
 508                                       ack, sha1_to_hex(result_sha1));
 509                         if (ack == ACK)
 510                                 return 0;
 511                         multi_ack = 1;
 512                         continue;
 513                 }
 514                 flushes--;
 515         }
 516         /* it is no error to fetch into a completely empty repo */
 517         return count ? retval : 0;
 518 }
 519
 520 static struct commit_list *complete;
 521
 522 static int mark_complete(const unsigned char *sha1)
 523 {
 524         struct object *o = parse_object(sha1);
 525
 526         while (o && o->type == OBJ_TAG) {
 527                 struct tag *t = (struct tag *) o;
 528                 if (!t->tagged)
 529                         break; /* broken repository */
 530                 o->flags |= COMPLETE;
 531                 o = parse_object(t->tagged->oid.hash);
 532         }
 533         if (o && o->type == OBJ_COMMIT) {
 534                 struct commit *commit = (struct commit *)o;
 535                 if (!(commit->object.flags & COMPLETE)) {
 536                         commit->object.flags |= COMPLETE;
 537                         commit_list_insert(commit, &complete);
 538                 }
 539         }
 540         return 0;
 541 }
 542
 543 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 544                              int flag, void *cb_data)
 545 {
 546         return mark_complete(oid->hash);
 547 }
 548
 549 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 550                                          unsigned long cutoff)
 551 {
 552         while (complete && cutoff <= complete->item->date) {
 553                 print_verbose(args, _("Marking %s as complete"),
 554                               oid_to_hex(&complete->item->object.oid));
 555                 pop_most_recent_commit(&complete, COMPLETE);
 556         }
 557 }
 558
 559 static void filter_refs(struct fetch_pack_args *args,
 560                         struct ref **refs,
 561                         struct ref **sought, int nr_sought)
 562 {
 563         struct ref *newlist = NULL;
 564         struct ref **newtail = &newlist;
 565         struct ref *ref, *next;
 566         int i;
 567
 568         i = 0;
 569         for (ref = *refs; ref; ref = next) {
 570                 int keep = 0;
 571                 next = ref->next;
 572
 573                 if (starts_with(ref->name, "refs/") &&
 574                     check_refname_format(ref->name, 0))
 575                         ; /* trash */
 576                 else {
 577                         while (i < nr_sought) {
 578                                 int cmp = strcmp(ref->name, sought[i]->name);
 579                                 if (cmp < 0)
 580                                         break; /* definitely do not have it */
 581                                 else if (cmp == 0) {
 582                                         keep = 1; /* definitely have it */
 583                                         sought[i]->match_status = REF_MATCHED;
 584                                 }
 585                                 i++;
 586                         }
 587                 }
 588
 589                 if (!keep && args->fetch_all &&
 590                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 591                         keep = 1;
 592
 593                 if (keep) {
 594                         *newtail = ref;
 595                         ref->next = NULL;
 596                         newtail = &ref->next;
 597                 } else {
 598                         free(ref);
 599                 }
 600         }
 601
 602         /* Append unmatched requests to the list */
 603         for (i = 0; i < nr_sought; i++) {
 604                 unsigned char sha1[20];
 605
 606                 ref = sought[i];
 607                 if (ref->match_status != REF_NOT_MATCHED)
 608                         continue;
 609                 if (get_sha1_hex(ref->name, sha1) ||
 610                     ref->name[40] != '\0' ||
 611                     hashcmp(sha1, ref->old_oid.hash))
 612                         continue;
 613
 614                 if ((allow_unadvertised_object_request &
 615                     (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1))) {
 616                         ref->match_status = REF_MATCHED;
 617                         *newtail = copy_ref(ref);
 618                         newtail = &(*newtail)->next;
 619                 } else {
 620                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 621                 }
 622         }
 623         *refs = newlist;
 624 }
 625
 626 static void mark_alternate_complete(const struct ref *ref, void *unused)
 627 {
 628         mark_complete(ref->old_oid.hash);
 629 }
 630
 631 static int everything_local(struct fetch_pack_args *args,
 632                             struct ref **refs,
 633                             struct ref **sought, int nr_sought)
 634 {
 635         struct ref *ref;
 636         int retval;
 637         unsigned long cutoff = 0;
 638
 639         save_commit_buffer = 0;
 640
 641         for (ref = *refs; ref; ref = ref->next) {
 642                 struct object *o;
 643
 644                 if (!has_object_file(&ref->old_oid))
 645                         continue;
 646
 647                 o = parse_object(ref->old_oid.hash);
 648                 if (!o)
 649                         continue;
 650
 651                 /* We already have it -- which may mean that we were
 652                  * in sync with the other side at some time after
 653                  * that (it is OK if we guess wrong here).
 654                  */
 655                 if (o->type == OBJ_COMMIT) {
 656                         struct commit *commit = (struct commit *)o;
 657                         if (!cutoff || cutoff < commit->date)
 658                                 cutoff = commit->date;
 659                 }
 660         }
 661
 662         if (!args->deepen) {
 663                 for_each_ref(mark_complete_oid, NULL);
 664                 for_each_alternate_ref(mark_alternate_complete, NULL);
 665                 commit_list_sort_by_date(&complete);
 666                 if (cutoff)
 667                         mark_recent_complete_commits(args, cutoff);
 668         }
 669
 670         /*
 671          * Mark all complete remote refs as common refs.
 672          * Don't mark them common yet; the server has to be told so first.
 673          */
 674         for (ref = *refs; ref; ref = ref->next) {
 675                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 676                                              NULL, 0);
 677
 678                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 679                         continue;
 680
 681                 if (!(o->flags & SEEN)) {
 682                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 683
 684                         mark_common((struct commit *)o, 1, 1);
 685                 }
 686         }
 687
 688         filter_refs(args, refs, sought, nr_sought);
 689
 690         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 691                 const unsigned char *remote = ref->old_oid.hash;
 692                 struct object *o;
 693
 694                 o = lookup_object(remote);
 695                 if (!o || !(o->flags & COMPLETE)) {
 696                         retval = 0;
 697                         print_verbose(args, "want %s (%s)", sha1_to_hex(remote),
 698                                       ref->name);
 699                         continue;
 700                 }
 701                 print_verbose(args, _("already have %s (%s)"), sha1_to_hex(remote),
 702                               ref->name);
 703         }
 704         return retval;
 705 }
 706
 707 static int sideband_demux(int in, int out, void *data)
 708 {
 709         int *xd = data;
 710         int ret;
 711
 712         ret = recv_sideband("fetch-pack", xd[0], out);
 713         close(out);
 714         return ret;
 715 }
 716
 717 static int get_pack(struct fetch_pack_args *args,
 718                     int xd[2], char **pack_lockfile)
 719 {
 720         struct async demux;
 721         int do_keep = args->keep_pack;
 722         const char *cmd_name;
 723         struct pack_header header;
 724         int pass_header = 0;
 725         struct child_process cmd = CHILD_PROCESS_INIT;
 726         int ret;
 727
 728         memset(&demux, 0, sizeof(demux));
 729         if (use_sideband) {
 730                 /* xd[] is talking with upload-pack; subprocess reads from
 731                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 732                  * through demux->out.
 733                  */
 734                 demux.proc = sideband_demux;
 735                 demux.data = xd;
 736                 demux.out = -1;
 737                 demux.isolate_sigpipe = 1;
 738                 if (start_async(&demux))
 739                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 740         }
 741         else
 742                 demux.out = xd[0];
 743
 744         if (!args->keep_pack && unpack_limit) {
 745
 746                 if (read_pack_header(demux.out, &header))
 747                         die(_("protocol error: bad pack header"));
 748                 pass_header = 1;
 749                 if (ntohl(header.hdr_entries) < unpack_limit)
 750                         do_keep = 0;
 751                 else
 752                         do_keep = 1;
 753         }
 754
 755         if (alternate_shallow_file) {
 756                 argv_array_push(&cmd.args, "--shallow-file");
 757                 argv_array_push(&cmd.args, alternate_shallow_file);
 758         }
 759
 760         if (do_keep) {
 761                 if (pack_lockfile)
 762                         cmd.out = -1;
 763                 cmd_name = "index-pack";
 764                 argv_array_push(&cmd.args, cmd_name);
 765                 argv_array_push(&cmd.args, "--stdin");
 766                 if (!args->quiet && !args->no_progress)
 767                         argv_array_push(&cmd.args, "-v");
 768                 if (args->use_thin_pack)
 769                         argv_array_push(&cmd.args, "--fix-thin");
 770                 if (args->lock_pack || unpack_limit) {
 771                         char hostname[256];
 772                         if (gethostname(hostname, sizeof(hostname)))
 773                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 774                         argv_array_pushf(&cmd.args,
 775                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 776                                         (uintmax_t)getpid(), hostname);
 777                 }
 778                 if (args->check_self_contained_and_connected)
 779                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 780         }
 781         else {
 782                 cmd_name = "unpack-objects";
 783                 argv_array_push(&cmd.args, cmd_name);
 784                 if (args->quiet || args->no_progress)
 785                         argv_array_push(&cmd.args, "-q");
 786                 args->check_self_contained_and_connected = 0;
 787         }
 788
 789         if (pass_header)
 790                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 791                                  ntohl(header.hdr_version),
 792                                  ntohl(header.hdr_entries));
 793         if (fetch_fsck_objects >= 0
 794             ? fetch_fsck_objects
 795             : transfer_fsck_objects >= 0
 796             ? transfer_fsck_objects
 797             : 0)
 798                 argv_array_push(&cmd.args, "--strict");
 799
 800         cmd.in = demux.out;
 801         cmd.git_cmd = 1;
 802         if (start_command(&cmd))
 803                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 804         if (do_keep && pack_lockfile) {
 805                 *pack_lockfile = index_pack_lockfile(cmd.out);
 806                 close(cmd.out);
 807         }
 808
 809         if (!use_sideband)
 810                 /* Closed by start_command() */
 811                 xd[0] = -1;
 812
 813         ret = finish_command(&cmd);
 814         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 815                 args->self_contained_and_connected =
 816                         args->check_self_contained_and_connected &&
 817                         ret == 0;
 818         else
 819                 die(_("%s failed"), cmd_name);
 820         if (use_sideband && finish_async(&demux))
 821                 die(_("error in sideband demultiplexer"));
 822         return 0;
 823 }
 824
 825 static int cmp_ref_by_name(const void *a_, const void *b_)
 826 {
 827         const struct ref *a = *((const struct ref **)a_);
 828         const struct ref *b = *((const struct ref **)b_);
 829         return strcmp(a->name, b->name);
 830 }
 831
 832 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 833                                  int fd[2],
 834                                  const struct ref *orig_ref,
 835                                  struct ref **sought, int nr_sought,
 836                                  struct shallow_info *si,
 837                                  char **pack_lockfile)
 838 {
 839         struct ref *ref = copy_ref_list(orig_ref);
 840         unsigned char sha1[20];
 841         const char *agent_feature;
 842         int agent_len;
 843
 844         sort_ref_list(&ref, ref_compare_name);
 845         QSORT(sought, nr_sought, cmp_ref_by_name);
 846
 847         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 848                 die(_("Server does not support shallow clients"));
 849         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 850                 args->deepen = 1;
 851         if (server_supports("multi_ack_detailed")) {
 852                 print_verbose(args, _("Server supports multi_ack_detailed"));
 853                 multi_ack = 2;
 854                 if (server_supports("no-done")) {
 855                         print_verbose(args, _("Server supports no-done"));
 856                         if (args->stateless_rpc)
 857                                 no_done = 1;
 858                 }
 859         }
 860         else if (server_supports("multi_ack")) {
 861                 print_verbose(args, _("Server supports multi_ack"));
 862                 multi_ack = 1;
 863         }
 864         if (server_supports("side-band-64k")) {
 865                 print_verbose(args, _("Server supports side-band-64k"));
 866                 use_sideband = 2;
 867         }
 868         else if (server_supports("side-band")) {
 869                 print_verbose(args, _("Server supports side-band"));
 870                 use_sideband = 1;
 871         }
 872         if (server_supports("allow-tip-sha1-in-want")) {
 873                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 874                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 875         }
 876         if (server_supports("allow-reachable-sha1-in-want")) {
 877                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 878                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 879         }
 880         if (!server_supports("thin-pack"))
 881                 args->use_thin_pack = 0;
 882         if (!server_supports("no-progress"))
 883                 args->no_progress = 0;
 884         if (!server_supports("include-tag"))
 885                 args->include_tag = 0;
 886         if (server_supports("ofs-delta"))
 887                 print_verbose(args, _("Server supports ofs-delta"));
 888         else
 889                 prefer_ofs_delta = 0;
 890
 891         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 892                 agent_supported = 1;
 893                 if (agent_len)
 894                         print_verbose(args, _("Server version is %.*s"),
 895                                       agent_len, agent_feature);
 896         }
 897         if (server_supports("deepen-since"))
 898                 deepen_since_ok = 1;
 899         else if (args->deepen_since)
 900                 die(_("Server does not support --shallow-since"));
 901         if (server_supports("deepen-not"))
 902                 deepen_not_ok = 1;
 903         else if (args->deepen_not)
 904                 die(_("Server does not support --shallow-exclude"));
 905         if (!server_supports("deepen-relative") && args->deepen_relative)
 906                 die(_("Server does not support --deepen"));
 907
 908         if (everything_local(args, &ref, sought, nr_sought)) {
 909                 packet_flush(fd[1]);
 910                 goto all_done;
 911         }
 912         if (find_common(args, fd, sha1, ref) < 0)
 913                 if (!args->keep_pack)
 914                         /* When cloning, it is not unusual to have
 915                          * no common commit.
 916                          */
 917                         warning(_("no common commits"));
 918
 919         if (args->stateless_rpc)
 920                 packet_flush(fd[1]);
 921         if (args->deepen)
 922                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 923                                         NULL);
 924         else if (si->nr_ours || si->nr_theirs)
 925                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 926         else
 927                 alternate_shallow_file = NULL;
 928         if (get_pack(args, fd, pack_lockfile))
 929                 die(_("git fetch-pack: fetch failed."));
 930
 931  all_done:
 932         return ref;
 933 }
 934
 935 static void fetch_pack_config(void)
 936 {
 937         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
 938         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
 939         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
 940         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
 941         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
 942
 943         git_config(git_default_config, NULL);
 944 }
 945
 946 static void fetch_pack_setup(void)
 947 {
 948         static int did_setup;
 949         if (did_setup)
 950                 return;
 951         fetch_pack_config();
 952         if (0 <= transfer_unpack_limit)
 953                 unpack_limit = transfer_unpack_limit;
 954         else if (0 <= fetch_unpack_limit)
 955                 unpack_limit = fetch_unpack_limit;
 956         did_setup = 1;
 957 }
 958
 959 static int remove_duplicates_in_refs(struct ref **ref, int nr)
 960 {
 961         struct string_list names = STRING_LIST_INIT_NODUP;
 962         int src, dst;
 963
 964         for (src = dst = 0; src < nr; src++) {
 965                 struct string_list_item *item;
 966                 item = string_list_insert(&names, ref[src]->name);
 967                 if (item->util)
 968                         continue; /* already have it */
 969                 item->util = ref[src];
 970                 if (src != dst)
 971                         ref[dst] = ref[src];
 972                 dst++;
 973         }
 974         for (src = dst; src < nr; src++)
 975                 ref[src] = NULL;
 976         string_list_clear(&names, 0);
 977         return dst;
 978 }
 979
 980 static void update_shallow(struct fetch_pack_args *args,
 981                            struct ref **sought, int nr_sought,
 982                            struct shallow_info *si)
 983 {
 984         struct sha1_array ref = SHA1_ARRAY_INIT;
 985         int *status;
 986         int i;
 987
 988         if (args->deepen && alternate_shallow_file) {
 989                 if (*alternate_shallow_file == '\0') { /* --unshallow */
 990                         unlink_or_warn(git_path_shallow());
 991                         rollback_lock_file(&shallow_lock);
 992                 } else
 993                         commit_lock_file(&shallow_lock);
 994                 return;
 995         }
 996
 997         if (!si->shallow || !si->shallow->nr)
 998                 return;
 999
1000         if (args->cloning) {
1001                 /*
1002                  * remote is shallow, but this is a clone, there are
1003                  * no objects in repo to worry about. Accept any
1004                  * shallow points that exist in the pack (iow in repo
1005                  * after get_pack() and reprepare_packed_git())
1006                  */
1007                 struct sha1_array extra = SHA1_ARRAY_INIT;
1008                 unsigned char (*sha1)[20] = si->shallow->sha1;
1009                 for (i = 0; i < si->shallow->nr; i++)
1010                         if (has_sha1_file(sha1[i]))
1011                                 sha1_array_append(&extra, sha1[i]);
1012                 if (extra.nr) {
1013                         setup_alternate_shallow(&shallow_lock,
1014                                                 &alternate_shallow_file,
1015                                                 &extra);
1016                         commit_lock_file(&shallow_lock);
1017                 }
1018                 sha1_array_clear(&extra);
1019                 return;
1020         }
1021
1022         if (!si->nr_ours && !si->nr_theirs)
1023                 return;
1024
1025         remove_nonexistent_theirs_shallow(si);
1026         if (!si->nr_ours && !si->nr_theirs)
1027                 return;
1028         for (i = 0; i < nr_sought; i++)
1029                 sha1_array_append(&ref, sought[i]->old_oid.hash);
1030         si->ref = &ref;
1031
1032         if (args->update_shallow) {
1033                 /*
1034                  * remote is also shallow, .git/shallow may be updated
1035                  * so all refs can be accepted. Make sure we only add
1036                  * shallow roots that are actually reachable from new
1037                  * refs.
1038                  */
1039                 struct sha1_array extra = SHA1_ARRAY_INIT;
1040                 unsigned char (*sha1)[20] = si->shallow->sha1;
1041                 assign_shallow_commits_to_refs(si, NULL, NULL);
1042                 if (!si->nr_ours && !si->nr_theirs) {
1043                         sha1_array_clear(&ref);
1044                         return;
1045                 }
1046                 for (i = 0; i < si->nr_ours; i++)
1047                         sha1_array_append(&extra, sha1[si->ours[i]]);
1048                 for (i = 0; i < si->nr_theirs; i++)
1049                         sha1_array_append(&extra, sha1[si->theirs[i]]);
1050                 setup_alternate_shallow(&shallow_lock,
1051                                         &alternate_shallow_file,
1052                                         &extra);
1053                 commit_lock_file(&shallow_lock);
1054                 sha1_array_clear(&extra);
1055                 sha1_array_clear(&ref);
1056                 return;
1057         }
1058
1059         /*
1060          * remote is also shallow, check what ref is safe to update
1061          * without updating .git/shallow
1062          */
1063         status = xcalloc(nr_sought, sizeof(*status));
1064         assign_shallow_commits_to_refs(si, NULL, status);
1065         if (si->nr_ours || si->nr_theirs) {
1066                 for (i = 0; i < nr_sought; i++)
1067                         if (status[i])
1068                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1069         }
1070         free(status);
1071         sha1_array_clear(&ref);
1072 }
1073
1074 struct ref *fetch_pack(struct fetch_pack_args *args,
1075                        int fd[], struct child_process *conn,
1076                        const struct ref *ref,
1077                        const char *dest,
1078                        struct ref **sought, int nr_sought,
1079                        struct sha1_array *shallow,
1080                        char **pack_lockfile)
1081 {
1082         struct ref *ref_cpy;
1083         struct shallow_info si;
1084
1085         fetch_pack_setup();
1086         if (nr_sought)
1087                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1088
1089         if (!ref) {
1090                 packet_flush(fd[1]);
1091                 die(_("no matching remote head"));
1092         }
1093         prepare_shallow_info(&si, shallow);
1094         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1095                                 &si, pack_lockfile);
1096         reprepare_packed_git();
1097         update_shallow(args, sought, nr_sought, &si);
1098         clear_shallow_info(&si);
1099         return ref_cpy;
1100 }
1101
1102 int report_unmatched_refs(struct ref **sought, int nr_sought)
1103 {
1104         int i, ret = 0;
1105
1106         for (i = 0; i < nr_sought; i++) {
1107                 if (!sought[i])
1108                         continue;
1109                 switch (sought[i]->match_status) {
1110                 case REF_MATCHED:
1111                         continue;
1112                 case REF_NOT_MATCHED:
1113                         error(_("no such remote ref %s"), sought[i]->name);
1114                         break;
1115                 case REF_UNADVERTISED_NOT_ALLOWED:
1116                         error(_("Server does not allow request for unadvertised object %s"),
1117                               sought[i]->name);
1118                         break;
1119                 }
1120                 ret = 1;
1121         }
1122         return ret;
1123 }