git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "lockfile.h"
   3 #include "refs.h"
   4 #include "pkt-line.h"
   5 #include "commit.h"
   6 #include "tag.h"
   7 #include "exec_cmd.h"
   8 #include "pack.h"
   9 #include "sideband.h"
  10 #include "fetch-pack.h"
  11 #include "remote.h"
  12 #include "run-command.h"
  13 #include "connect.h"
  14 #include "transport.h"
  15 #include "version.h"
  16 #include "prio-queue.h"
  17 #include "sha1-array.h"
  18 #include "oidset.h"
  19
  20 static int transfer_unpack_limit = -1;
  21 static int fetch_unpack_limit = -1;
  22 static int unpack_limit = 100;
  23 static int prefer_ofs_delta = 1;
  24 static int no_done;
  25 static int deepen_since_ok;
  26 static int deepen_not_ok;
  27 static int fetch_fsck_objects = -1;
  28 static int transfer_fsck_objects = -1;
  29 static int agent_supported;
  30 static struct lock_file shallow_lock;
  31 static const char *alternate_shallow_file;
  32
  33 /* Remember to update object flag allocation in object.h */
  34 #define COMPLETE        (1U << 0)
  35 #define COMMON          (1U << 1)
  36 #define COMMON_REF      (1U << 2)
  37 #define SEEN            (1U << 3)
  38 #define POPPED          (1U << 4)
  39 #define ALTERNATE       (1U << 5)
  40
  41 static int marked;
  42
  43 /*
  44  * After sending this many "have"s if we do not get any new ACK , we
  45  * give up traversing our history.
  46  */
  47 #define MAX_IN_VAIN 256
  48
  49 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  50 static int non_common_revs, multi_ack, use_sideband;
  51 /* Allow specifying sha1 if it is a ref tip. */
  52 #define ALLOW_TIP_SHA1  01
  53 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  54 #define ALLOW_REACHABLE_SHA1    02
  55 static unsigned int allow_unadvertised_object_request;
  56
  57 __attribute__((format (printf, 2, 3)))
  58 static inline void print_verbose(const struct fetch_pack_args *args,
  59                                  const char *fmt, ...)
  60 {
  61         va_list params;
  62
  63         if (!args->verbose)
  64                 return;
  65
  66         va_start(params, fmt);
  67         vfprintf(stderr, fmt, params);
  68         va_end(params);
  69         fputc('\n', stderr);
  70 }
  71
  72 struct alternate_object_cache {
  73         struct object **items;
  74         size_t nr, alloc;
  75 };
  76
  77 static void cache_one_alternate(const char *refname,
  78                                 const struct object_id *oid,
  79                                 void *vcache)
  80 {
  81         struct alternate_object_cache *cache = vcache;
  82         struct object *obj = parse_object(oid);
  83
  84         if (!obj || (obj->flags & ALTERNATE))
  85                 return;
  86
  87         obj->flags |= ALTERNATE;
  88         ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  89         cache->items[cache->nr++] = obj;
  90 }
  91
  92 static void for_each_cached_alternate(void (*cb)(struct object *))
  93 {
  94         static int initialized;
  95         static struct alternate_object_cache cache;
  96         size_t i;
  97
  98         if (!initialized) {
  99                 for_each_alternate_ref(cache_one_alternate, &cache);
 100                 initialized = 1;
 101         }
 102
 103         for (i = 0; i < cache.nr; i++)
 104                 cb(cache.items[i]);
 105 }
 106
 107 static void rev_list_push(struct commit *commit, int mark)
 108 {
 109         if (!(commit->object.flags & mark)) {
 110                 commit->object.flags |= mark;
 111
 112                 if (parse_commit(commit))
 113                         return;
 114
 115                 prio_queue_put(&rev_list, commit);
 116
 117                 if (!(commit->object.flags & COMMON))
 118                         non_common_revs++;
 119         }
 120 }
 121
 122 static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 123 {
 124         struct object *o = deref_tag(parse_object(oid), refname, 0);
 125
 126         if (o && o->type == OBJ_COMMIT)
 127                 rev_list_push((struct commit *)o, SEEN);
 128
 129         return 0;
 130 }
 131
 132 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 133                                    int flag, void *cb_data)
 134 {
 135         return rev_list_insert_ref(refname, oid);
 136 }
 137
 138 static int clear_marks(const char *refname, const struct object_id *oid,
 139                        int flag, void *cb_data)
 140 {
 141         struct object *o = deref_tag(parse_object(oid), refname, 0);
 142
 143         if (o && o->type == OBJ_COMMIT)
 144                 clear_commit_marks((struct commit *)o,
 145                                    COMMON | COMMON_REF | SEEN | POPPED);
 146         return 0;
 147 }
 148
 149 /*
 150    This function marks a rev and its ancestors as common.
 151    In some cases, it is desirable to mark only the ancestors (for example
 152    when only the server does not yet know that they are common).
 153 */
 154
 155 static void mark_common(struct commit *commit,
 156                 int ancestors_only, int dont_parse)
 157 {
 158         if (commit != NULL && !(commit->object.flags & COMMON)) {
 159                 struct object *o = (struct object *)commit;
 160
 161                 if (!ancestors_only)
 162                         o->flags |= COMMON;
 163
 164                 if (!(o->flags & SEEN))
 165                         rev_list_push(commit, SEEN);
 166                 else {
 167                         struct commit_list *parents;
 168
 169                         if (!ancestors_only && !(o->flags & POPPED))
 170                                 non_common_revs--;
 171                         if (!o->parsed && !dont_parse)
 172                                 if (parse_commit(commit))
 173                                         return;
 174
 175                         for (parents = commit->parents;
 176                                         parents;
 177                                         parents = parents->next)
 178                                 mark_common(parents->item, 0, dont_parse);
 179                 }
 180         }
 181 }
 182
 183 /*
 184   Get the next rev to send, ignoring the common.
 185 */
 186
 187 static const struct object_id *get_rev(void)
 188 {
 189         struct commit *commit = NULL;
 190
 191         while (commit == NULL) {
 192                 unsigned int mark;
 193                 struct commit_list *parents;
 194
 195                 if (rev_list.nr == 0 || non_common_revs == 0)
 196                         return NULL;
 197
 198                 commit = prio_queue_get(&rev_list);
 199                 parse_commit(commit);
 200                 parents = commit->parents;
 201
 202                 commit->object.flags |= POPPED;
 203                 if (!(commit->object.flags & COMMON))
 204                         non_common_revs--;
 205
 206                 if (commit->object.flags & COMMON) {
 207                         /* do not send "have", and ignore ancestors */
 208                         commit = NULL;
 209                         mark = COMMON | SEEN;
 210                 } else if (commit->object.flags & COMMON_REF)
 211                         /* send "have", and ignore ancestors */
 212                         mark = COMMON | SEEN;
 213                 else
 214                         /* send "have", also for its ancestors */
 215                         mark = SEEN;
 216
 217                 while (parents) {
 218                         if (!(parents->item->object.flags & SEEN))
 219                                 rev_list_push(parents->item, mark);
 220                         if (mark & COMMON)
 221                                 mark_common(parents->item, 1, 0);
 222                         parents = parents->next;
 223                 }
 224         }
 225
 226         return &commit->object.oid;
 227 }
 228
 229 enum ack_type {
 230         NAK = 0,
 231         ACK,
 232         ACK_continue,
 233         ACK_common,
 234         ACK_ready
 235 };
 236
 237 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 238 {
 239         if (args->stateless_rpc && args->deepen) {
 240                 /* If we sent a depth we will get back "duplicate"
 241                  * shallow and unshallow commands every time there
 242                  * is a block of have lines exchanged.
 243                  */
 244                 char *line;
 245                 while ((line = packet_read_line(fd, NULL))) {
 246                         if (starts_with(line, "shallow "))
 247                                 continue;
 248                         if (starts_with(line, "unshallow "))
 249                                 continue;
 250                         die(_("git fetch-pack: expected shallow list"));
 251                 }
 252         }
 253 }
 254
 255 static enum ack_type get_ack(int fd, struct object_id *result_oid)
 256 {
 257         int len;
 258         char *line = packet_read_line(fd, &len);
 259         const char *arg;
 260
 261         if (!len)
 262                 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
 263         if (!strcmp(line, "NAK"))
 264                 return NAK;
 265         if (skip_prefix(line, "ACK ", &arg)) {
 266                 if (!get_oid_hex(arg, result_oid)) {
 267                         arg += 40;
 268                         len -= arg - line;
 269                         if (len < 1)
 270                                 return ACK;
 271                         if (strstr(arg, "continue"))
 272                                 return ACK_continue;
 273                         if (strstr(arg, "common"))
 274                                 return ACK_common;
 275                         if (strstr(arg, "ready"))
 276                                 return ACK_ready;
 277                         return ACK;
 278                 }
 279         }
 280         if (skip_prefix(line, "ERR ", &arg))
 281                 die(_("remote error: %s"), arg);
 282         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 283 }
 284
 285 static void send_request(struct fetch_pack_args *args,
 286                          int fd, struct strbuf *buf)
 287 {
 288         if (args->stateless_rpc) {
 289                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 290                 packet_flush(fd);
 291         } else
 292                 write_or_die(fd, buf->buf, buf->len);
 293 }
 294
 295 static void insert_one_alternate_object(struct object *obj)
 296 {
 297         rev_list_insert_ref(NULL, &obj->oid);
 298 }
 299
 300 #define INITIAL_FLUSH 16
 301 #define PIPESAFE_FLUSH 32
 302 #define LARGE_FLUSH 16384
 303
 304 static int next_flush(struct fetch_pack_args *args, int count)
 305 {
 306         if (args->stateless_rpc) {
 307                 if (count < LARGE_FLUSH)
 308                         count <<= 1;
 309                 else
 310                         count = count * 11 / 10;
 311         } else {
 312                 if (count < PIPESAFE_FLUSH)
 313                         count <<= 1;
 314                 else
 315                         count += PIPESAFE_FLUSH;
 316         }
 317         return count;
 318 }
 319
 320 static int find_common(struct fetch_pack_args *args,
 321                        int fd[2], struct object_id *result_oid,
 322                        struct ref *refs)
 323 {
 324         int fetching;
 325         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 326         const struct object_id *oid;
 327         unsigned in_vain = 0;
 328         int got_continue = 0;
 329         int got_ready = 0;
 330         struct strbuf req_buf = STRBUF_INIT;
 331         size_t state_len = 0;
 332
 333         if (args->stateless_rpc && multi_ack == 1)
 334                 die(_("--stateless-rpc requires multi_ack_detailed"));
 335         if (marked)
 336                 for_each_ref(clear_marks, NULL);
 337         marked = 1;
 338
 339         for_each_ref(rev_list_insert_ref_oid, NULL);
 340         for_each_cached_alternate(insert_one_alternate_object);
 341
 342         fetching = 0;
 343         for ( ; refs ; refs = refs->next) {
 344                 struct object_id *remote = &refs->old_oid;
 345                 const char *remote_hex;
 346                 struct object *o;
 347
 348                 /*
 349                  * If that object is complete (i.e. it is an ancestor of a
 350                  * local ref), we tell them we have it but do not have to
 351                  * tell them about its ancestors, which they already know
 352                  * about.
 353                  *
 354                  * We use lookup_object here because we are only
 355                  * interested in the case we *know* the object is
 356                  * reachable and we have already scanned it.
 357                  */
 358                 if (((o = lookup_object(remote->hash)) != NULL) &&
 359                                 (o->flags & COMPLETE)) {
 360                         continue;
 361                 }
 362
 363                 remote_hex = oid_to_hex(remote);
 364                 if (!fetching) {
 365                         struct strbuf c = STRBUF_INIT;
 366                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 367                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 368                         if (no_done)            strbuf_addstr(&c, " no-done");
 369                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 370                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 371                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 372                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 373                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 374                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 375                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 376                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 377                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 378                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 379                                                             git_user_agent_sanitized());
 380                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 381                         strbuf_release(&c);
 382                 } else
 383                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 384                 fetching++;
 385         }
 386
 387         if (!fetching) {
 388                 strbuf_release(&req_buf);
 389                 packet_flush(fd[1]);
 390                 return 1;
 391         }
 392
 393         if (is_repository_shallow())
 394                 write_shallow_commits(&req_buf, 1, NULL);
 395         if (args->depth > 0)
 396                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 397         if (args->deepen_since) {
 398                 timestamp_t max_age = approxidate(args->deepen_since);
 399                 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 400         }
 401         if (args->deepen_not) {
 402                 int i;
 403                 for (i = 0; i < args->deepen_not->nr; i++) {
 404                         struct string_list_item *s = args->deepen_not->items + i;
 405                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 406                 }
 407         }
 408         packet_buf_flush(&req_buf);
 409         state_len = req_buf.len;
 410
 411         if (args->deepen) {
 412                 char *line;
 413                 const char *arg;
 414                 struct object_id oid;
 415
 416                 send_request(args, fd[1], &req_buf);
 417                 while ((line = packet_read_line(fd[0], NULL))) {
 418                         if (skip_prefix(line, "shallow ", &arg)) {
 419                                 if (get_oid_hex(arg, &oid))
 420                                         die(_("invalid shallow line: %s"), line);
 421                                 register_shallow(&oid);
 422                                 continue;
 423                         }
 424                         if (skip_prefix(line, "unshallow ", &arg)) {
 425                                 if (get_oid_hex(arg, &oid))
 426                                         die(_("invalid unshallow line: %s"), line);
 427                                 if (!lookup_object(oid.hash))
 428                                         die(_("object not found: %s"), line);
 429                                 /* make sure that it is parsed as shallow */
 430                                 if (!parse_object(&oid))
 431                                         die(_("error in object: %s"), line);
 432                                 if (unregister_shallow(&oid))
 433                                         die(_("no shallow found: %s"), line);
 434                                 continue;
 435                         }
 436                         die(_("expected shallow/unshallow, got %s"), line);
 437                 }
 438         } else if (!args->stateless_rpc)
 439                 send_request(args, fd[1], &req_buf);
 440
 441         if (!args->stateless_rpc) {
 442                 /* If we aren't using the stateless-rpc interface
 443                  * we don't need to retain the headers.
 444                  */
 445                 strbuf_setlen(&req_buf, 0);
 446                 state_len = 0;
 447         }
 448
 449         flushes = 0;
 450         retval = -1;
 451         while ((oid = get_rev())) {
 452                 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 453                 print_verbose(args, "have %s", oid_to_hex(oid));
 454                 in_vain++;
 455                 if (flush_at <= ++count) {
 456                         int ack;
 457
 458                         packet_buf_flush(&req_buf);
 459                         send_request(args, fd[1], &req_buf);
 460                         strbuf_setlen(&req_buf, state_len);
 461                         flushes++;
 462                         flush_at = next_flush(args, count);
 463
 464                         /*
 465                          * We keep one window "ahead" of the other side, and
 466                          * will wait for an ACK only on the next one
 467                          */
 468                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 469                                 continue;
 470
 471                         consume_shallow_list(args, fd[0]);
 472                         do {
 473                                 ack = get_ack(fd[0], result_oid);
 474                                 if (ack)
 475                                         print_verbose(args, _("got %s %d %s"), "ack",
 476                                                       ack, oid_to_hex(result_oid));
 477                                 switch (ack) {
 478                                 case ACK:
 479                                         flushes = 0;
 480                                         multi_ack = 0;
 481                                         retval = 0;
 482                                         goto done;
 483                                 case ACK_common:
 484                                 case ACK_ready:
 485                                 case ACK_continue: {
 486                                         struct commit *commit =
 487                                                 lookup_commit(result_oid);
 488                                         if (!commit)
 489                                                 die(_("invalid commit %s"), oid_to_hex(result_oid));
 490                                         if (args->stateless_rpc
 491                                          && ack == ACK_common
 492                                          && !(commit->object.flags & COMMON)) {
 493                                                 /* We need to replay the have for this object
 494                                                  * on the next RPC request so the peer knows
 495                                                  * it is in common with us.
 496                                                  */
 497                                                 const char *hex = oid_to_hex(result_oid);
 498                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 499                                                 state_len = req_buf.len;
 500                                                 /*
 501                                                  * Reset in_vain because an ack
 502                                                  * for this commit has not been
 503                                                  * seen.
 504                                                  */
 505                                                 in_vain = 0;
 506                                         } else if (!args->stateless_rpc
 507                                                    || ack != ACK_common)
 508                                                 in_vain = 0;
 509                                         mark_common(commit, 0, 1);
 510                                         retval = 0;
 511                                         got_continue = 1;
 512                                         if (ack == ACK_ready) {
 513                                                 clear_prio_queue(&rev_list);
 514                                                 got_ready = 1;
 515                                         }
 516                                         break;
 517                                         }
 518                                 }
 519                         } while (ack);
 520                         flushes--;
 521                         if (got_continue && MAX_IN_VAIN < in_vain) {
 522                                 print_verbose(args, _("giving up"));
 523                                 break; /* give up */
 524                         }
 525                 }
 526         }
 527 done:
 528         if (!got_ready || !no_done) {
 529                 packet_buf_write(&req_buf, "done\n");
 530                 send_request(args, fd[1], &req_buf);
 531         }
 532         print_verbose(args, _("done"));
 533         if (retval != 0) {
 534                 multi_ack = 0;
 535                 flushes++;
 536         }
 537         strbuf_release(&req_buf);
 538
 539         if (!got_ready || !no_done)
 540                 consume_shallow_list(args, fd[0]);
 541         while (flushes || multi_ack) {
 542                 int ack = get_ack(fd[0], result_oid);
 543                 if (ack) {
 544                         print_verbose(args, _("got %s (%d) %s"), "ack",
 545                                       ack, oid_to_hex(result_oid));
 546                         if (ack == ACK)
 547                                 return 0;
 548                         multi_ack = 1;
 549                         continue;
 550                 }
 551                 flushes--;
 552         }
 553         /* it is no error to fetch into a completely empty repo */
 554         return count ? retval : 0;
 555 }
 556
 557 static struct commit_list *complete;
 558
 559 static int mark_complete(const struct object_id *oid)
 560 {
 561         struct object *o = parse_object(oid);
 562
 563         while (o && o->type == OBJ_TAG) {
 564                 struct tag *t = (struct tag *) o;
 565                 if (!t->tagged)
 566                         break; /* broken repository */
 567                 o->flags |= COMPLETE;
 568                 o = parse_object(&t->tagged->oid);
 569         }
 570         if (o && o->type == OBJ_COMMIT) {
 571                 struct commit *commit = (struct commit *)o;
 572                 if (!(commit->object.flags & COMPLETE)) {
 573                         commit->object.flags |= COMPLETE;
 574                         commit_list_insert(commit, &complete);
 575                 }
 576         }
 577         return 0;
 578 }
 579
 580 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 581                              int flag, void *cb_data)
 582 {
 583         return mark_complete(oid);
 584 }
 585
 586 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 587                                          timestamp_t cutoff)
 588 {
 589         while (complete && cutoff <= complete->item->date) {
 590                 print_verbose(args, _("Marking %s as complete"),
 591                               oid_to_hex(&complete->item->object.oid));
 592                 pop_most_recent_commit(&complete, COMPLETE);
 593         }
 594 }
 595
 596 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 597 {
 598         for (; refs; refs = refs->next)
 599                 oidset_insert(oids, &refs->old_oid);
 600 }
 601
 602 static int tip_oids_contain(struct oidset *tip_oids,
 603                             struct ref *unmatched, struct ref *newlist,
 604                             const struct object_id *id)
 605 {
 606         /*
 607          * Note that this only looks at the ref lists the first time it's
 608          * called. This works out in filter_refs() because even though it may
 609          * add to "newlist" between calls, the additions will always be for
 610          * oids that are already in the set.
 611          */
 612         if (!tip_oids->map.tablesize) {
 613                 add_refs_to_oidset(tip_oids, unmatched);
 614                 add_refs_to_oidset(tip_oids, newlist);
 615         }
 616         return oidset_contains(tip_oids, id);
 617 }
 618
 619 static void filter_refs(struct fetch_pack_args *args,
 620                         struct ref **refs,
 621                         struct ref **sought, int nr_sought)
 622 {
 623         struct ref *newlist = NULL;
 624         struct ref **newtail = &newlist;
 625         struct ref *unmatched = NULL;
 626         struct ref *ref, *next;
 627         struct oidset tip_oids = OIDSET_INIT;
 628         int i;
 629
 630         i = 0;
 631         for (ref = *refs; ref; ref = next) {
 632                 int keep = 0;
 633                 next = ref->next;
 634
 635                 if (starts_with(ref->name, "refs/") &&
 636                     check_refname_format(ref->name, 0))
 637                         ; /* trash */
 638                 else {
 639                         while (i < nr_sought) {
 640                                 int cmp = strcmp(ref->name, sought[i]->name);
 641                                 if (cmp < 0)
 642                                         break; /* definitely do not have it */
 643                                 else if (cmp == 0) {
 644                                         keep = 1; /* definitely have it */
 645                                         sought[i]->match_status = REF_MATCHED;
 646                                 }
 647                                 i++;
 648                         }
 649                 }
 650
 651                 if (!keep && args->fetch_all &&
 652                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 653                         keep = 1;
 654
 655                 if (keep) {
 656                         *newtail = ref;
 657                         ref->next = NULL;
 658                         newtail = &ref->next;
 659                 } else {
 660                         ref->next = unmatched;
 661                         unmatched = ref;
 662                 }
 663         }
 664
 665         /* Append unmatched requests to the list */
 666         for (i = 0; i < nr_sought; i++) {
 667                 struct object_id oid;
 668                 const char *p;
 669
 670                 ref = sought[i];
 671                 if (ref->match_status != REF_NOT_MATCHED)
 672                         continue;
 673                 if (parse_oid_hex(ref->name, &oid, &p) ||
 674                     *p != '\0' ||
 675                     oidcmp(&oid, &ref->old_oid))
 676                         continue;
 677
 678                 if ((allow_unadvertised_object_request &
 679                      (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 680                     tip_oids_contain(&tip_oids, unmatched, newlist,
 681                                      &ref->old_oid)) {
 682                         ref->match_status = REF_MATCHED;
 683                         *newtail = copy_ref(ref);
 684                         newtail = &(*newtail)->next;
 685                 } else {
 686                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 687                 }
 688         }
 689
 690         oidset_clear(&tip_oids);
 691         for (ref = unmatched; ref; ref = next) {
 692                 next = ref->next;
 693                 free(ref);
 694         }
 695
 696         *refs = newlist;
 697 }
 698
 699 static void mark_alternate_complete(struct object *obj)
 700 {
 701         mark_complete(&obj->oid);
 702 }
 703
 704 static int everything_local(struct fetch_pack_args *args,
 705                             struct ref **refs,
 706                             struct ref **sought, int nr_sought)
 707 {
 708         struct ref *ref;
 709         int retval;
 710         timestamp_t cutoff = 0;
 711
 712         save_commit_buffer = 0;
 713
 714         for (ref = *refs; ref; ref = ref->next) {
 715                 struct object *o;
 716
 717                 if (!has_object_file(&ref->old_oid))
 718                         continue;
 719
 720                 o = parse_object(&ref->old_oid);
 721                 if (!o)
 722                         continue;
 723
 724                 /* We already have it -- which may mean that we were
 725                  * in sync with the other side at some time after
 726                  * that (it is OK if we guess wrong here).
 727                  */
 728                 if (o->type == OBJ_COMMIT) {
 729                         struct commit *commit = (struct commit *)o;
 730                         if (!cutoff || cutoff < commit->date)
 731                                 cutoff = commit->date;
 732                 }
 733         }
 734
 735         if (!args->deepen) {
 736                 for_each_ref(mark_complete_oid, NULL);
 737                 for_each_cached_alternate(mark_alternate_complete);
 738                 commit_list_sort_by_date(&complete);
 739                 if (cutoff)
 740                         mark_recent_complete_commits(args, cutoff);
 741         }
 742
 743         /*
 744          * Mark all complete remote refs as common refs.
 745          * Don't mark them common yet; the server has to be told so first.
 746          */
 747         for (ref = *refs; ref; ref = ref->next) {
 748                 struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 749                                              NULL, 0);
 750
 751                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 752                         continue;
 753
 754                 if (!(o->flags & SEEN)) {
 755                         rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 756
 757                         mark_common((struct commit *)o, 1, 1);
 758                 }
 759         }
 760
 761         filter_refs(args, refs, sought, nr_sought);
 762
 763         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 764                 const struct object_id *remote = &ref->old_oid;
 765                 struct object *o;
 766
 767                 o = lookup_object(remote->hash);
 768                 if (!o || !(o->flags & COMPLETE)) {
 769                         retval = 0;
 770                         print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 771                                       ref->name);
 772                         continue;
 773                 }
 774                 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 775                               ref->name);
 776         }
 777         return retval;
 778 }
 779
 780 static int sideband_demux(int in, int out, void *data)
 781 {
 782         int *xd = data;
 783         int ret;
 784
 785         ret = recv_sideband("fetch-pack", xd[0], out);
 786         close(out);
 787         return ret;
 788 }
 789
 790 static int get_pack(struct fetch_pack_args *args,
 791                     int xd[2], char **pack_lockfile)
 792 {
 793         struct async demux;
 794         int do_keep = args->keep_pack;
 795         const char *cmd_name;
 796         struct pack_header header;
 797         int pass_header = 0;
 798         struct child_process cmd = CHILD_PROCESS_INIT;
 799         int ret;
 800
 801         memset(&demux, 0, sizeof(demux));
 802         if (use_sideband) {
 803                 /* xd[] is talking with upload-pack; subprocess reads from
 804                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 805                  * through demux->out.
 806                  */
 807                 demux.proc = sideband_demux;
 808                 demux.data = xd;
 809                 demux.out = -1;
 810                 demux.isolate_sigpipe = 1;
 811                 if (start_async(&demux))
 812                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 813         }
 814         else
 815                 demux.out = xd[0];
 816
 817         if (!args->keep_pack && unpack_limit) {
 818
 819                 if (read_pack_header(demux.out, &header))
 820                         die(_("protocol error: bad pack header"));
 821                 pass_header = 1;
 822                 if (ntohl(header.hdr_entries) < unpack_limit)
 823                         do_keep = 0;
 824                 else
 825                         do_keep = 1;
 826         }
 827
 828         if (alternate_shallow_file) {
 829                 argv_array_push(&cmd.args, "--shallow-file");
 830                 argv_array_push(&cmd.args, alternate_shallow_file);
 831         }
 832
 833         if (do_keep) {
 834                 if (pack_lockfile)
 835                         cmd.out = -1;
 836                 cmd_name = "index-pack";
 837                 argv_array_push(&cmd.args, cmd_name);
 838                 argv_array_push(&cmd.args, "--stdin");
 839                 if (!args->quiet && !args->no_progress)
 840                         argv_array_push(&cmd.args, "-v");
 841                 if (args->use_thin_pack)
 842                         argv_array_push(&cmd.args, "--fix-thin");
 843                 if (args->lock_pack || unpack_limit) {
 844                         char hostname[HOST_NAME_MAX + 1];
 845                         if (xgethostname(hostname, sizeof(hostname)))
 846                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 847                         argv_array_pushf(&cmd.args,
 848                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 849                                         (uintmax_t)getpid(), hostname);
 850                 }
 851                 if (args->check_self_contained_and_connected)
 852                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 853         }
 854         else {
 855                 cmd_name = "unpack-objects";
 856                 argv_array_push(&cmd.args, cmd_name);
 857                 if (args->quiet || args->no_progress)
 858                         argv_array_push(&cmd.args, "-q");
 859                 args->check_self_contained_and_connected = 0;
 860         }
 861
 862         if (pass_header)
 863                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 864                                  ntohl(header.hdr_version),
 865                                  ntohl(header.hdr_entries));
 866         if (fetch_fsck_objects >= 0
 867             ? fetch_fsck_objects
 868             : transfer_fsck_objects >= 0
 869             ? transfer_fsck_objects
 870             : 0)
 871                 argv_array_push(&cmd.args, "--strict");
 872
 873         cmd.in = demux.out;
 874         cmd.git_cmd = 1;
 875         if (start_command(&cmd))
 876                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 877         if (do_keep && pack_lockfile) {
 878                 *pack_lockfile = index_pack_lockfile(cmd.out);
 879                 close(cmd.out);
 880         }
 881
 882         if (!use_sideband)
 883                 /* Closed by start_command() */
 884                 xd[0] = -1;
 885
 886         ret = finish_command(&cmd);
 887         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 888                 args->self_contained_and_connected =
 889                         args->check_self_contained_and_connected &&
 890                         ret == 0;
 891         else
 892                 die(_("%s failed"), cmd_name);
 893         if (use_sideband && finish_async(&demux))
 894                 die(_("error in sideband demultiplexer"));
 895         return 0;
 896 }
 897
 898 static int cmp_ref_by_name(const void *a_, const void *b_)
 899 {
 900         const struct ref *a = *((const struct ref **)a_);
 901         const struct ref *b = *((const struct ref **)b_);
 902         return strcmp(a->name, b->name);
 903 }
 904
 905 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 906                                  int fd[2],
 907                                  const struct ref *orig_ref,
 908                                  struct ref **sought, int nr_sought,
 909                                  struct shallow_info *si,
 910                                  char **pack_lockfile)
 911 {
 912         struct ref *ref = copy_ref_list(orig_ref);
 913         struct object_id oid;
 914         const char *agent_feature;
 915         int agent_len;
 916
 917         sort_ref_list(&ref, ref_compare_name);
 918         QSORT(sought, nr_sought, cmp_ref_by_name);
 919
 920         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 921                 die(_("Server does not support shallow clients"));
 922         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 923                 args->deepen = 1;
 924         if (server_supports("multi_ack_detailed")) {
 925                 print_verbose(args, _("Server supports multi_ack_detailed"));
 926                 multi_ack = 2;
 927                 if (server_supports("no-done")) {
 928                         print_verbose(args, _("Server supports no-done"));
 929                         if (args->stateless_rpc)
 930                                 no_done = 1;
 931                 }
 932         }
 933         else if (server_supports("multi_ack")) {
 934                 print_verbose(args, _("Server supports multi_ack"));
 935                 multi_ack = 1;
 936         }
 937         if (server_supports("side-band-64k")) {
 938                 print_verbose(args, _("Server supports side-band-64k"));
 939                 use_sideband = 2;
 940         }
 941         else if (server_supports("side-band")) {
 942                 print_verbose(args, _("Server supports side-band"));
 943                 use_sideband = 1;
 944         }
 945         if (server_supports("allow-tip-sha1-in-want")) {
 946                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 947                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 948         }
 949         if (server_supports("allow-reachable-sha1-in-want")) {
 950                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 951                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 952         }
 953         if (!server_supports("thin-pack"))
 954                 args->use_thin_pack = 0;
 955         if (!server_supports("no-progress"))
 956                 args->no_progress = 0;
 957         if (!server_supports("include-tag"))
 958                 args->include_tag = 0;
 959         if (server_supports("ofs-delta"))
 960                 print_verbose(args, _("Server supports ofs-delta"));
 961         else
 962                 prefer_ofs_delta = 0;
 963
 964         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 965                 agent_supported = 1;
 966                 if (agent_len)
 967                         print_verbose(args, _("Server version is %.*s"),
 968                                       agent_len, agent_feature);
 969         }
 970         if (server_supports("deepen-since"))
 971                 deepen_since_ok = 1;
 972         else if (args->deepen_since)
 973                 die(_("Server does not support --shallow-since"));
 974         if (server_supports("deepen-not"))
 975                 deepen_not_ok = 1;
 976         else if (args->deepen_not)
 977                 die(_("Server does not support --shallow-exclude"));
 978         if (!server_supports("deepen-relative") && args->deepen_relative)
 979                 die(_("Server does not support --deepen"));
 980
 981         if (everything_local(args, &ref, sought, nr_sought)) {
 982                 packet_flush(fd[1]);
 983                 goto all_done;
 984         }
 985         if (find_common(args, fd, &oid, ref) < 0)
 986                 if (!args->keep_pack)
 987                         /* When cloning, it is not unusual to have
 988                          * no common commit.
 989                          */
 990                         warning(_("no common commits"));
 991
 992         if (args->stateless_rpc)
 993                 packet_flush(fd[1]);
 994         if (args->deepen)
 995                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
 996                                         NULL);
 997         else if (si->nr_ours || si->nr_theirs)
 998                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
 999         else
1000                 alternate_shallow_file = NULL;
1001         if (get_pack(args, fd, pack_lockfile))
1002                 die(_("git fetch-pack: fetch failed."));
1003
1004  all_done:
1005         return ref;
1006 }
1007
1008 static void fetch_pack_config(void)
1009 {
1010         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1011         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1012         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1013         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1014         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1015
1016         git_config(git_default_config, NULL);
1017 }
1018
1019 static void fetch_pack_setup(void)
1020 {
1021         static int did_setup;
1022         if (did_setup)
1023                 return;
1024         fetch_pack_config();
1025         if (0 <= transfer_unpack_limit)
1026                 unpack_limit = transfer_unpack_limit;
1027         else if (0 <= fetch_unpack_limit)
1028                 unpack_limit = fetch_unpack_limit;
1029         did_setup = 1;
1030 }
1031
1032 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1033 {
1034         struct string_list names = STRING_LIST_INIT_NODUP;
1035         int src, dst;
1036
1037         for (src = dst = 0; src < nr; src++) {
1038                 struct string_list_item *item;
1039                 item = string_list_insert(&names, ref[src]->name);
1040                 if (item->util)
1041                         continue; /* already have it */
1042                 item->util = ref[src];
1043                 if (src != dst)
1044                         ref[dst] = ref[src];
1045                 dst++;
1046         }
1047         for (src = dst; src < nr; src++)
1048                 ref[src] = NULL;
1049         string_list_clear(&names, 0);
1050         return dst;
1051 }
1052
1053 static void update_shallow(struct fetch_pack_args *args,
1054                            struct ref **sought, int nr_sought,
1055                            struct shallow_info *si)
1056 {
1057         struct oid_array ref = OID_ARRAY_INIT;
1058         int *status;
1059         int i;
1060
1061         if (args->deepen && alternate_shallow_file) {
1062                 if (*alternate_shallow_file == '\0') { /* --unshallow */
1063                         unlink_or_warn(git_path_shallow());
1064                         rollback_lock_file(&shallow_lock);
1065                 } else
1066                         commit_lock_file(&shallow_lock);
1067                 return;
1068         }
1069
1070         if (!si->shallow || !si->shallow->nr)
1071                 return;
1072
1073         if (args->cloning) {
1074                 /*
1075                  * remote is shallow, but this is a clone, there are
1076                  * no objects in repo to worry about. Accept any
1077                  * shallow points that exist in the pack (iow in repo
1078                  * after get_pack() and reprepare_packed_git())
1079                  */
1080                 struct oid_array extra = OID_ARRAY_INIT;
1081                 struct object_id *oid = si->shallow->oid;
1082                 for (i = 0; i < si->shallow->nr; i++)
1083                         if (has_object_file(&oid[i]))
1084                                 oid_array_append(&extra, &oid[i]);
1085                 if (extra.nr) {
1086                         setup_alternate_shallow(&shallow_lock,
1087                                                 &alternate_shallow_file,
1088                                                 &extra);
1089                         commit_lock_file(&shallow_lock);
1090                 }
1091                 oid_array_clear(&extra);
1092                 return;
1093         }
1094
1095         if (!si->nr_ours && !si->nr_theirs)
1096                 return;
1097
1098         remove_nonexistent_theirs_shallow(si);
1099         if (!si->nr_ours && !si->nr_theirs)
1100                 return;
1101         for (i = 0; i < nr_sought; i++)
1102                 oid_array_append(&ref, &sought[i]->old_oid);
1103         si->ref = &ref;
1104
1105         if (args->update_shallow) {
1106                 /*
1107                  * remote is also shallow, .git/shallow may be updated
1108                  * so all refs can be accepted. Make sure we only add
1109                  * shallow roots that are actually reachable from new
1110                  * refs.
1111                  */
1112                 struct oid_array extra = OID_ARRAY_INIT;
1113                 struct object_id *oid = si->shallow->oid;
1114                 assign_shallow_commits_to_refs(si, NULL, NULL);
1115                 if (!si->nr_ours && !si->nr_theirs) {
1116                         oid_array_clear(&ref);
1117                         return;
1118                 }
1119                 for (i = 0; i < si->nr_ours; i++)
1120                         oid_array_append(&extra, &oid[si->ours[i]]);
1121                 for (i = 0; i < si->nr_theirs; i++)
1122                         oid_array_append(&extra, &oid[si->theirs[i]]);
1123                 setup_alternate_shallow(&shallow_lock,
1124                                         &alternate_shallow_file,
1125                                         &extra);
1126                 commit_lock_file(&shallow_lock);
1127                 oid_array_clear(&extra);
1128                 oid_array_clear(&ref);
1129                 return;
1130         }
1131
1132         /*
1133          * remote is also shallow, check what ref is safe to update
1134          * without updating .git/shallow
1135          */
1136         status = xcalloc(nr_sought, sizeof(*status));
1137         assign_shallow_commits_to_refs(si, NULL, status);
1138         if (si->nr_ours || si->nr_theirs) {
1139                 for (i = 0; i < nr_sought; i++)
1140                         if (status[i])
1141                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1142         }
1143         free(status);
1144         oid_array_clear(&ref);
1145 }
1146
1147 struct ref *fetch_pack(struct fetch_pack_args *args,
1148                        int fd[], struct child_process *conn,
1149                        const struct ref *ref,
1150                        const char *dest,
1151                        struct ref **sought, int nr_sought,
1152                        struct oid_array *shallow,
1153                        char **pack_lockfile)
1154 {
1155         struct ref *ref_cpy;
1156         struct shallow_info si;
1157
1158         fetch_pack_setup();
1159         if (nr_sought)
1160                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1161
1162         if (!ref) {
1163                 packet_flush(fd[1]);
1164                 die(_("no matching remote head"));
1165         }
1166         prepare_shallow_info(&si, shallow);
1167         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1168                                 &si, pack_lockfile);
1169         reprepare_packed_git();
1170         update_shallow(args, sought, nr_sought, &si);
1171         clear_shallow_info(&si);
1172         return ref_cpy;
1173 }
1174
1175 int report_unmatched_refs(struct ref **sought, int nr_sought)
1176 {
1177         int i, ret = 0;
1178
1179         for (i = 0; i < nr_sought; i++) {
1180                 if (!sought[i])
1181                         continue;
1182                 switch (sought[i]->match_status) {
1183                 case REF_MATCHED:
1184                         continue;
1185                 case REF_NOT_MATCHED:
1186                         error(_("no such remote ref %s"), sought[i]->name);
1187                         break;
1188                 case REF_UNADVERTISED_NOT_ALLOWED:
1189                         error(_("Server does not allow request for unadvertised object %s"),
1190                               sought[i]->name);
1191                         break;
1192                 }
1193                 ret = 1;
1194         }
1195         return ret;
1196 }