git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "repository.h"
   3 #include "config.h"
   4 #include "lockfile.h"
   5 #include "refs.h"
   6 #include "pkt-line.h"
   7 #include "commit.h"
   8 #include "tag.h"
   9 #include "exec_cmd.h"
  10 #include "pack.h"
  11 #include "sideband.h"
  12 #include "fetch-pack.h"
  13 #include "remote.h"
  14 #include "run-command.h"
  15 #include "connect.h"
  16 #include "transport.h"
  17 #include "version.h"
  18 #include "prio-queue.h"
  19 #include "sha1-array.h"
  20 #include "oidset.h"
  21 #include "packfile.h"
  22
  23 static int transfer_unpack_limit = -1;
  24 static int fetch_unpack_limit = -1;
  25 static int unpack_limit = 100;
  26 static int prefer_ofs_delta = 1;
  27 static int no_done;
  28 static int deepen_since_ok;
  29 static int deepen_not_ok;
  30 static int fetch_fsck_objects = -1;
  31 static int transfer_fsck_objects = -1;
  32 static int agent_supported;
  33 static int server_supports_filtering;
  34 static struct lock_file shallow_lock;
  35 static const char *alternate_shallow_file;
  36
  37 /* Remember to update object flag allocation in object.h */
  38 #define COMPLETE        (1U << 0)
  39 #define COMMON          (1U << 1)
  40 #define COMMON_REF      (1U << 2)
  41 #define SEEN            (1U << 3)
  42 #define POPPED          (1U << 4)
  43 #define ALTERNATE       (1U << 5)
  44
  45 static int marked;
  46
  47 /*
  48  * After sending this many "have"s if we do not get any new ACK , we
  49  * give up traversing our history.
  50  */
  51 #define MAX_IN_VAIN 256
  52
  53 static struct prio_queue rev_list = { compare_commits_by_commit_date };
  54 static int non_common_revs, multi_ack, use_sideband;
  55 /* Allow specifying sha1 if it is a ref tip. */
  56 #define ALLOW_TIP_SHA1  01
  57 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  58 #define ALLOW_REACHABLE_SHA1    02
  59 static unsigned int allow_unadvertised_object_request;
  60
  61 __attribute__((format (printf, 2, 3)))
  62 static inline void print_verbose(const struct fetch_pack_args *args,
  63                                  const char *fmt, ...)
  64 {
  65         va_list params;
  66
  67         if (!args->verbose)
  68                 return;
  69
  70         va_start(params, fmt);
  71         vfprintf(stderr, fmt, params);
  72         va_end(params);
  73         fputc('\n', stderr);
  74 }
  75
  76 struct alternate_object_cache {
  77         struct object **items;
  78         size_t nr, alloc;
  79 };
  80
  81 static void cache_one_alternate(const char *refname,
  82                                 const struct object_id *oid,
  83                                 void *vcache)
  84 {
  85         struct alternate_object_cache *cache = vcache;
  86         struct object *obj = parse_object(oid);
  87
  88         if (!obj || (obj->flags & ALTERNATE))
  89                 return;
  90
  91         obj->flags |= ALTERNATE;
  92         ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  93         cache->items[cache->nr++] = obj;
  94 }
  95
  96 static void for_each_cached_alternate(void (*cb)(struct object *))
  97 {
  98         static int initialized;
  99         static struct alternate_object_cache cache;
 100         size_t i;
 101
 102         if (!initialized) {
 103                 for_each_alternate_ref(cache_one_alternate, &cache);
 104                 initialized = 1;
 105         }
 106
 107         for (i = 0; i < cache.nr; i++)
 108                 cb(cache.items[i]);
 109 }
 110
 111 static void rev_list_push(struct commit *commit, int mark)
 112 {
 113         if (!(commit->object.flags & mark)) {
 114                 commit->object.flags |= mark;
 115
 116                 if (parse_commit(commit))
 117                         return;
 118
 119                 prio_queue_put(&rev_list, commit);
 120
 121                 if (!(commit->object.flags & COMMON))
 122                         non_common_revs++;
 123         }
 124 }
 125
 126 static int rev_list_insert_ref(const char *refname, const struct object_id *oid)
 127 {
 128         struct object *o = deref_tag(parse_object(oid), refname, 0);
 129
 130         if (o && o->type == OBJ_COMMIT)
 131                 rev_list_push((struct commit *)o, SEEN);
 132
 133         return 0;
 134 }
 135
 136 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 137                                    int flag, void *cb_data)
 138 {
 139         return rev_list_insert_ref(refname, oid);
 140 }
 141
 142 static int clear_marks(const char *refname, const struct object_id *oid,
 143                        int flag, void *cb_data)
 144 {
 145         struct object *o = deref_tag(parse_object(oid), refname, 0);
 146
 147         if (o && o->type == OBJ_COMMIT)
 148                 clear_commit_marks((struct commit *)o,
 149                                    COMMON | COMMON_REF | SEEN | POPPED);
 150         return 0;
 151 }
 152
 153 /*
 154    This function marks a rev and its ancestors as common.
 155    In some cases, it is desirable to mark only the ancestors (for example
 156    when only the server does not yet know that they are common).
 157 */
 158
 159 static void mark_common(struct commit *commit,
 160                 int ancestors_only, int dont_parse)
 161 {
 162         if (commit != NULL && !(commit->object.flags & COMMON)) {
 163                 struct object *o = (struct object *)commit;
 164
 165                 if (!ancestors_only)
 166                         o->flags |= COMMON;
 167
 168                 if (!(o->flags & SEEN))
 169                         rev_list_push(commit, SEEN);
 170                 else {
 171                         struct commit_list *parents;
 172
 173                         if (!ancestors_only && !(o->flags & POPPED))
 174                                 non_common_revs--;
 175                         if (!o->parsed && !dont_parse)
 176                                 if (parse_commit(commit))
 177                                         return;
 178
 179                         for (parents = commit->parents;
 180                                         parents;
 181                                         parents = parents->next)
 182                                 mark_common(parents->item, 0, dont_parse);
 183                 }
 184         }
 185 }
 186
 187 /*
 188   Get the next rev to send, ignoring the common.
 189 */
 190
 191 static const struct object_id *get_rev(void)
 192 {
 193         struct commit *commit = NULL;
 194
 195         while (commit == NULL) {
 196                 unsigned int mark;
 197                 struct commit_list *parents;
 198
 199                 if (rev_list.nr == 0 || non_common_revs == 0)
 200                         return NULL;
 201
 202                 commit = prio_queue_get(&rev_list);
 203                 parse_commit(commit);
 204                 parents = commit->parents;
 205
 206                 commit->object.flags |= POPPED;
 207                 if (!(commit->object.flags & COMMON))
 208                         non_common_revs--;
 209
 210                 if (commit->object.flags & COMMON) {
 211                         /* do not send "have", and ignore ancestors */
 212                         commit = NULL;
 213                         mark = COMMON | SEEN;
 214                 } else if (commit->object.flags & COMMON_REF)
 215                         /* send "have", and ignore ancestors */
 216                         mark = COMMON | SEEN;
 217                 else
 218                         /* send "have", also for its ancestors */
 219                         mark = SEEN;
 220
 221                 while (parents) {
 222                         if (!(parents->item->object.flags & SEEN))
 223                                 rev_list_push(parents->item, mark);
 224                         if (mark & COMMON)
 225                                 mark_common(parents->item, 1, 0);
 226                         parents = parents->next;
 227                 }
 228         }
 229
 230         return &commit->object.oid;
 231 }
 232
 233 enum ack_type {
 234         NAK = 0,
 235         ACK,
 236         ACK_continue,
 237         ACK_common,
 238         ACK_ready
 239 };
 240
 241 static void consume_shallow_list(struct fetch_pack_args *args, int fd)
 242 {
 243         if (args->stateless_rpc && args->deepen) {
 244                 /* If we sent a depth we will get back "duplicate"
 245                  * shallow and unshallow commands every time there
 246                  * is a block of have lines exchanged.
 247                  */
 248                 char *line;
 249                 while ((line = packet_read_line(fd, NULL))) {
 250                         if (starts_with(line, "shallow "))
 251                                 continue;
 252                         if (starts_with(line, "unshallow "))
 253                                 continue;
 254                         die(_("git fetch-pack: expected shallow list"));
 255                 }
 256         }
 257 }
 258
 259 static enum ack_type get_ack(int fd, struct object_id *result_oid)
 260 {
 261         int len;
 262         char *line = packet_read_line(fd, &len);
 263         const char *arg;
 264
 265         if (!len)
 266                 die(_("git fetch-pack: expected ACK/NAK, got EOF"));
 267         if (!strcmp(line, "NAK"))
 268                 return NAK;
 269         if (skip_prefix(line, "ACK ", &arg)) {
 270                 if (!get_oid_hex(arg, result_oid)) {
 271                         arg += 40;
 272                         len -= arg - line;
 273                         if (len < 1)
 274                                 return ACK;
 275                         if (strstr(arg, "continue"))
 276                                 return ACK_continue;
 277                         if (strstr(arg, "common"))
 278                                 return ACK_common;
 279                         if (strstr(arg, "ready"))
 280                                 return ACK_ready;
 281                         return ACK;
 282                 }
 283         }
 284         if (skip_prefix(line, "ERR ", &arg))
 285                 die(_("remote error: %s"), arg);
 286         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), line);
 287 }
 288
 289 static void send_request(struct fetch_pack_args *args,
 290                          int fd, struct strbuf *buf)
 291 {
 292         if (args->stateless_rpc) {
 293                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 294                 packet_flush(fd);
 295         } else
 296                 write_or_die(fd, buf->buf, buf->len);
 297 }
 298
 299 static void insert_one_alternate_object(struct object *obj)
 300 {
 301         rev_list_insert_ref(NULL, &obj->oid);
 302 }
 303
 304 #define INITIAL_FLUSH 16
 305 #define PIPESAFE_FLUSH 32
 306 #define LARGE_FLUSH 16384
 307
 308 static int next_flush(struct fetch_pack_args *args, int count)
 309 {
 310         if (args->stateless_rpc) {
 311                 if (count < LARGE_FLUSH)
 312                         count <<= 1;
 313                 else
 314                         count = count * 11 / 10;
 315         } else {
 316                 if (count < PIPESAFE_FLUSH)
 317                         count <<= 1;
 318                 else
 319                         count += PIPESAFE_FLUSH;
 320         }
 321         return count;
 322 }
 323
 324 static int find_common(struct fetch_pack_args *args,
 325                        int fd[2], struct object_id *result_oid,
 326                        struct ref *refs)
 327 {
 328         int fetching;
 329         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 330         const struct object_id *oid;
 331         unsigned in_vain = 0;
 332         int got_continue = 0;
 333         int got_ready = 0;
 334         struct strbuf req_buf = STRBUF_INIT;
 335         size_t state_len = 0;
 336
 337         if (args->stateless_rpc && multi_ack == 1)
 338                 die(_("--stateless-rpc requires multi_ack_detailed"));
 339         if (marked)
 340                 for_each_ref(clear_marks, NULL);
 341         marked = 1;
 342
 343         for_each_ref(rev_list_insert_ref_oid, NULL);
 344         for_each_cached_alternate(insert_one_alternate_object);
 345
 346         fetching = 0;
 347         for ( ; refs ; refs = refs->next) {
 348                 struct object_id *remote = &refs->old_oid;
 349                 const char *remote_hex;
 350                 struct object *o;
 351
 352                 /*
 353                  * If that object is complete (i.e. it is an ancestor of a
 354                  * local ref), we tell them we have it but do not have to
 355                  * tell them about its ancestors, which they already know
 356                  * about.
 357                  *
 358                  * We use lookup_object here because we are only
 359                  * interested in the case we *know* the object is
 360                  * reachable and we have already scanned it.
 361                  */
 362                 if (((o = lookup_object(remote->hash)) != NULL) &&
 363                                 (o->flags & COMPLETE)) {
 364                         continue;
 365                 }
 366
 367                 remote_hex = oid_to_hex(remote);
 368                 if (!fetching) {
 369                         struct strbuf c = STRBUF_INIT;
 370                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 371                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 372                         if (no_done)            strbuf_addstr(&c, " no-done");
 373                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 374                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 375                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 376                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 377                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 378                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 379                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 380                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 381                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 382                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 383                                                             git_user_agent_sanitized());
 384                         if (args->filter_options.choice)
 385                                 strbuf_addstr(&c, " filter");
 386                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 387                         strbuf_release(&c);
 388                 } else
 389                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 390                 fetching++;
 391         }
 392
 393         if (!fetching) {
 394                 strbuf_release(&req_buf);
 395                 packet_flush(fd[1]);
 396                 return 1;
 397         }
 398
 399         if (is_repository_shallow())
 400                 write_shallow_commits(&req_buf, 1, NULL);
 401         if (args->depth > 0)
 402                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 403         if (args->deepen_since) {
 404                 timestamp_t max_age = approxidate(args->deepen_since);
 405                 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 406         }
 407         if (args->deepen_not) {
 408                 int i;
 409                 for (i = 0; i < args->deepen_not->nr; i++) {
 410                         struct string_list_item *s = args->deepen_not->items + i;
 411                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 412                 }
 413         }
 414         if (server_supports_filtering && args->filter_options.choice)
 415                 packet_buf_write(&req_buf, "filter %s",
 416                                  args->filter_options.filter_spec);
 417         packet_buf_flush(&req_buf);
 418         state_len = req_buf.len;
 419
 420         if (args->deepen) {
 421                 char *line;
 422                 const char *arg;
 423                 struct object_id oid;
 424
 425                 send_request(args, fd[1], &req_buf);
 426                 while ((line = packet_read_line(fd[0], NULL))) {
 427                         if (skip_prefix(line, "shallow ", &arg)) {
 428                                 if (get_oid_hex(arg, &oid))
 429                                         die(_("invalid shallow line: %s"), line);
 430                                 register_shallow(&oid);
 431                                 continue;
 432                         }
 433                         if (skip_prefix(line, "unshallow ", &arg)) {
 434                                 if (get_oid_hex(arg, &oid))
 435                                         die(_("invalid unshallow line: %s"), line);
 436                                 if (!lookup_object(oid.hash))
 437                                         die(_("object not found: %s"), line);
 438                                 /* make sure that it is parsed as shallow */
 439                                 if (!parse_object(&oid))
 440                                         die(_("error in object: %s"), line);
 441                                 if (unregister_shallow(&oid))
 442                                         die(_("no shallow found: %s"), line);
 443                                 continue;
 444                         }
 445                         die(_("expected shallow/unshallow, got %s"), line);
 446                 }
 447         } else if (!args->stateless_rpc)
 448                 send_request(args, fd[1], &req_buf);
 449
 450         if (!args->stateless_rpc) {
 451                 /* If we aren't using the stateless-rpc interface
 452                  * we don't need to retain the headers.
 453                  */
 454                 strbuf_setlen(&req_buf, 0);
 455                 state_len = 0;
 456         }
 457
 458         flushes = 0;
 459         retval = -1;
 460         if (args->no_dependents)
 461                 goto done;
 462         while ((oid = get_rev())) {
 463                 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 464                 print_verbose(args, "have %s", oid_to_hex(oid));
 465                 in_vain++;
 466                 if (flush_at <= ++count) {
 467                         int ack;
 468
 469                         packet_buf_flush(&req_buf);
 470                         send_request(args, fd[1], &req_buf);
 471                         strbuf_setlen(&req_buf, state_len);
 472                         flushes++;
 473                         flush_at = next_flush(args, count);
 474
 475                         /*
 476                          * We keep one window "ahead" of the other side, and
 477                          * will wait for an ACK only on the next one
 478                          */
 479                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 480                                 continue;
 481
 482                         consume_shallow_list(args, fd[0]);
 483                         do {
 484                                 ack = get_ack(fd[0], result_oid);
 485                                 if (ack)
 486                                         print_verbose(args, _("got %s %d %s"), "ack",
 487                                                       ack, oid_to_hex(result_oid));
 488                                 switch (ack) {
 489                                 case ACK:
 490                                         flushes = 0;
 491                                         multi_ack = 0;
 492                                         retval = 0;
 493                                         goto done;
 494                                 case ACK_common:
 495                                 case ACK_ready:
 496                                 case ACK_continue: {
 497                                         struct commit *commit =
 498                                                 lookup_commit(result_oid);
 499                                         if (!commit)
 500                                                 die(_("invalid commit %s"), oid_to_hex(result_oid));
 501                                         if (args->stateless_rpc
 502                                          && ack == ACK_common
 503                                          && !(commit->object.flags & COMMON)) {
 504                                                 /* We need to replay the have for this object
 505                                                  * on the next RPC request so the peer knows
 506                                                  * it is in common with us.
 507                                                  */
 508                                                 const char *hex = oid_to_hex(result_oid);
 509                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 510                                                 state_len = req_buf.len;
 511                                                 /*
 512                                                  * Reset in_vain because an ack
 513                                                  * for this commit has not been
 514                                                  * seen.
 515                                                  */
 516                                                 in_vain = 0;
 517                                         } else if (!args->stateless_rpc
 518                                                    || ack != ACK_common)
 519                                                 in_vain = 0;
 520                                         mark_common(commit, 0, 1);
 521                                         retval = 0;
 522                                         got_continue = 1;
 523                                         if (ack == ACK_ready) {
 524                                                 clear_prio_queue(&rev_list);
 525                                                 got_ready = 1;
 526                                         }
 527                                         break;
 528                                         }
 529                                 }
 530                         } while (ack);
 531                         flushes--;
 532                         if (got_continue && MAX_IN_VAIN < in_vain) {
 533                                 print_verbose(args, _("giving up"));
 534                                 break; /* give up */
 535                         }
 536                 }
 537         }
 538 done:
 539         if (!got_ready || !no_done) {
 540                 packet_buf_write(&req_buf, "done\n");
 541                 send_request(args, fd[1], &req_buf);
 542         }
 543         print_verbose(args, _("done"));
 544         if (retval != 0) {
 545                 multi_ack = 0;
 546                 flushes++;
 547         }
 548         strbuf_release(&req_buf);
 549
 550         if (!got_ready || !no_done)
 551                 consume_shallow_list(args, fd[0]);
 552         while (flushes || multi_ack) {
 553                 int ack = get_ack(fd[0], result_oid);
 554                 if (ack) {
 555                         print_verbose(args, _("got %s (%d) %s"), "ack",
 556                                       ack, oid_to_hex(result_oid));
 557                         if (ack == ACK)
 558                                 return 0;
 559                         multi_ack = 1;
 560                         continue;
 561                 }
 562                 flushes--;
 563         }
 564         /* it is no error to fetch into a completely empty repo */
 565         return count ? retval : 0;
 566 }
 567
 568 static struct commit_list *complete;
 569
 570 static int mark_complete(const struct object_id *oid)
 571 {
 572         struct object *o = parse_object(oid);
 573
 574         while (o && o->type == OBJ_TAG) {
 575                 struct tag *t = (struct tag *) o;
 576                 if (!t->tagged)
 577                         break; /* broken repository */
 578                 o->flags |= COMPLETE;
 579                 o = parse_object(&t->tagged->oid);
 580         }
 581         if (o && o->type == OBJ_COMMIT) {
 582                 struct commit *commit = (struct commit *)o;
 583                 if (!(commit->object.flags & COMPLETE)) {
 584                         commit->object.flags |= COMPLETE;
 585                         commit_list_insert(commit, &complete);
 586                 }
 587         }
 588         return 0;
 589 }
 590
 591 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 592                              int flag, void *cb_data)
 593 {
 594         return mark_complete(oid);
 595 }
 596
 597 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 598                                          timestamp_t cutoff)
 599 {
 600         while (complete && cutoff <= complete->item->date) {
 601                 print_verbose(args, _("Marking %s as complete"),
 602                               oid_to_hex(&complete->item->object.oid));
 603                 pop_most_recent_commit(&complete, COMPLETE);
 604         }
 605 }
 606
 607 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 608 {
 609         for (; refs; refs = refs->next)
 610                 oidset_insert(oids, &refs->old_oid);
 611 }
 612
 613 static int tip_oids_contain(struct oidset *tip_oids,
 614                             struct ref *unmatched, struct ref *newlist,
 615                             const struct object_id *id)
 616 {
 617         /*
 618          * Note that this only looks at the ref lists the first time it's
 619          * called. This works out in filter_refs() because even though it may
 620          * add to "newlist" between calls, the additions will always be for
 621          * oids that are already in the set.
 622          */
 623         if (!tip_oids->map.map.tablesize) {
 624                 add_refs_to_oidset(tip_oids, unmatched);
 625                 add_refs_to_oidset(tip_oids, newlist);
 626         }
 627         return oidset_contains(tip_oids, id);
 628 }
 629
 630 static void filter_refs(struct fetch_pack_args *args,
 631                         struct ref **refs,
 632                         struct ref **sought, int nr_sought)
 633 {
 634         struct ref *newlist = NULL;
 635         struct ref **newtail = &newlist;
 636         struct ref *unmatched = NULL;
 637         struct ref *ref, *next;
 638         struct oidset tip_oids = OIDSET_INIT;
 639         int i;
 640
 641         i = 0;
 642         for (ref = *refs; ref; ref = next) {
 643                 int keep = 0;
 644                 next = ref->next;
 645
 646                 if (starts_with(ref->name, "refs/") &&
 647                     check_refname_format(ref->name, 0))
 648                         ; /* trash */
 649                 else {
 650                         while (i < nr_sought) {
 651                                 int cmp = strcmp(ref->name, sought[i]->name);
 652                                 if (cmp < 0)
 653                                         break; /* definitely do not have it */
 654                                 else if (cmp == 0) {
 655                                         keep = 1; /* definitely have it */
 656                                         sought[i]->match_status = REF_MATCHED;
 657                                 }
 658                                 i++;
 659                         }
 660                 }
 661
 662                 if (!keep && args->fetch_all &&
 663                     (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 664                         keep = 1;
 665
 666                 if (keep) {
 667                         *newtail = ref;
 668                         ref->next = NULL;
 669                         newtail = &ref->next;
 670                 } else {
 671                         ref->next = unmatched;
 672                         unmatched = ref;
 673                 }
 674         }
 675
 676         /* Append unmatched requests to the list */
 677         for (i = 0; i < nr_sought; i++) {
 678                 struct object_id oid;
 679                 const char *p;
 680
 681                 ref = sought[i];
 682                 if (ref->match_status != REF_NOT_MATCHED)
 683                         continue;
 684                 if (parse_oid_hex(ref->name, &oid, &p) ||
 685                     *p != '\0' ||
 686                     oidcmp(&oid, &ref->old_oid))
 687                         continue;
 688
 689                 if ((allow_unadvertised_object_request &
 690                      (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1)) ||
 691                     tip_oids_contain(&tip_oids, unmatched, newlist,
 692                                      &ref->old_oid)) {
 693                         ref->match_status = REF_MATCHED;
 694                         *newtail = copy_ref(ref);
 695                         newtail = &(*newtail)->next;
 696                 } else {
 697                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 698                 }
 699         }
 700
 701         oidset_clear(&tip_oids);
 702         for (ref = unmatched; ref; ref = next) {
 703                 next = ref->next;
 704                 free(ref);
 705         }
 706
 707         *refs = newlist;
 708 }
 709
 710 static void mark_alternate_complete(struct object *obj)
 711 {
 712         mark_complete(&obj->oid);
 713 }
 714
 715 static int everything_local(struct fetch_pack_args *args,
 716                             struct ref **refs,
 717                             struct ref **sought, int nr_sought)
 718 {
 719         struct ref *ref;
 720         int retval;
 721         int old_save_commit_buffer = save_commit_buffer;
 722         timestamp_t cutoff = 0;
 723
 724         save_commit_buffer = 0;
 725
 726         for (ref = *refs; ref; ref = ref->next) {
 727                 struct object *o;
 728
 729                 if (!has_object_file_with_flags(&ref->old_oid,
 730                                                 OBJECT_INFO_QUICK))
 731                         continue;
 732
 733                 o = parse_object(&ref->old_oid);
 734                 if (!o)
 735                         continue;
 736
 737                 /* We already have it -- which may mean that we were
 738                  * in sync with the other side at some time after
 739                  * that (it is OK if we guess wrong here).
 740                  */
 741                 if (o->type == OBJ_COMMIT) {
 742                         struct commit *commit = (struct commit *)o;
 743                         if (!cutoff || cutoff < commit->date)
 744                                 cutoff = commit->date;
 745                 }
 746         }
 747
 748         if (!args->no_dependents) {
 749                 if (!args->deepen) {
 750                         for_each_ref(mark_complete_oid, NULL);
 751                         for_each_cached_alternate(mark_alternate_complete);
 752                         commit_list_sort_by_date(&complete);
 753                         if (cutoff)
 754                                 mark_recent_complete_commits(args, cutoff);
 755                 }
 756
 757                 /*
 758                  * Mark all complete remote refs as common refs.
 759                  * Don't mark them common yet; the server has to be told so first.
 760                  */
 761                 for (ref = *refs; ref; ref = ref->next) {
 762                         struct object *o = deref_tag(lookup_object(ref->old_oid.hash),
 763                                                      NULL, 0);
 764
 765                         if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 766                                 continue;
 767
 768                         if (!(o->flags & SEEN)) {
 769                                 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
 770
 771                                 mark_common((struct commit *)o, 1, 1);
 772                         }
 773                 }
 774         }
 775
 776         filter_refs(args, refs, sought, nr_sought);
 777
 778         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 779                 const struct object_id *remote = &ref->old_oid;
 780                 struct object *o;
 781
 782                 o = lookup_object(remote->hash);
 783                 if (!o || !(o->flags & COMPLETE)) {
 784                         retval = 0;
 785                         print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 786                                       ref->name);
 787                         continue;
 788                 }
 789                 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 790                               ref->name);
 791         }
 792
 793         save_commit_buffer = old_save_commit_buffer;
 794
 795         return retval;
 796 }
 797
 798 static int sideband_demux(int in, int out, void *data)
 799 {
 800         int *xd = data;
 801         int ret;
 802
 803         ret = recv_sideband("fetch-pack", xd[0], out);
 804         close(out);
 805         return ret;
 806 }
 807
 808 static int get_pack(struct fetch_pack_args *args,
 809                     int xd[2], char **pack_lockfile)
 810 {
 811         struct async demux;
 812         int do_keep = args->keep_pack;
 813         const char *cmd_name;
 814         struct pack_header header;
 815         int pass_header = 0;
 816         struct child_process cmd = CHILD_PROCESS_INIT;
 817         int ret;
 818
 819         memset(&demux, 0, sizeof(demux));
 820         if (use_sideband) {
 821                 /* xd[] is talking with upload-pack; subprocess reads from
 822                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 823                  * through demux->out.
 824                  */
 825                 demux.proc = sideband_demux;
 826                 demux.data = xd;
 827                 demux.out = -1;
 828                 demux.isolate_sigpipe = 1;
 829                 if (start_async(&demux))
 830                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 831         }
 832         else
 833                 demux.out = xd[0];
 834
 835         if (!args->keep_pack && unpack_limit) {
 836
 837                 if (read_pack_header(demux.out, &header))
 838                         die(_("protocol error: bad pack header"));
 839                 pass_header = 1;
 840                 if (ntohl(header.hdr_entries) < unpack_limit)
 841                         do_keep = 0;
 842                 else
 843                         do_keep = 1;
 844         }
 845
 846         if (alternate_shallow_file) {
 847                 argv_array_push(&cmd.args, "--shallow-file");
 848                 argv_array_push(&cmd.args, alternate_shallow_file);
 849         }
 850
 851         if (do_keep || args->from_promisor) {
 852                 if (pack_lockfile)
 853                         cmd.out = -1;
 854                 cmd_name = "index-pack";
 855                 argv_array_push(&cmd.args, cmd_name);
 856                 argv_array_push(&cmd.args, "--stdin");
 857                 if (!args->quiet && !args->no_progress)
 858                         argv_array_push(&cmd.args, "-v");
 859                 if (args->use_thin_pack)
 860                         argv_array_push(&cmd.args, "--fix-thin");
 861                 if (do_keep && (args->lock_pack || unpack_limit)) {
 862                         char hostname[HOST_NAME_MAX + 1];
 863                         if (xgethostname(hostname, sizeof(hostname)))
 864                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 865                         argv_array_pushf(&cmd.args,
 866                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 867                                         (uintmax_t)getpid(), hostname);
 868                 }
 869                 if (args->check_self_contained_and_connected)
 870                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 871                 if (args->from_promisor)
 872                         argv_array_push(&cmd.args, "--promisor");
 873         }
 874         else {
 875                 cmd_name = "unpack-objects";
 876                 argv_array_push(&cmd.args, cmd_name);
 877                 if (args->quiet || args->no_progress)
 878                         argv_array_push(&cmd.args, "-q");
 879                 args->check_self_contained_and_connected = 0;
 880         }
 881
 882         if (pass_header)
 883                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 884                                  ntohl(header.hdr_version),
 885                                  ntohl(header.hdr_entries));
 886         if (fetch_fsck_objects >= 0
 887             ? fetch_fsck_objects
 888             : transfer_fsck_objects >= 0
 889             ? transfer_fsck_objects
 890             : 0)
 891                 argv_array_push(&cmd.args, "--strict");
 892
 893         cmd.in = demux.out;
 894         cmd.git_cmd = 1;
 895         if (start_command(&cmd))
 896                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 897         if (do_keep && pack_lockfile) {
 898                 *pack_lockfile = index_pack_lockfile(cmd.out);
 899                 close(cmd.out);
 900         }
 901
 902         if (!use_sideband)
 903                 /* Closed by start_command() */
 904                 xd[0] = -1;
 905
 906         ret = finish_command(&cmd);
 907         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 908                 args->self_contained_and_connected =
 909                         args->check_self_contained_and_connected &&
 910                         ret == 0;
 911         else
 912                 die(_("%s failed"), cmd_name);
 913         if (use_sideband && finish_async(&demux))
 914                 die(_("error in sideband demultiplexer"));
 915         return 0;
 916 }
 917
 918 static int cmp_ref_by_name(const void *a_, const void *b_)
 919 {
 920         const struct ref *a = *((const struct ref **)a_);
 921         const struct ref *b = *((const struct ref **)b_);
 922         return strcmp(a->name, b->name);
 923 }
 924
 925 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 926                                  int fd[2],
 927                                  const struct ref *orig_ref,
 928                                  struct ref **sought, int nr_sought,
 929                                  struct shallow_info *si,
 930                                  char **pack_lockfile)
 931 {
 932         struct ref *ref = copy_ref_list(orig_ref);
 933         struct object_id oid;
 934         const char *agent_feature;
 935         int agent_len;
 936
 937         sort_ref_list(&ref, ref_compare_name);
 938         QSORT(sought, nr_sought, cmp_ref_by_name);
 939
 940         if ((args->depth > 0 || is_repository_shallow()) && !server_supports("shallow"))
 941                 die(_("Server does not support shallow clients"));
 942         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 943                 args->deepen = 1;
 944         if (server_supports("multi_ack_detailed")) {
 945                 print_verbose(args, _("Server supports multi_ack_detailed"));
 946                 multi_ack = 2;
 947                 if (server_supports("no-done")) {
 948                         print_verbose(args, _("Server supports no-done"));
 949                         if (args->stateless_rpc)
 950                                 no_done = 1;
 951                 }
 952         }
 953         else if (server_supports("multi_ack")) {
 954                 print_verbose(args, _("Server supports multi_ack"));
 955                 multi_ack = 1;
 956         }
 957         if (server_supports("side-band-64k")) {
 958                 print_verbose(args, _("Server supports side-band-64k"));
 959                 use_sideband = 2;
 960         }
 961         else if (server_supports("side-band")) {
 962                 print_verbose(args, _("Server supports side-band"));
 963                 use_sideband = 1;
 964         }
 965         if (server_supports("allow-tip-sha1-in-want")) {
 966                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 967                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 968         }
 969         if (server_supports("allow-reachable-sha1-in-want")) {
 970                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 971                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 972         }
 973         if (!server_supports("thin-pack"))
 974                 args->use_thin_pack = 0;
 975         if (!server_supports("no-progress"))
 976                 args->no_progress = 0;
 977         if (!server_supports("include-tag"))
 978                 args->include_tag = 0;
 979         if (server_supports("ofs-delta"))
 980                 print_verbose(args, _("Server supports ofs-delta"));
 981         else
 982                 prefer_ofs_delta = 0;
 983
 984         if (server_supports("filter")) {
 985                 server_supports_filtering = 1;
 986                 print_verbose(args, _("Server supports filter"));
 987         } else if (args->filter_options.choice) {
 988                 warning("filtering not recognized by server, ignoring");
 989         }
 990
 991         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 992                 agent_supported = 1;
 993                 if (agent_len)
 994                         print_verbose(args, _("Server version is %.*s"),
 995                                       agent_len, agent_feature);
 996         }
 997         if (server_supports("deepen-since"))
 998                 deepen_since_ok = 1;
 999         else if (args->deepen_since)
1000                 die(_("Server does not support --shallow-since"));
1001         if (server_supports("deepen-not"))
1002                 deepen_not_ok = 1;
1003         else if (args->deepen_not)
1004                 die(_("Server does not support --shallow-exclude"));
1005         if (!server_supports("deepen-relative") && args->deepen_relative)
1006                 die(_("Server does not support --deepen"));
1007
1008         if (everything_local(args, &ref, sought, nr_sought)) {
1009                 packet_flush(fd[1]);
1010                 goto all_done;
1011         }
1012         if (find_common(args, fd, &oid, ref) < 0)
1013                 if (!args->keep_pack)
1014                         /* When cloning, it is not unusual to have
1015                          * no common commit.
1016                          */
1017                         warning(_("no common commits"));
1018
1019         if (args->stateless_rpc)
1020                 packet_flush(fd[1]);
1021         if (args->deepen)
1022                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1023                                         NULL);
1024         else if (si->nr_ours || si->nr_theirs)
1025                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1026         else
1027                 alternate_shallow_file = NULL;
1028         if (get_pack(args, fd, pack_lockfile))
1029                 die(_("git fetch-pack: fetch failed."));
1030
1031  all_done:
1032         return ref;
1033 }
1034
1035 static void fetch_pack_config(void)
1036 {
1037         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1038         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1039         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1040         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1041         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1042
1043         git_config(git_default_config, NULL);
1044 }
1045
1046 static void fetch_pack_setup(void)
1047 {
1048         static int did_setup;
1049         if (did_setup)
1050                 return;
1051         fetch_pack_config();
1052         if (0 <= transfer_unpack_limit)
1053                 unpack_limit = transfer_unpack_limit;
1054         else if (0 <= fetch_unpack_limit)
1055                 unpack_limit = fetch_unpack_limit;
1056         did_setup = 1;
1057 }
1058
1059 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1060 {
1061         struct string_list names = STRING_LIST_INIT_NODUP;
1062         int src, dst;
1063
1064         for (src = dst = 0; src < nr; src++) {
1065                 struct string_list_item *item;
1066                 item = string_list_insert(&names, ref[src]->name);
1067                 if (item->util)
1068                         continue; /* already have it */
1069                 item->util = ref[src];
1070                 if (src != dst)
1071                         ref[dst] = ref[src];
1072                 dst++;
1073         }
1074         for (src = dst; src < nr; src++)
1075                 ref[src] = NULL;
1076         string_list_clear(&names, 0);
1077         return dst;
1078 }
1079
1080 static void update_shallow(struct fetch_pack_args *args,
1081                            struct ref **sought, int nr_sought,
1082                            struct shallow_info *si)
1083 {
1084         struct oid_array ref = OID_ARRAY_INIT;
1085         int *status;
1086         int i;
1087
1088         if (args->deepen && alternate_shallow_file) {
1089                 if (*alternate_shallow_file == '\0') { /* --unshallow */
1090                         unlink_or_warn(git_path_shallow());
1091                         rollback_lock_file(&shallow_lock);
1092                 } else
1093                         commit_lock_file(&shallow_lock);
1094                 return;
1095         }
1096
1097         if (!si->shallow || !si->shallow->nr)
1098                 return;
1099
1100         if (args->cloning) {
1101                 /*
1102                  * remote is shallow, but this is a clone, there are
1103                  * no objects in repo to worry about. Accept any
1104                  * shallow points that exist in the pack (iow in repo
1105                  * after get_pack() and reprepare_packed_git())
1106                  */
1107                 struct oid_array extra = OID_ARRAY_INIT;
1108                 struct object_id *oid = si->shallow->oid;
1109                 for (i = 0; i < si->shallow->nr; i++)
1110                         if (has_object_file(&oid[i]))
1111                                 oid_array_append(&extra, &oid[i]);
1112                 if (extra.nr) {
1113                         setup_alternate_shallow(&shallow_lock,
1114                                                 &alternate_shallow_file,
1115                                                 &extra);
1116                         commit_lock_file(&shallow_lock);
1117                 }
1118                 oid_array_clear(&extra);
1119                 return;
1120         }
1121
1122         if (!si->nr_ours && !si->nr_theirs)
1123                 return;
1124
1125         remove_nonexistent_theirs_shallow(si);
1126         if (!si->nr_ours && !si->nr_theirs)
1127                 return;
1128         for (i = 0; i < nr_sought; i++)
1129                 oid_array_append(&ref, &sought[i]->old_oid);
1130         si->ref = &ref;
1131
1132         if (args->update_shallow) {
1133                 /*
1134                  * remote is also shallow, .git/shallow may be updated
1135                  * so all refs can be accepted. Make sure we only add
1136                  * shallow roots that are actually reachable from new
1137                  * refs.
1138                  */
1139                 struct oid_array extra = OID_ARRAY_INIT;
1140                 struct object_id *oid = si->shallow->oid;
1141                 assign_shallow_commits_to_refs(si, NULL, NULL);
1142                 if (!si->nr_ours && !si->nr_theirs) {
1143                         oid_array_clear(&ref);
1144                         return;
1145                 }
1146                 for (i = 0; i < si->nr_ours; i++)
1147                         oid_array_append(&extra, &oid[si->ours[i]]);
1148                 for (i = 0; i < si->nr_theirs; i++)
1149                         oid_array_append(&extra, &oid[si->theirs[i]]);
1150                 setup_alternate_shallow(&shallow_lock,
1151                                         &alternate_shallow_file,
1152                                         &extra);
1153                 commit_lock_file(&shallow_lock);
1154                 oid_array_clear(&extra);
1155                 oid_array_clear(&ref);
1156                 return;
1157         }
1158
1159         /*
1160          * remote is also shallow, check what ref is safe to update
1161          * without updating .git/shallow
1162          */
1163         status = xcalloc(nr_sought, sizeof(*status));
1164         assign_shallow_commits_to_refs(si, NULL, status);
1165         if (si->nr_ours || si->nr_theirs) {
1166                 for (i = 0; i < nr_sought; i++)
1167                         if (status[i])
1168                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1169         }
1170         free(status);
1171         oid_array_clear(&ref);
1172 }
1173
1174 struct ref *fetch_pack(struct fetch_pack_args *args,
1175                        int fd[], struct child_process *conn,
1176                        const struct ref *ref,
1177                        const char *dest,
1178                        struct ref **sought, int nr_sought,
1179                        struct oid_array *shallow,
1180                        char **pack_lockfile)
1181 {
1182         struct ref *ref_cpy;
1183         struct shallow_info si;
1184
1185         fetch_pack_setup();
1186         if (nr_sought)
1187                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1188
1189         if (!ref) {
1190                 packet_flush(fd[1]);
1191                 die(_("no matching remote head"));
1192         }
1193         prepare_shallow_info(&si, shallow);
1194         ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1195                                 &si, pack_lockfile);
1196         reprepare_packed_git(the_repository);
1197         update_shallow(args, sought, nr_sought, &si);
1198         clear_shallow_info(&si);
1199         return ref_cpy;
1200 }
1201
1202 int report_unmatched_refs(struct ref **sought, int nr_sought)
1203 {
1204         int i, ret = 0;
1205
1206         for (i = 0; i < nr_sought; i++) {
1207                 if (!sought[i])
1208                         continue;
1209                 switch (sought[i]->match_status) {
1210                 case REF_MATCHED:
1211                         continue;
1212                 case REF_NOT_MATCHED:
1213                         error(_("no such remote ref %s"), sought[i]->name);
1214                         break;
1215                 case REF_UNADVERTISED_NOT_ALLOWED:
1216                         error(_("Server does not allow request for unadvertised object %s"),
1217                               sought[i]->name);
1218                         break;
1219                 }
1220                 ret = 1;
1221         }
1222         return ret;
1223 }