git.oblomov.eu Git - git/blob - fetch-pack.c

   1 #include "cache.h"
   2 #include "repository.h"
   3 #include "config.h"
   4 #include "lockfile.h"
   5 #include "refs.h"
   6 #include "pkt-line.h"
   7 #include "commit.h"
   8 #include "tag.h"
   9 #include "exec-cmd.h"
  10 #include "pack.h"
  11 #include "sideband.h"
  12 #include "fetch-pack.h"
  13 #include "remote.h"
  14 #include "run-command.h"
  15 #include "connect.h"
  16 #include "transport.h"
  17 #include "version.h"
  18 #include "sha1-array.h"
  19 #include "oidset.h"
  20 #include "packfile.h"
  21 #include "object-store.h"
  22 #include "connected.h"
  23 #include "fetch-negotiator.h"
  24 #include "fsck.h"
  25
  26 static int transfer_unpack_limit = -1;
  27 static int fetch_unpack_limit = -1;
  28 static int unpack_limit = 100;
  29 static int prefer_ofs_delta = 1;
  30 static int no_done;
  31 static int deepen_since_ok;
  32 static int deepen_not_ok;
  33 static int fetch_fsck_objects = -1;
  34 static int transfer_fsck_objects = -1;
  35 static int agent_supported;
  36 static int server_supports_filtering;
  37 static struct lock_file shallow_lock;
  38 static const char *alternate_shallow_file;
  39 static char *negotiation_algorithm;
  40 static struct strbuf fsck_msg_types = STRBUF_INIT;
  41
  42 /* Remember to update object flag allocation in object.h */
  43 #define COMPLETE        (1U << 0)
  44 #define ALTERNATE       (1U << 1)
  45
  46 /*
  47  * After sending this many "have"s if we do not get any new ACK , we
  48  * give up traversing our history.
  49  */
  50 #define MAX_IN_VAIN 256
  51
  52 static int multi_ack, use_sideband;
  53 /* Allow specifying sha1 if it is a ref tip. */
  54 #define ALLOW_TIP_SHA1  01
  55 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
  56 #define ALLOW_REACHABLE_SHA1    02
  57 static unsigned int allow_unadvertised_object_request;
  58
  59 __attribute__((format (printf, 2, 3)))
  60 static inline void print_verbose(const struct fetch_pack_args *args,
  61                                  const char *fmt, ...)
  62 {
  63         va_list params;
  64
  65         if (!args->verbose)
  66                 return;
  67
  68         va_start(params, fmt);
  69         vfprintf(stderr, fmt, params);
  70         va_end(params);
  71         fputc('\n', stderr);
  72 }
  73
  74 struct alternate_object_cache {
  75         struct object **items;
  76         size_t nr, alloc;
  77 };
  78
  79 static void cache_one_alternate(const struct object_id *oid,
  80                                 void *vcache)
  81 {
  82         struct alternate_object_cache *cache = vcache;
  83         struct object *obj = parse_object(the_repository, oid);
  84
  85         if (!obj || (obj->flags & ALTERNATE))
  86                 return;
  87
  88         obj->flags |= ALTERNATE;
  89         ALLOC_GROW(cache->items, cache->nr + 1, cache->alloc);
  90         cache->items[cache->nr++] = obj;
  91 }
  92
  93 static void for_each_cached_alternate(struct fetch_negotiator *negotiator,
  94                                       void (*cb)(struct fetch_negotiator *,
  95                                                  struct object *))
  96 {
  97         static int initialized;
  98         static struct alternate_object_cache cache;
  99         size_t i;
 100
 101         if (!initialized) {
 102                 for_each_alternate_ref(cache_one_alternate, &cache);
 103                 initialized = 1;
 104         }
 105
 106         for (i = 0; i < cache.nr; i++)
 107                 cb(negotiator, cache.items[i]);
 108 }
 109
 110 static int rev_list_insert_ref(struct fetch_negotiator *negotiator,
 111                                const char *refname,
 112                                const struct object_id *oid)
 113 {
 114         struct object *o = deref_tag(the_repository,
 115                                      parse_object(the_repository, oid),
 116                                      refname, 0);
 117
 118         if (o && o->type == OBJ_COMMIT)
 119                 negotiator->add_tip(negotiator, (struct commit *)o);
 120
 121         return 0;
 122 }
 123
 124 static int rev_list_insert_ref_oid(const char *refname, const struct object_id *oid,
 125                                    int flag, void *cb_data)
 126 {
 127         return rev_list_insert_ref(cb_data, refname, oid);
 128 }
 129
 130 enum ack_type {
 131         NAK = 0,
 132         ACK,
 133         ACK_continue,
 134         ACK_common,
 135         ACK_ready
 136 };
 137
 138 static void consume_shallow_list(struct fetch_pack_args *args,
 139                                  struct packet_reader *reader)
 140 {
 141         if (args->stateless_rpc && args->deepen) {
 142                 /* If we sent a depth we will get back "duplicate"
 143                  * shallow and unshallow commands every time there
 144                  * is a block of have lines exchanged.
 145                  */
 146                 while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
 147                         if (starts_with(reader->line, "shallow "))
 148                                 continue;
 149                         if (starts_with(reader->line, "unshallow "))
 150                                 continue;
 151                         die(_("git fetch-pack: expected shallow list"));
 152                 }
 153                 if (reader->status != PACKET_READ_FLUSH)
 154                         die(_("git fetch-pack: expected a flush packet after shallow list"));
 155         }
 156 }
 157
 158 static enum ack_type get_ack(struct packet_reader *reader,
 159                              struct object_id *result_oid)
 160 {
 161         int len;
 162         const char *arg;
 163
 164         if (packet_reader_read(reader) != PACKET_READ_NORMAL)
 165                 die(_("git fetch-pack: expected ACK/NAK, got a flush packet"));
 166         len = reader->pktlen;
 167
 168         if (!strcmp(reader->line, "NAK"))
 169                 return NAK;
 170         if (skip_prefix(reader->line, "ACK ", &arg)) {
 171                 if (!get_oid_hex(arg, result_oid)) {
 172                         arg += 40;
 173                         len -= arg - reader->line;
 174                         if (len < 1)
 175                                 return ACK;
 176                         if (strstr(arg, "continue"))
 177                                 return ACK_continue;
 178                         if (strstr(arg, "common"))
 179                                 return ACK_common;
 180                         if (strstr(arg, "ready"))
 181                                 return ACK_ready;
 182                         return ACK;
 183                 }
 184         }
 185         if (skip_prefix(reader->line, "ERR ", &arg))
 186                 die(_("remote error: %s"), arg);
 187         die(_("git fetch-pack: expected ACK/NAK, got '%s'"), reader->line);
 188 }
 189
 190 static void send_request(struct fetch_pack_args *args,
 191                          int fd, struct strbuf *buf)
 192 {
 193         if (args->stateless_rpc) {
 194                 send_sideband(fd, -1, buf->buf, buf->len, LARGE_PACKET_MAX);
 195                 packet_flush(fd);
 196         } else
 197                 write_or_die(fd, buf->buf, buf->len);
 198 }
 199
 200 static void insert_one_alternate_object(struct fetch_negotiator *negotiator,
 201                                         struct object *obj)
 202 {
 203         rev_list_insert_ref(negotiator, NULL, &obj->oid);
 204 }
 205
 206 #define INITIAL_FLUSH 16
 207 #define PIPESAFE_FLUSH 32
 208 #define LARGE_FLUSH 16384
 209
 210 static int next_flush(int stateless_rpc, int count)
 211 {
 212         if (stateless_rpc) {
 213                 if (count < LARGE_FLUSH)
 214                         count <<= 1;
 215                 else
 216                         count = count * 11 / 10;
 217         } else {
 218                 if (count < PIPESAFE_FLUSH)
 219                         count <<= 1;
 220                 else
 221                         count += PIPESAFE_FLUSH;
 222         }
 223         return count;
 224 }
 225
 226 static void mark_tips(struct fetch_negotiator *negotiator,
 227                       const struct oid_array *negotiation_tips)
 228 {
 229         int i;
 230
 231         if (!negotiation_tips) {
 232                 for_each_ref(rev_list_insert_ref_oid, negotiator);
 233                 return;
 234         }
 235
 236         for (i = 0; i < negotiation_tips->nr; i++)
 237                 rev_list_insert_ref(negotiator, NULL,
 238                                     &negotiation_tips->oid[i]);
 239         return;
 240 }
 241
 242 static int find_common(struct fetch_negotiator *negotiator,
 243                        struct fetch_pack_args *args,
 244                        int fd[2], struct object_id *result_oid,
 245                        struct ref *refs)
 246 {
 247         int fetching;
 248         int count = 0, flushes = 0, flush_at = INITIAL_FLUSH, retval;
 249         const struct object_id *oid;
 250         unsigned in_vain = 0;
 251         int got_continue = 0;
 252         int got_ready = 0;
 253         struct strbuf req_buf = STRBUF_INIT;
 254         size_t state_len = 0;
 255         struct packet_reader reader;
 256
 257         if (args->stateless_rpc && multi_ack == 1)
 258                 die(_("--stateless-rpc requires multi_ack_detailed"));
 259
 260         packet_reader_init(&reader, fd[0], NULL, 0,
 261                            PACKET_READ_CHOMP_NEWLINE);
 262
 263         if (!args->no_dependents) {
 264                 mark_tips(negotiator, args->negotiation_tips);
 265                 for_each_cached_alternate(negotiator, insert_one_alternate_object);
 266         }
 267
 268         fetching = 0;
 269         for ( ; refs ; refs = refs->next) {
 270                 struct object_id *remote = &refs->old_oid;
 271                 const char *remote_hex;
 272                 struct object *o;
 273
 274                 /*
 275                  * If that object is complete (i.e. it is an ancestor of a
 276                  * local ref), we tell them we have it but do not have to
 277                  * tell them about its ancestors, which they already know
 278                  * about.
 279                  *
 280                  * We use lookup_object here because we are only
 281                  * interested in the case we *know* the object is
 282                  * reachable and we have already scanned it.
 283                  *
 284                  * Do this only if args->no_dependents is false (if it is true,
 285                  * we cannot trust the object flags).
 286                  */
 287                 if (!args->no_dependents &&
 288                     ((o = lookup_object(the_repository, remote->hash)) != NULL) &&
 289                                 (o->flags & COMPLETE)) {
 290                         continue;
 291                 }
 292
 293                 remote_hex = oid_to_hex(remote);
 294                 if (!fetching) {
 295                         struct strbuf c = STRBUF_INIT;
 296                         if (multi_ack == 2)     strbuf_addstr(&c, " multi_ack_detailed");
 297                         if (multi_ack == 1)     strbuf_addstr(&c, " multi_ack");
 298                         if (no_done)            strbuf_addstr(&c, " no-done");
 299                         if (use_sideband == 2)  strbuf_addstr(&c, " side-band-64k");
 300                         if (use_sideband == 1)  strbuf_addstr(&c, " side-band");
 301                         if (args->deepen_relative) strbuf_addstr(&c, " deepen-relative");
 302                         if (args->use_thin_pack) strbuf_addstr(&c, " thin-pack");
 303                         if (args->no_progress)   strbuf_addstr(&c, " no-progress");
 304                         if (args->include_tag)   strbuf_addstr(&c, " include-tag");
 305                         if (prefer_ofs_delta)   strbuf_addstr(&c, " ofs-delta");
 306                         if (deepen_since_ok)    strbuf_addstr(&c, " deepen-since");
 307                         if (deepen_not_ok)      strbuf_addstr(&c, " deepen-not");
 308                         if (agent_supported)    strbuf_addf(&c, " agent=%s",
 309                                                             git_user_agent_sanitized());
 310                         if (args->filter_options.choice)
 311                                 strbuf_addstr(&c, " filter");
 312                         packet_buf_write(&req_buf, "want %s%s\n", remote_hex, c.buf);
 313                         strbuf_release(&c);
 314                 } else
 315                         packet_buf_write(&req_buf, "want %s\n", remote_hex);
 316                 fetching++;
 317         }
 318
 319         if (!fetching) {
 320                 strbuf_release(&req_buf);
 321                 packet_flush(fd[1]);
 322                 return 1;
 323         }
 324
 325         if (is_repository_shallow(the_repository))
 326                 write_shallow_commits(&req_buf, 1, NULL);
 327         if (args->depth > 0)
 328                 packet_buf_write(&req_buf, "deepen %d", args->depth);
 329         if (args->deepen_since) {
 330                 timestamp_t max_age = approxidate(args->deepen_since);
 331                 packet_buf_write(&req_buf, "deepen-since %"PRItime, max_age);
 332         }
 333         if (args->deepen_not) {
 334                 int i;
 335                 for (i = 0; i < args->deepen_not->nr; i++) {
 336                         struct string_list_item *s = args->deepen_not->items + i;
 337                         packet_buf_write(&req_buf, "deepen-not %s", s->string);
 338                 }
 339         }
 340         if (server_supports_filtering && args->filter_options.choice)
 341                 packet_buf_write(&req_buf, "filter %s",
 342                                  args->filter_options.filter_spec);
 343         packet_buf_flush(&req_buf);
 344         state_len = req_buf.len;
 345
 346         if (args->deepen) {
 347                 const char *arg;
 348                 struct object_id oid;
 349
 350                 send_request(args, fd[1], &req_buf);
 351                 while (packet_reader_read(&reader) == PACKET_READ_NORMAL) {
 352                         if (skip_prefix(reader.line, "shallow ", &arg)) {
 353                                 if (get_oid_hex(arg, &oid))
 354                                         die(_("invalid shallow line: %s"), reader.line);
 355                                 register_shallow(the_repository, &oid);
 356                                 continue;
 357                         }
 358                         if (skip_prefix(reader.line, "unshallow ", &arg)) {
 359                                 if (get_oid_hex(arg, &oid))
 360                                         die(_("invalid unshallow line: %s"), reader.line);
 361                                 if (!lookup_object(the_repository, oid.hash))
 362                                         die(_("object not found: %s"), reader.line);
 363                                 /* make sure that it is parsed as shallow */
 364                                 if (!parse_object(the_repository, &oid))
 365                                         die(_("error in object: %s"), reader.line);
 366                                 if (unregister_shallow(&oid))
 367                                         die(_("no shallow found: %s"), reader.line);
 368                                 continue;
 369                         }
 370                         die(_("expected shallow/unshallow, got %s"), reader.line);
 371                 }
 372         } else if (!args->stateless_rpc)
 373                 send_request(args, fd[1], &req_buf);
 374
 375         if (!args->stateless_rpc) {
 376                 /* If we aren't using the stateless-rpc interface
 377                  * we don't need to retain the headers.
 378                  */
 379                 strbuf_setlen(&req_buf, 0);
 380                 state_len = 0;
 381         }
 382
 383         flushes = 0;
 384         retval = -1;
 385         if (args->no_dependents)
 386                 goto done;
 387         while ((oid = negotiator->next(negotiator))) {
 388                 packet_buf_write(&req_buf, "have %s\n", oid_to_hex(oid));
 389                 print_verbose(args, "have %s", oid_to_hex(oid));
 390                 in_vain++;
 391                 if (flush_at <= ++count) {
 392                         int ack;
 393
 394                         packet_buf_flush(&req_buf);
 395                         send_request(args, fd[1], &req_buf);
 396                         strbuf_setlen(&req_buf, state_len);
 397                         flushes++;
 398                         flush_at = next_flush(args->stateless_rpc, count);
 399
 400                         /*
 401                          * We keep one window "ahead" of the other side, and
 402                          * will wait for an ACK only on the next one
 403                          */
 404                         if (!args->stateless_rpc && count == INITIAL_FLUSH)
 405                                 continue;
 406
 407                         consume_shallow_list(args, &reader);
 408                         do {
 409                                 ack = get_ack(&reader, result_oid);
 410                                 if (ack)
 411                                         print_verbose(args, _("got %s %d %s"), "ack",
 412                                                       ack, oid_to_hex(result_oid));
 413                                 switch (ack) {
 414                                 case ACK:
 415                                         flushes = 0;
 416                                         multi_ack = 0;
 417                                         retval = 0;
 418                                         goto done;
 419                                 case ACK_common:
 420                                 case ACK_ready:
 421                                 case ACK_continue: {
 422                                         struct commit *commit =
 423                                                 lookup_commit(the_repository,
 424                                                               result_oid);
 425                                         int was_common;
 426
 427                                         if (!commit)
 428                                                 die(_("invalid commit %s"), oid_to_hex(result_oid));
 429                                         was_common = negotiator->ack(negotiator, commit);
 430                                         if (args->stateless_rpc
 431                                          && ack == ACK_common
 432                                          && !was_common) {
 433                                                 /* We need to replay the have for this object
 434                                                  * on the next RPC request so the peer knows
 435                                                  * it is in common with us.
 436                                                  */
 437                                                 const char *hex = oid_to_hex(result_oid);
 438                                                 packet_buf_write(&req_buf, "have %s\n", hex);
 439                                                 state_len = req_buf.len;
 440                                                 /*
 441                                                  * Reset in_vain because an ack
 442                                                  * for this commit has not been
 443                                                  * seen.
 444                                                  */
 445                                                 in_vain = 0;
 446                                         } else if (!args->stateless_rpc
 447                                                    || ack != ACK_common)
 448                                                 in_vain = 0;
 449                                         retval = 0;
 450                                         got_continue = 1;
 451                                         if (ack == ACK_ready)
 452                                                 got_ready = 1;
 453                                         break;
 454                                         }
 455                                 }
 456                         } while (ack);
 457                         flushes--;
 458                         if (got_continue && MAX_IN_VAIN < in_vain) {
 459                                 print_verbose(args, _("giving up"));
 460                                 break; /* give up */
 461                         }
 462                         if (got_ready)
 463                                 break;
 464                 }
 465         }
 466 done:
 467         if (!got_ready || !no_done) {
 468                 packet_buf_write(&req_buf, "done\n");
 469                 send_request(args, fd[1], &req_buf);
 470         }
 471         print_verbose(args, _("done"));
 472         if (retval != 0) {
 473                 multi_ack = 0;
 474                 flushes++;
 475         }
 476         strbuf_release(&req_buf);
 477
 478         if (!got_ready || !no_done)
 479                 consume_shallow_list(args, &reader);
 480         while (flushes || multi_ack) {
 481                 int ack = get_ack(&reader, result_oid);
 482                 if (ack) {
 483                         print_verbose(args, _("got %s (%d) %s"), "ack",
 484                                       ack, oid_to_hex(result_oid));
 485                         if (ack == ACK)
 486                                 return 0;
 487                         multi_ack = 1;
 488                         continue;
 489                 }
 490                 flushes--;
 491         }
 492         /* it is no error to fetch into a completely empty repo */
 493         return count ? retval : 0;
 494 }
 495
 496 static struct commit_list *complete;
 497
 498 static int mark_complete(const struct object_id *oid)
 499 {
 500         struct object *o = parse_object(the_repository, oid);
 501
 502         while (o && o->type == OBJ_TAG) {
 503                 struct tag *t = (struct tag *) o;
 504                 if (!t->tagged)
 505                         break; /* broken repository */
 506                 o->flags |= COMPLETE;
 507                 o = parse_object(the_repository, &t->tagged->oid);
 508         }
 509         if (o && o->type == OBJ_COMMIT) {
 510                 struct commit *commit = (struct commit *)o;
 511                 if (!(commit->object.flags & COMPLETE)) {
 512                         commit->object.flags |= COMPLETE;
 513                         commit_list_insert(commit, &complete);
 514                 }
 515         }
 516         return 0;
 517 }
 518
 519 static int mark_complete_oid(const char *refname, const struct object_id *oid,
 520                              int flag, void *cb_data)
 521 {
 522         return mark_complete(oid);
 523 }
 524
 525 static void mark_recent_complete_commits(struct fetch_pack_args *args,
 526                                          timestamp_t cutoff)
 527 {
 528         while (complete && cutoff <= complete->item->date) {
 529                 print_verbose(args, _("Marking %s as complete"),
 530                               oid_to_hex(&complete->item->object.oid));
 531                 pop_most_recent_commit(&complete, COMPLETE);
 532         }
 533 }
 534
 535 static void add_refs_to_oidset(struct oidset *oids, struct ref *refs)
 536 {
 537         for (; refs; refs = refs->next)
 538                 oidset_insert(oids, &refs->old_oid);
 539 }
 540
 541 static int is_unmatched_ref(const struct ref *ref)
 542 {
 543         struct object_id oid;
 544         const char *p;
 545         return  ref->match_status == REF_NOT_MATCHED &&
 546                 !parse_oid_hex(ref->name, &oid, &p) &&
 547                 *p == '\0' &&
 548                 oideq(&oid, &ref->old_oid);
 549 }
 550
 551 static void filter_refs(struct fetch_pack_args *args,
 552                         struct ref **refs,
 553                         struct ref **sought, int nr_sought)
 554 {
 555         struct ref *newlist = NULL;
 556         struct ref **newtail = &newlist;
 557         struct ref *unmatched = NULL;
 558         struct ref *ref, *next;
 559         struct oidset tip_oids = OIDSET_INIT;
 560         int i;
 561         int strict = !(allow_unadvertised_object_request &
 562                        (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
 563
 564         i = 0;
 565         for (ref = *refs; ref; ref = next) {
 566                 int keep = 0;
 567                 next = ref->next;
 568
 569                 if (starts_with(ref->name, "refs/") &&
 570                     check_refname_format(ref->name, 0))
 571                         ; /* trash */
 572                 else {
 573                         while (i < nr_sought) {
 574                                 int cmp = strcmp(ref->name, sought[i]->name);
 575                                 if (cmp < 0)
 576                                         break; /* definitely do not have it */
 577                                 else if (cmp == 0) {
 578                                         keep = 1; /* definitely have it */
 579                                         sought[i]->match_status = REF_MATCHED;
 580                                 }
 581                                 i++;
 582                         }
 583
 584                         if (!keep && args->fetch_all &&
 585                             (!args->deepen || !starts_with(ref->name, "refs/tags/")))
 586                                 keep = 1;
 587                 }
 588
 589                 if (keep) {
 590                         *newtail = ref;
 591                         ref->next = NULL;
 592                         newtail = &ref->next;
 593                 } else {
 594                         ref->next = unmatched;
 595                         unmatched = ref;
 596                 }
 597         }
 598
 599         if (strict) {
 600                 for (i = 0; i < nr_sought; i++) {
 601                         ref = sought[i];
 602                         if (!is_unmatched_ref(ref))
 603                                 continue;
 604
 605                         add_refs_to_oidset(&tip_oids, unmatched);
 606                         add_refs_to_oidset(&tip_oids, newlist);
 607                         break;
 608                 }
 609         }
 610
 611         /* Append unmatched requests to the list */
 612         for (i = 0; i < nr_sought; i++) {
 613                 ref = sought[i];
 614                 if (!is_unmatched_ref(ref))
 615                         continue;
 616
 617                 if (!strict || oidset_contains(&tip_oids, &ref->old_oid)) {
 618                         ref->match_status = REF_MATCHED;
 619                         *newtail = copy_ref(ref);
 620                         newtail = &(*newtail)->next;
 621                 } else {
 622                         ref->match_status = REF_UNADVERTISED_NOT_ALLOWED;
 623                 }
 624         }
 625
 626         oidset_clear(&tip_oids);
 627         for (ref = unmatched; ref; ref = next) {
 628                 next = ref->next;
 629                 free(ref);
 630         }
 631
 632         *refs = newlist;
 633 }
 634
 635 static void mark_alternate_complete(struct fetch_negotiator *unused,
 636                                     struct object *obj)
 637 {
 638         mark_complete(&obj->oid);
 639 }
 640
 641 struct loose_object_iter {
 642         struct oidset *loose_object_set;
 643         struct ref *refs;
 644 };
 645
 646 /*
 647  *  If the number of refs is not larger than the number of loose objects,
 648  *  this function stops inserting.
 649  */
 650 static int add_loose_objects_to_set(const struct object_id *oid,
 651                                     const char *path,
 652                                     void *data)
 653 {
 654         struct loose_object_iter *iter = data;
 655         oidset_insert(iter->loose_object_set, oid);
 656         if (iter->refs == NULL)
 657                 return 1;
 658
 659         iter->refs = iter->refs->next;
 660         return 0;
 661 }
 662
 663 /*
 664  * Mark recent commits available locally and reachable from a local ref as
 665  * COMPLETE. If args->no_dependents is false, also mark COMPLETE remote refs as
 666  * COMMON_REF (otherwise, we are not planning to participate in negotiation, and
 667  * thus do not need COMMON_REF marks).
 668  *
 669  * The cutoff time for recency is determined by this heuristic: it is the
 670  * earliest commit time of the objects in refs that are commits and that we know
 671  * the commit time of.
 672  */
 673 static void mark_complete_and_common_ref(struct fetch_negotiator *negotiator,
 674                                          struct fetch_pack_args *args,
 675                                          struct ref **refs)
 676 {
 677         struct ref *ref;
 678         int old_save_commit_buffer = save_commit_buffer;
 679         timestamp_t cutoff = 0;
 680         struct oidset loose_oid_set = OIDSET_INIT;
 681         int use_oidset = 0;
 682         struct loose_object_iter iter = {&loose_oid_set, *refs};
 683
 684         /* Enumerate all loose objects or know refs are not so many. */
 685         use_oidset = !for_each_loose_object(add_loose_objects_to_set,
 686                                             &iter, 0);
 687
 688         save_commit_buffer = 0;
 689
 690         for (ref = *refs; ref; ref = ref->next) {
 691                 struct object *o;
 692                 unsigned int flags = OBJECT_INFO_QUICK;
 693
 694                 if (use_oidset &&
 695                     !oidset_contains(&loose_oid_set, &ref->old_oid)) {
 696                         /*
 697                          * I know this does not exist in the loose form,
 698                          * so check if it exists in a non-loose form.
 699                          */
 700                         flags |= OBJECT_INFO_IGNORE_LOOSE;
 701                 }
 702
 703                 if (!has_object_file_with_flags(&ref->old_oid, flags))
 704                         continue;
 705                 o = parse_object(the_repository, &ref->old_oid);
 706                 if (!o)
 707                         continue;
 708
 709                 /* We already have it -- which may mean that we were
 710                  * in sync with the other side at some time after
 711                  * that (it is OK if we guess wrong here).
 712                  */
 713                 if (o->type == OBJ_COMMIT) {
 714                         struct commit *commit = (struct commit *)o;
 715                         if (!cutoff || cutoff < commit->date)
 716                                 cutoff = commit->date;
 717                 }
 718         }
 719
 720         oidset_clear(&loose_oid_set);
 721
 722         if (!args->deepen) {
 723                 for_each_ref(mark_complete_oid, NULL);
 724                 for_each_cached_alternate(NULL, mark_alternate_complete);
 725                 commit_list_sort_by_date(&complete);
 726                 if (cutoff)
 727                         mark_recent_complete_commits(args, cutoff);
 728         }
 729
 730         /*
 731          * Mark all complete remote refs as common refs.
 732          * Don't mark them common yet; the server has to be told so first.
 733          */
 734         for (ref = *refs; ref; ref = ref->next) {
 735                 struct object *o = deref_tag(the_repository,
 736                                              lookup_object(the_repository,
 737                                              ref->old_oid.hash),
 738                                              NULL, 0);
 739
 740                 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
 741                         continue;
 742
 743                 negotiator->known_common(negotiator,
 744                                          (struct commit *)o);
 745         }
 746
 747         save_commit_buffer = old_save_commit_buffer;
 748 }
 749
 750 /*
 751  * Returns 1 if every object pointed to by the given remote refs is available
 752  * locally and reachable from a local ref, and 0 otherwise.
 753  */
 754 static int everything_local(struct fetch_pack_args *args,
 755                             struct ref **refs)
 756 {
 757         struct ref *ref;
 758         int retval;
 759
 760         for (retval = 1, ref = *refs; ref ; ref = ref->next) {
 761                 const struct object_id *remote = &ref->old_oid;
 762                 struct object *o;
 763
 764                 o = lookup_object(the_repository, remote->hash);
 765                 if (!o || !(o->flags & COMPLETE)) {
 766                         retval = 0;
 767                         print_verbose(args, "want %s (%s)", oid_to_hex(remote),
 768                                       ref->name);
 769                         continue;
 770                 }
 771                 print_verbose(args, _("already have %s (%s)"), oid_to_hex(remote),
 772                               ref->name);
 773         }
 774
 775         return retval;
 776 }
 777
 778 static int sideband_demux(int in, int out, void *data)
 779 {
 780         int *xd = data;
 781         int ret;
 782
 783         ret = recv_sideband("fetch-pack", xd[0], out);
 784         close(out);
 785         return ret;
 786 }
 787
 788 static int get_pack(struct fetch_pack_args *args,
 789                     int xd[2], char **pack_lockfile)
 790 {
 791         struct async demux;
 792         int do_keep = args->keep_pack;
 793         const char *cmd_name;
 794         struct pack_header header;
 795         int pass_header = 0;
 796         struct child_process cmd = CHILD_PROCESS_INIT;
 797         int ret;
 798
 799         memset(&demux, 0, sizeof(demux));
 800         if (use_sideband) {
 801                 /* xd[] is talking with upload-pack; subprocess reads from
 802                  * xd[0], spits out band#2 to stderr, and feeds us band#1
 803                  * through demux->out.
 804                  */
 805                 demux.proc = sideband_demux;
 806                 demux.data = xd;
 807                 demux.out = -1;
 808                 demux.isolate_sigpipe = 1;
 809                 if (start_async(&demux))
 810                         die(_("fetch-pack: unable to fork off sideband demultiplexer"));
 811         }
 812         else
 813                 demux.out = xd[0];
 814
 815         if (!args->keep_pack && unpack_limit) {
 816
 817                 if (read_pack_header(demux.out, &header))
 818                         die(_("protocol error: bad pack header"));
 819                 pass_header = 1;
 820                 if (ntohl(header.hdr_entries) < unpack_limit)
 821                         do_keep = 0;
 822                 else
 823                         do_keep = 1;
 824         }
 825
 826         if (alternate_shallow_file) {
 827                 argv_array_push(&cmd.args, "--shallow-file");
 828                 argv_array_push(&cmd.args, alternate_shallow_file);
 829         }
 830
 831         if (do_keep || args->from_promisor) {
 832                 if (pack_lockfile)
 833                         cmd.out = -1;
 834                 cmd_name = "index-pack";
 835                 argv_array_push(&cmd.args, cmd_name);
 836                 argv_array_push(&cmd.args, "--stdin");
 837                 if (!args->quiet && !args->no_progress)
 838                         argv_array_push(&cmd.args, "-v");
 839                 if (args->use_thin_pack)
 840                         argv_array_push(&cmd.args, "--fix-thin");
 841                 if (do_keep && (args->lock_pack || unpack_limit)) {
 842                         char hostname[HOST_NAME_MAX + 1];
 843                         if (xgethostname(hostname, sizeof(hostname)))
 844                                 xsnprintf(hostname, sizeof(hostname), "localhost");
 845                         argv_array_pushf(&cmd.args,
 846                                         "--keep=fetch-pack %"PRIuMAX " on %s",
 847                                         (uintmax_t)getpid(), hostname);
 848                 }
 849                 if (args->check_self_contained_and_connected)
 850                         argv_array_push(&cmd.args, "--check-self-contained-and-connected");
 851                 if (args->from_promisor)
 852                         argv_array_push(&cmd.args, "--promisor");
 853         }
 854         else {
 855                 cmd_name = "unpack-objects";
 856                 argv_array_push(&cmd.args, cmd_name);
 857                 if (args->quiet || args->no_progress)
 858                         argv_array_push(&cmd.args, "-q");
 859                 args->check_self_contained_and_connected = 0;
 860         }
 861
 862         if (pass_header)
 863                 argv_array_pushf(&cmd.args, "--pack_header=%"PRIu32",%"PRIu32,
 864                                  ntohl(header.hdr_version),
 865                                  ntohl(header.hdr_entries));
 866         if (fetch_fsck_objects >= 0
 867             ? fetch_fsck_objects
 868             : transfer_fsck_objects >= 0
 869             ? transfer_fsck_objects
 870             : 0) {
 871                 if (args->from_promisor)
 872                         /*
 873                          * We cannot use --strict in index-pack because it
 874                          * checks both broken objects and links, but we only
 875                          * want to check for broken objects.
 876                          */
 877                         argv_array_push(&cmd.args, "--fsck-objects");
 878                 else
 879                         argv_array_pushf(&cmd.args, "--strict%s",
 880                                          fsck_msg_types.buf);
 881         }
 882
 883         cmd.in = demux.out;
 884         cmd.git_cmd = 1;
 885         if (start_command(&cmd))
 886                 die(_("fetch-pack: unable to fork off %s"), cmd_name);
 887         if (do_keep && pack_lockfile) {
 888                 *pack_lockfile = index_pack_lockfile(cmd.out);
 889                 close(cmd.out);
 890         }
 891
 892         if (!use_sideband)
 893                 /* Closed by start_command() */
 894                 xd[0] = -1;
 895
 896         ret = finish_command(&cmd);
 897         if (!ret || (args->check_self_contained_and_connected && ret == 1))
 898                 args->self_contained_and_connected =
 899                         args->check_self_contained_and_connected &&
 900                         ret == 0;
 901         else
 902                 die(_("%s failed"), cmd_name);
 903         if (use_sideband && finish_async(&demux))
 904                 die(_("error in sideband demultiplexer"));
 905         return 0;
 906 }
 907
 908 static int cmp_ref_by_name(const void *a_, const void *b_)
 909 {
 910         const struct ref *a = *((const struct ref **)a_);
 911         const struct ref *b = *((const struct ref **)b_);
 912         return strcmp(a->name, b->name);
 913 }
 914
 915 static struct ref *do_fetch_pack(struct fetch_pack_args *args,
 916                                  int fd[2],
 917                                  const struct ref *orig_ref,
 918                                  struct ref **sought, int nr_sought,
 919                                  struct shallow_info *si,
 920                                  char **pack_lockfile)
 921 {
 922         struct ref *ref = copy_ref_list(orig_ref);
 923         struct object_id oid;
 924         const char *agent_feature;
 925         int agent_len;
 926         struct fetch_negotiator negotiator;
 927         fetch_negotiator_init(&negotiator, negotiation_algorithm);
 928
 929         sort_ref_list(&ref, ref_compare_name);
 930         QSORT(sought, nr_sought, cmp_ref_by_name);
 931
 932         if ((args->depth > 0 || is_repository_shallow(the_repository)) && !server_supports("shallow"))
 933                 die(_("Server does not support shallow clients"));
 934         if (args->depth > 0 || args->deepen_since || args->deepen_not)
 935                 args->deepen = 1;
 936         if (server_supports("multi_ack_detailed")) {
 937                 print_verbose(args, _("Server supports multi_ack_detailed"));
 938                 multi_ack = 2;
 939                 if (server_supports("no-done")) {
 940                         print_verbose(args, _("Server supports no-done"));
 941                         if (args->stateless_rpc)
 942                                 no_done = 1;
 943                 }
 944         }
 945         else if (server_supports("multi_ack")) {
 946                 print_verbose(args, _("Server supports multi_ack"));
 947                 multi_ack = 1;
 948         }
 949         if (server_supports("side-band-64k")) {
 950                 print_verbose(args, _("Server supports side-band-64k"));
 951                 use_sideband = 2;
 952         }
 953         else if (server_supports("side-band")) {
 954                 print_verbose(args, _("Server supports side-band"));
 955                 use_sideband = 1;
 956         }
 957         if (server_supports("allow-tip-sha1-in-want")) {
 958                 print_verbose(args, _("Server supports allow-tip-sha1-in-want"));
 959                 allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
 960         }
 961         if (server_supports("allow-reachable-sha1-in-want")) {
 962                 print_verbose(args, _("Server supports allow-reachable-sha1-in-want"));
 963                 allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
 964         }
 965         if (!server_supports("thin-pack"))
 966                 args->use_thin_pack = 0;
 967         if (!server_supports("no-progress"))
 968                 args->no_progress = 0;
 969         if (!server_supports("include-tag"))
 970                 args->include_tag = 0;
 971         if (server_supports("ofs-delta"))
 972                 print_verbose(args, _("Server supports ofs-delta"));
 973         else
 974                 prefer_ofs_delta = 0;
 975
 976         if (server_supports("filter")) {
 977                 server_supports_filtering = 1;
 978                 print_verbose(args, _("Server supports filter"));
 979         } else if (args->filter_options.choice) {
 980                 warning("filtering not recognized by server, ignoring");
 981         }
 982
 983         if ((agent_feature = server_feature_value("agent", &agent_len))) {
 984                 agent_supported = 1;
 985                 if (agent_len)
 986                         print_verbose(args, _("Server version is %.*s"),
 987                                       agent_len, agent_feature);
 988         }
 989         if (server_supports("deepen-since"))
 990                 deepen_since_ok = 1;
 991         else if (args->deepen_since)
 992                 die(_("Server does not support --shallow-since"));
 993         if (server_supports("deepen-not"))
 994                 deepen_not_ok = 1;
 995         else if (args->deepen_not)
 996                 die(_("Server does not support --shallow-exclude"));
 997         if (!server_supports("deepen-relative") && args->deepen_relative)
 998                 die(_("Server does not support --deepen"));
 999
1000         if (!args->no_dependents) {
1001                 mark_complete_and_common_ref(&negotiator, args, &ref);
1002                 filter_refs(args, &ref, sought, nr_sought);
1003                 if (everything_local(args, &ref)) {
1004                         packet_flush(fd[1]);
1005                         goto all_done;
1006                 }
1007         } else {
1008                 filter_refs(args, &ref, sought, nr_sought);
1009         }
1010         if (find_common(&negotiator, args, fd, &oid, ref) < 0)
1011                 if (!args->keep_pack)
1012                         /* When cloning, it is not unusual to have
1013                          * no common commit.
1014                          */
1015                         warning(_("no common commits"));
1016
1017         if (args->stateless_rpc)
1018                 packet_flush(fd[1]);
1019         if (args->deepen)
1020                 setup_alternate_shallow(&shallow_lock, &alternate_shallow_file,
1021                                         NULL);
1022         else if (si->nr_ours || si->nr_theirs)
1023                 alternate_shallow_file = setup_temporary_shallow(si->shallow);
1024         else
1025                 alternate_shallow_file = NULL;
1026         if (get_pack(args, fd, pack_lockfile))
1027                 die(_("git fetch-pack: fetch failed."));
1028
1029  all_done:
1030         negotiator.release(&negotiator);
1031         return ref;
1032 }
1033
1034 static void add_shallow_requests(struct strbuf *req_buf,
1035                                  const struct fetch_pack_args *args)
1036 {
1037         if (is_repository_shallow(the_repository))
1038                 write_shallow_commits(req_buf, 1, NULL);
1039         if (args->depth > 0)
1040                 packet_buf_write(req_buf, "deepen %d", args->depth);
1041         if (args->deepen_since) {
1042                 timestamp_t max_age = approxidate(args->deepen_since);
1043                 packet_buf_write(req_buf, "deepen-since %"PRItime, max_age);
1044         }
1045         if (args->deepen_not) {
1046                 int i;
1047                 for (i = 0; i < args->deepen_not->nr; i++) {
1048                         struct string_list_item *s = args->deepen_not->items + i;
1049                         packet_buf_write(req_buf, "deepen-not %s", s->string);
1050                 }
1051         }
1052 }
1053
1054 static void add_wants(int no_dependents, const struct ref *wants, struct strbuf *req_buf)
1055 {
1056         int use_ref_in_want = server_supports_feature("fetch", "ref-in-want", 0);
1057
1058         for ( ; wants ; wants = wants->next) {
1059                 const struct object_id *remote = &wants->old_oid;
1060                 struct object *o;
1061
1062                 /*
1063                  * If that object is complete (i.e. it is an ancestor of a
1064                  * local ref), we tell them we have it but do not have to
1065                  * tell them about its ancestors, which they already know
1066                  * about.
1067                  *
1068                  * We use lookup_object here because we are only
1069                  * interested in the case we *know* the object is
1070                  * reachable and we have already scanned it.
1071                  *
1072                  * Do this only if args->no_dependents is false (if it is true,
1073                  * we cannot trust the object flags).
1074                  */
1075                 if (!no_dependents &&
1076                     ((o = lookup_object(the_repository, remote->hash)) != NULL) &&
1077                     (o->flags & COMPLETE)) {
1078                         continue;
1079                 }
1080
1081                 if (!use_ref_in_want || wants->exact_oid)
1082                         packet_buf_write(req_buf, "want %s\n", oid_to_hex(remote));
1083                 else
1084                         packet_buf_write(req_buf, "want-ref %s\n", wants->name);
1085         }
1086 }
1087
1088 static void add_common(struct strbuf *req_buf, struct oidset *common)
1089 {
1090         struct oidset_iter iter;
1091         const struct object_id *oid;
1092         oidset_iter_init(common, &iter);
1093
1094         while ((oid = oidset_iter_next(&iter))) {
1095                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1096         }
1097 }
1098
1099 static int add_haves(struct fetch_negotiator *negotiator,
1100                      struct strbuf *req_buf,
1101                      int *haves_to_send, int *in_vain)
1102 {
1103         int ret = 0;
1104         int haves_added = 0;
1105         const struct object_id *oid;
1106
1107         while ((oid = negotiator->next(negotiator))) {
1108                 packet_buf_write(req_buf, "have %s\n", oid_to_hex(oid));
1109                 if (++haves_added >= *haves_to_send)
1110                         break;
1111         }
1112
1113         *in_vain += haves_added;
1114         if (!haves_added || *in_vain >= MAX_IN_VAIN) {
1115                 /* Send Done */
1116                 packet_buf_write(req_buf, "done\n");
1117                 ret = 1;
1118         }
1119
1120         /* Increase haves to send on next round */
1121         *haves_to_send = next_flush(1, *haves_to_send);
1122
1123         return ret;
1124 }
1125
1126 static int send_fetch_request(struct fetch_negotiator *negotiator, int fd_out,
1127                               const struct fetch_pack_args *args,
1128                               const struct ref *wants, struct oidset *common,
1129                               int *haves_to_send, int *in_vain)
1130 {
1131         int ret = 0;
1132         struct strbuf req_buf = STRBUF_INIT;
1133
1134         if (server_supports_v2("fetch", 1))
1135                 packet_buf_write(&req_buf, "command=fetch");
1136         if (server_supports_v2("agent", 0))
1137                 packet_buf_write(&req_buf, "agent=%s", git_user_agent_sanitized());
1138         if (args->server_options && args->server_options->nr &&
1139             server_supports_v2("server-option", 1)) {
1140                 int i;
1141                 for (i = 0; i < args->server_options->nr; i++)
1142                         packet_write_fmt(fd_out, "server-option=%s",
1143                                          args->server_options->items[i].string);
1144         }
1145
1146         packet_buf_delim(&req_buf);
1147         if (args->use_thin_pack)
1148                 packet_buf_write(&req_buf, "thin-pack");
1149         if (args->no_progress)
1150                 packet_buf_write(&req_buf, "no-progress");
1151         if (args->include_tag)
1152                 packet_buf_write(&req_buf, "include-tag");
1153         if (prefer_ofs_delta)
1154                 packet_buf_write(&req_buf, "ofs-delta");
1155
1156         /* Add shallow-info and deepen request */
1157         if (server_supports_feature("fetch", "shallow", 0))
1158                 add_shallow_requests(&req_buf, args);
1159         else if (is_repository_shallow(the_repository) || args->deepen)
1160                 die(_("Server does not support shallow requests"));
1161
1162         /* Add filter */
1163         if (server_supports_feature("fetch", "filter", 0) &&
1164             args->filter_options.choice) {
1165                 print_verbose(args, _("Server supports filter"));
1166                 packet_buf_write(&req_buf, "filter %s",
1167                                  args->filter_options.filter_spec);
1168         } else if (args->filter_options.choice) {
1169                 warning("filtering not recognized by server, ignoring");
1170         }
1171
1172         /* add wants */
1173         add_wants(args->no_dependents, wants, &req_buf);
1174
1175         if (args->no_dependents) {
1176                 packet_buf_write(&req_buf, "done");
1177                 ret = 1;
1178         } else {
1179                 /* Add all of the common commits we've found in previous rounds */
1180                 add_common(&req_buf, common);
1181
1182                 /* Add initial haves */
1183                 ret = add_haves(negotiator, &req_buf, haves_to_send, in_vain);
1184         }
1185
1186         /* Send request */
1187         packet_buf_flush(&req_buf);
1188         write_or_die(fd_out, req_buf.buf, req_buf.len);
1189
1190         strbuf_release(&req_buf);
1191         return ret;
1192 }
1193
1194 /*
1195  * Processes a section header in a server's response and checks if it matches
1196  * `section`.  If the value of `peek` is 1, the header line will be peeked (and
1197  * not consumed); if 0, the line will be consumed and the function will die if
1198  * the section header doesn't match what was expected.
1199  */
1200 static int process_section_header(struct packet_reader *reader,
1201                                   const char *section, int peek)
1202 {
1203         int ret;
1204
1205         if (packet_reader_peek(reader) != PACKET_READ_NORMAL)
1206                 die(_("error reading section header '%s'"), section);
1207
1208         ret = !strcmp(reader->line, section);
1209
1210         if (!peek) {
1211                 if (!ret)
1212                         die(_("expected '%s', received '%s'"),
1213                             section, reader->line);
1214                 packet_reader_read(reader);
1215         }
1216
1217         return ret;
1218 }
1219
1220 static int process_acks(struct fetch_negotiator *negotiator,
1221                         struct packet_reader *reader,
1222                         struct oidset *common)
1223 {
1224         /* received */
1225         int received_ready = 0;
1226         int received_ack = 0;
1227
1228         process_section_header(reader, "acknowledgments", 0);
1229         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1230                 const char *arg;
1231
1232                 if (!strcmp(reader->line, "NAK"))
1233                         continue;
1234
1235                 if (skip_prefix(reader->line, "ACK ", &arg)) {
1236                         struct object_id oid;
1237                         if (!get_oid_hex(arg, &oid)) {
1238                                 struct commit *commit;
1239                                 oidset_insert(common, &oid);
1240                                 commit = lookup_commit(the_repository, &oid);
1241                                 negotiator->ack(negotiator, commit);
1242                         }
1243                         continue;
1244                 }
1245
1246                 if (!strcmp(reader->line, "ready")) {
1247                         received_ready = 1;
1248                         continue;
1249                 }
1250
1251                 die(_("unexpected acknowledgment line: '%s'"), reader->line);
1252         }
1253
1254         if (reader->status != PACKET_READ_FLUSH &&
1255             reader->status != PACKET_READ_DELIM)
1256                 die(_("error processing acks: %d"), reader->status);
1257
1258         /*
1259          * If an "acknowledgments" section is sent, a packfile is sent if and
1260          * only if "ready" was sent in this section. The other sections
1261          * ("shallow-info" and "wanted-refs") are sent only if a packfile is
1262          * sent. Therefore, a DELIM is expected if "ready" is sent, and a FLUSH
1263          * otherwise.
1264          */
1265         if (received_ready && reader->status != PACKET_READ_DELIM)
1266                 die(_("expected packfile to be sent after 'ready'"));
1267         if (!received_ready && reader->status != PACKET_READ_FLUSH)
1268                 die(_("expected no other sections to be sent after no 'ready'"));
1269
1270         /* return 0 if no common, 1 if there are common, or 2 if ready */
1271         return received_ready ? 2 : (received_ack ? 1 : 0);
1272 }
1273
1274 static void receive_shallow_info(struct fetch_pack_args *args,
1275                                  struct packet_reader *reader)
1276 {
1277         process_section_header(reader, "shallow-info", 0);
1278         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1279                 const char *arg;
1280                 struct object_id oid;
1281
1282                 if (skip_prefix(reader->line, "shallow ", &arg)) {
1283                         if (get_oid_hex(arg, &oid))
1284                                 die(_("invalid shallow line: %s"), reader->line);
1285                         register_shallow(the_repository, &oid);
1286                         continue;
1287                 }
1288                 if (skip_prefix(reader->line, "unshallow ", &arg)) {
1289                         if (get_oid_hex(arg, &oid))
1290                                 die(_("invalid unshallow line: %s"), reader->line);
1291                         if (!lookup_object(the_repository, oid.hash))
1292                                 die(_("object not found: %s"), reader->line);
1293                         /* make sure that it is parsed as shallow */
1294                         if (!parse_object(the_repository, &oid))
1295                                 die(_("error in object: %s"), reader->line);
1296                         if (unregister_shallow(&oid))
1297                                 die(_("no shallow found: %s"), reader->line);
1298                         continue;
1299                 }
1300                 die(_("expected shallow/unshallow, got %s"), reader->line);
1301         }
1302
1303         if (reader->status != PACKET_READ_FLUSH &&
1304             reader->status != PACKET_READ_DELIM)
1305                 die(_("error processing shallow info: %d"), reader->status);
1306
1307         setup_alternate_shallow(&shallow_lock, &alternate_shallow_file, NULL);
1308         args->deepen = 1;
1309 }
1310
1311 static void receive_wanted_refs(struct packet_reader *reader,
1312                                 struct ref **sought, int nr_sought)
1313 {
1314         process_section_header(reader, "wanted-refs", 0);
1315         while (packet_reader_read(reader) == PACKET_READ_NORMAL) {
1316                 struct object_id oid;
1317                 const char *end;
1318                 int i;
1319
1320                 if (parse_oid_hex(reader->line, &oid, &end) || *end++ != ' ')
1321                         die(_("expected wanted-ref, got '%s'"), reader->line);
1322
1323                 for (i = 0; i < nr_sought; i++) {
1324                         if (!strcmp(end, sought[i]->name)) {
1325                                 oidcpy(&sought[i]->old_oid, &oid);
1326                                 break;
1327                         }
1328                 }
1329
1330                 if (i == nr_sought)
1331                         die(_("unexpected wanted-ref: '%s'"), reader->line);
1332         }
1333
1334         if (reader->status != PACKET_READ_DELIM)
1335                 die(_("error processing wanted refs: %d"), reader->status);
1336 }
1337
1338 enum fetch_state {
1339         FETCH_CHECK_LOCAL = 0,
1340         FETCH_SEND_REQUEST,
1341         FETCH_PROCESS_ACKS,
1342         FETCH_GET_PACK,
1343         FETCH_DONE,
1344 };
1345
1346 static struct ref *do_fetch_pack_v2(struct fetch_pack_args *args,
1347                                     int fd[2],
1348                                     const struct ref *orig_ref,
1349                                     struct ref **sought, int nr_sought,
1350                                     char **pack_lockfile)
1351 {
1352         struct ref *ref = copy_ref_list(orig_ref);
1353         enum fetch_state state = FETCH_CHECK_LOCAL;
1354         struct oidset common = OIDSET_INIT;
1355         struct packet_reader reader;
1356         int in_vain = 0;
1357         int haves_to_send = INITIAL_FLUSH;
1358         struct fetch_negotiator negotiator;
1359         fetch_negotiator_init(&negotiator, negotiation_algorithm);
1360         packet_reader_init(&reader, fd[0], NULL, 0,
1361                            PACKET_READ_CHOMP_NEWLINE);
1362
1363         while (state != FETCH_DONE) {
1364                 switch (state) {
1365                 case FETCH_CHECK_LOCAL:
1366                         sort_ref_list(&ref, ref_compare_name);
1367                         QSORT(sought, nr_sought, cmp_ref_by_name);
1368
1369                         /* v2 supports these by default */
1370                         allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1371                         use_sideband = 2;
1372                         if (args->depth > 0 || args->deepen_since || args->deepen_not)
1373                                 args->deepen = 1;
1374
1375                         /* Filter 'ref' by 'sought' and those that aren't local */
1376                         if (!args->no_dependents) {
1377                                 mark_complete_and_common_ref(&negotiator, args, &ref);
1378                                 filter_refs(args, &ref, sought, nr_sought);
1379                                 if (everything_local(args, &ref))
1380                                         state = FETCH_DONE;
1381                                 else
1382                                         state = FETCH_SEND_REQUEST;
1383
1384                                 mark_tips(&negotiator, args->negotiation_tips);
1385                                 for_each_cached_alternate(&negotiator,
1386                                                           insert_one_alternate_object);
1387                         } else {
1388                                 filter_refs(args, &ref, sought, nr_sought);
1389                                 state = FETCH_SEND_REQUEST;
1390                         }
1391                         break;
1392                 case FETCH_SEND_REQUEST:
1393                         if (send_fetch_request(&negotiator, fd[1], args, ref,
1394                                                &common,
1395                                                &haves_to_send, &in_vain))
1396                                 state = FETCH_GET_PACK;
1397                         else
1398                                 state = FETCH_PROCESS_ACKS;
1399                         break;
1400                 case FETCH_PROCESS_ACKS:
1401                         /* Process ACKs/NAKs */
1402                         switch (process_acks(&negotiator, &reader, &common)) {
1403                         case 2:
1404                                 state = FETCH_GET_PACK;
1405                                 break;
1406                         case 1:
1407                                 in_vain = 0;
1408                                 /* fallthrough */
1409                         default:
1410                                 state = FETCH_SEND_REQUEST;
1411                                 break;
1412                         }
1413                         break;
1414                 case FETCH_GET_PACK:
1415                         /* Check for shallow-info section */
1416                         if (process_section_header(&reader, "shallow-info", 1))
1417                                 receive_shallow_info(args, &reader);
1418
1419                         if (process_section_header(&reader, "wanted-refs", 1))
1420                                 receive_wanted_refs(&reader, sought, nr_sought);
1421
1422                         /* get the pack */
1423                         process_section_header(&reader, "packfile", 0);
1424                         if (get_pack(args, fd, pack_lockfile))
1425                                 die(_("git fetch-pack: fetch failed."));
1426
1427                         state = FETCH_DONE;
1428                         break;
1429                 case FETCH_DONE:
1430                         continue;
1431                 }
1432         }
1433
1434         negotiator.release(&negotiator);
1435         oidset_clear(&common);
1436         return ref;
1437 }
1438
1439 static int fetch_pack_config_cb(const char *var, const char *value, void *cb)
1440 {
1441         if (strcmp(var, "fetch.fsck.skiplist") == 0) {
1442                 const char *path;
1443
1444                 if (git_config_pathname(&path, var, value))
1445                         return 1;
1446                 strbuf_addf(&fsck_msg_types, "%cskiplist=%s",
1447                         fsck_msg_types.len ? ',' : '=', path);
1448                 free((char *)path);
1449                 return 0;
1450         }
1451
1452         if (skip_prefix(var, "fetch.fsck.", &var)) {
1453                 if (is_valid_msg_type(var, value))
1454                         strbuf_addf(&fsck_msg_types, "%c%s=%s",
1455                                 fsck_msg_types.len ? ',' : '=', var, value);
1456                 else
1457                         warning("Skipping unknown msg id '%s'", var);
1458                 return 0;
1459         }
1460
1461         return git_default_config(var, value, cb);
1462 }
1463
1464 static void fetch_pack_config(void)
1465 {
1466         git_config_get_int("fetch.unpacklimit", &fetch_unpack_limit);
1467         git_config_get_int("transfer.unpacklimit", &transfer_unpack_limit);
1468         git_config_get_bool("repack.usedeltabaseoffset", &prefer_ofs_delta);
1469         git_config_get_bool("fetch.fsckobjects", &fetch_fsck_objects);
1470         git_config_get_bool("transfer.fsckobjects", &transfer_fsck_objects);
1471         git_config_get_string("fetch.negotiationalgorithm",
1472                               &negotiation_algorithm);
1473
1474         git_config(fetch_pack_config_cb, NULL);
1475 }
1476
1477 static void fetch_pack_setup(void)
1478 {
1479         static int did_setup;
1480         if (did_setup)
1481                 return;
1482         fetch_pack_config();
1483         if (0 <= transfer_unpack_limit)
1484                 unpack_limit = transfer_unpack_limit;
1485         else if (0 <= fetch_unpack_limit)
1486                 unpack_limit = fetch_unpack_limit;
1487         did_setup = 1;
1488 }
1489
1490 static int remove_duplicates_in_refs(struct ref **ref, int nr)
1491 {
1492         struct string_list names = STRING_LIST_INIT_NODUP;
1493         int src, dst;
1494
1495         for (src = dst = 0; src < nr; src++) {
1496                 struct string_list_item *item;
1497                 item = string_list_insert(&names, ref[src]->name);
1498                 if (item->util)
1499                         continue; /* already have it */
1500                 item->util = ref[src];
1501                 if (src != dst)
1502                         ref[dst] = ref[src];
1503                 dst++;
1504         }
1505         for (src = dst; src < nr; src++)
1506                 ref[src] = NULL;
1507         string_list_clear(&names, 0);
1508         return dst;
1509 }
1510
1511 static void update_shallow(struct fetch_pack_args *args,
1512                            struct ref **sought, int nr_sought,
1513                            struct shallow_info *si)
1514 {
1515         struct oid_array ref = OID_ARRAY_INIT;
1516         int *status;
1517         int i;
1518
1519         if (args->deepen && alternate_shallow_file) {
1520                 if (*alternate_shallow_file == '\0') { /* --unshallow */
1521                         unlink_or_warn(git_path_shallow(the_repository));
1522                         rollback_lock_file(&shallow_lock);
1523                 } else
1524                         commit_lock_file(&shallow_lock);
1525                 return;
1526         }
1527
1528         if (!si->shallow || !si->shallow->nr)
1529                 return;
1530
1531         if (args->cloning) {
1532                 /*
1533                  * remote is shallow, but this is a clone, there are
1534                  * no objects in repo to worry about. Accept any
1535                  * shallow points that exist in the pack (iow in repo
1536                  * after get_pack() and reprepare_packed_git())
1537                  */
1538                 struct oid_array extra = OID_ARRAY_INIT;
1539                 struct object_id *oid = si->shallow->oid;
1540                 for (i = 0; i < si->shallow->nr; i++)
1541                         if (has_object_file(&oid[i]))
1542                                 oid_array_append(&extra, &oid[i]);
1543                 if (extra.nr) {
1544                         setup_alternate_shallow(&shallow_lock,
1545                                                 &alternate_shallow_file,
1546                                                 &extra);
1547                         commit_lock_file(&shallow_lock);
1548                 }
1549                 oid_array_clear(&extra);
1550                 return;
1551         }
1552
1553         if (!si->nr_ours && !si->nr_theirs)
1554                 return;
1555
1556         remove_nonexistent_theirs_shallow(si);
1557         if (!si->nr_ours && !si->nr_theirs)
1558                 return;
1559         for (i = 0; i < nr_sought; i++)
1560                 oid_array_append(&ref, &sought[i]->old_oid);
1561         si->ref = &ref;
1562
1563         if (args->update_shallow) {
1564                 /*
1565                  * remote is also shallow, .git/shallow may be updated
1566                  * so all refs can be accepted. Make sure we only add
1567                  * shallow roots that are actually reachable from new
1568                  * refs.
1569                  */
1570                 struct oid_array extra = OID_ARRAY_INIT;
1571                 struct object_id *oid = si->shallow->oid;
1572                 assign_shallow_commits_to_refs(si, NULL, NULL);
1573                 if (!si->nr_ours && !si->nr_theirs) {
1574                         oid_array_clear(&ref);
1575                         return;
1576                 }
1577                 for (i = 0; i < si->nr_ours; i++)
1578                         oid_array_append(&extra, &oid[si->ours[i]]);
1579                 for (i = 0; i < si->nr_theirs; i++)
1580                         oid_array_append(&extra, &oid[si->theirs[i]]);
1581                 setup_alternate_shallow(&shallow_lock,
1582                                         &alternate_shallow_file,
1583                                         &extra);
1584                 commit_lock_file(&shallow_lock);
1585                 oid_array_clear(&extra);
1586                 oid_array_clear(&ref);
1587                 return;
1588         }
1589
1590         /*
1591          * remote is also shallow, check what ref is safe to update
1592          * without updating .git/shallow
1593          */
1594         status = xcalloc(nr_sought, sizeof(*status));
1595         assign_shallow_commits_to_refs(si, NULL, status);
1596         if (si->nr_ours || si->nr_theirs) {
1597                 for (i = 0; i < nr_sought; i++)
1598                         if (status[i])
1599                                 sought[i]->status = REF_STATUS_REJECT_SHALLOW;
1600         }
1601         free(status);
1602         oid_array_clear(&ref);
1603 }
1604
1605 static int iterate_ref_map(void *cb_data, struct object_id *oid)
1606 {
1607         struct ref **rm = cb_data;
1608         struct ref *ref = *rm;
1609
1610         if (!ref)
1611                 return -1; /* end of the list */
1612         *rm = ref->next;
1613         oidcpy(oid, &ref->old_oid);
1614         return 0;
1615 }
1616
1617 struct ref *fetch_pack(struct fetch_pack_args *args,
1618                        int fd[], struct child_process *conn,
1619                        const struct ref *ref,
1620                        const char *dest,
1621                        struct ref **sought, int nr_sought,
1622                        struct oid_array *shallow,
1623                        char **pack_lockfile,
1624                        enum protocol_version version)
1625 {
1626         struct ref *ref_cpy;
1627         struct shallow_info si;
1628
1629         fetch_pack_setup();
1630         if (nr_sought)
1631                 nr_sought = remove_duplicates_in_refs(sought, nr_sought);
1632
1633         if (args->no_dependents && !args->filter_options.choice) {
1634                 /*
1635                  * The protocol does not support requesting that only the
1636                  * wanted objects be sent, so approximate this by setting a
1637                  * "blob:none" filter if no filter is already set. This works
1638                  * for all object types: note that wanted blobs will still be
1639                  * sent because they are directly specified as a "want".
1640                  *
1641                  * NEEDSWORK: Add an option in the protocol to request that
1642                  * only the wanted objects be sent, and implement it.
1643                  */
1644                 parse_list_objects_filter(&args->filter_options, "blob:none");
1645         }
1646
1647         if (version != protocol_v2 && !ref) {
1648                 packet_flush(fd[1]);
1649                 die(_("no matching remote head"));
1650         }
1651         prepare_shallow_info(&si, shallow);
1652         if (version == protocol_v2)
1653                 ref_cpy = do_fetch_pack_v2(args, fd, ref, sought, nr_sought,
1654                                            pack_lockfile);
1655         else
1656                 ref_cpy = do_fetch_pack(args, fd, ref, sought, nr_sought,
1657                                         &si, pack_lockfile);
1658         reprepare_packed_git(the_repository);
1659
1660         if (!args->cloning && args->deepen) {
1661                 struct check_connected_options opt = CHECK_CONNECTED_INIT;
1662                 struct ref *iterator = ref_cpy;
1663                 opt.shallow_file = alternate_shallow_file;
1664                 if (args->deepen)
1665                         opt.is_deepening_fetch = 1;
1666                 if (check_connected(iterate_ref_map, &iterator, &opt)) {
1667                         error(_("remote did not send all necessary objects"));
1668                         free_refs(ref_cpy);
1669                         ref_cpy = NULL;
1670                         rollback_lock_file(&shallow_lock);
1671                         goto cleanup;
1672                 }
1673                 args->connectivity_checked = 1;
1674         }
1675
1676         update_shallow(args, sought, nr_sought, &si);
1677 cleanup:
1678         clear_shallow_info(&si);
1679         return ref_cpy;
1680 }
1681
1682 int report_unmatched_refs(struct ref **sought, int nr_sought)
1683 {
1684         int i, ret = 0;
1685
1686         for (i = 0; i < nr_sought; i++) {
1687                 if (!sought[i])
1688                         continue;
1689                 switch (sought[i]->match_status) {
1690                 case REF_MATCHED:
1691                         continue;
1692                 case REF_NOT_MATCHED:
1693                         error(_("no such remote ref %s"), sought[i]->name);
1694                         break;
1695                 case REF_UNADVERTISED_NOT_ALLOWED:
1696                         error(_("Server does not allow request for unadvertised object %s"),
1697                               sought[i]->name);
1698                         break;
1699                 }
1700                 ret = 1;
1701         }
1702         return ret;
1703 }