9 #include "fetch-pack.h"
11 #include "run-command.h"
13 static int transfer_unpack_limit = -1;
14 static int fetch_unpack_limit = -1;
15 static int unpack_limit = 100;
16 static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
20 static const char fetch_pack_usage[] =
21 "git fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--include-tag] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
23 #define COMPLETE (1U << 0)
24 #define COMMON (1U << 1)
25 #define COMMON_REF (1U << 2)
26 #define SEEN (1U << 3)
27 #define POPPED (1U << 4)
32 * After sending this many "have"s if we do not get any new ACK , we
33 * give up traversing our history.
35 #define MAX_IN_VAIN 256
37 static struct commit_list *rev_list;
38 static int non_common_revs, multi_ack, use_sideband;
40 static void rev_list_push(struct commit *commit, int mark)
42 if (!(commit->object.flags & mark)) {
43 commit->object.flags |= mark;
45 if (!(commit->object.parsed))
46 if (parse_commit(commit))
49 insert_by_date(commit, &rev_list);
51 if (!(commit->object.flags & COMMON))
56 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
58 struct object *o = deref_tag(parse_object(sha1), path, 0);
60 if (o && o->type == OBJ_COMMIT)
61 rev_list_push((struct commit *)o, SEEN);
66 static int clear_marks(const char *path, const unsigned char *sha1, int flag, void *cb_data)
68 struct object *o = deref_tag(parse_object(sha1), path, 0);
70 if (o && o->type == OBJ_COMMIT)
71 clear_commit_marks((struct commit *)o,
72 COMMON | COMMON_REF | SEEN | POPPED);
77 This function marks a rev and its ancestors as common.
78 In some cases, it is desirable to mark only the ancestors (for example
79 when only the server does not yet know that they are common).
82 static void mark_common(struct commit *commit,
83 int ancestors_only, int dont_parse)
85 if (commit != NULL && !(commit->object.flags & COMMON)) {
86 struct object *o = (struct object *)commit;
91 if (!(o->flags & SEEN))
92 rev_list_push(commit, SEEN);
94 struct commit_list *parents;
96 if (!ancestors_only && !(o->flags & POPPED))
98 if (!o->parsed && !dont_parse)
99 if (parse_commit(commit))
102 for (parents = commit->parents;
104 parents = parents->next)
105 mark_common(parents->item, 0, dont_parse);
111 Get the next rev to send, ignoring the common.
114 static const unsigned char* get_rev(void)
116 struct commit *commit = NULL;
118 while (commit == NULL) {
120 struct commit_list *parents;
122 if (rev_list == NULL || non_common_revs == 0)
125 commit = rev_list->item;
126 if (!commit->object.parsed)
127 parse_commit(commit);
128 parents = commit->parents;
130 commit->object.flags |= POPPED;
131 if (!(commit->object.flags & COMMON))
134 if (commit->object.flags & COMMON) {
135 /* do not send "have", and ignore ancestors */
137 mark = COMMON | SEEN;
138 } else if (commit->object.flags & COMMON_REF)
139 /* send "have", and ignore ancestors */
140 mark = COMMON | SEEN;
142 /* send "have", also for its ancestors */
146 if (!(parents->item->object.flags & SEEN))
147 rev_list_push(parents->item, mark);
149 mark_common(parents->item, 1, 0);
150 parents = parents->next;
153 rev_list = rev_list->next;
156 return commit->object.sha1;
159 static int find_common(int fd[2], unsigned char *result_sha1,
163 int count = 0, flushes = 0, retval;
164 const unsigned char *sha1;
165 unsigned in_vain = 0;
166 int got_continue = 0;
169 for_each_ref(clear_marks, NULL);
172 for_each_ref(rev_list_insert_ref, NULL);
175 for ( ; refs ; refs = refs->next) {
176 unsigned char *remote = refs->old_sha1;
180 * If that object is complete (i.e. it is an ancestor of a
181 * local ref), we tell them we have it but do not have to
182 * tell them about its ancestors, which they already know
185 * We use lookup_object here because we are only
186 * interested in the case we *know* the object is
187 * reachable and we have already scanned it.
189 if (((o = lookup_object(remote)) != NULL) &&
190 (o->flags & COMPLETE)) {
195 packet_write(fd[1], "want %s%s%s%s%s%s%s%s\n",
197 (multi_ack ? " multi_ack" : ""),
198 (use_sideband == 2 ? " side-band-64k" : ""),
199 (use_sideband == 1 ? " side-band" : ""),
200 (args.use_thin_pack ? " thin-pack" : ""),
201 (args.no_progress ? " no-progress" : ""),
202 (args.include_tag ? " include-tag" : ""),
205 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
208 if (is_repository_shallow())
209 write_shallow_commits(fd[1], 1);
211 packet_write(fd[1], "deepen %d", args.depth);
216 if (args.depth > 0) {
218 unsigned char sha1[20];
221 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
222 if (!prefixcmp(line, "shallow ")) {
223 if (get_sha1_hex(line + 8, sha1))
224 die("invalid shallow line: %s", line);
225 register_shallow(sha1);
228 if (!prefixcmp(line, "unshallow ")) {
229 if (get_sha1_hex(line + 10, sha1))
230 die("invalid unshallow line: %s", line);
231 if (!lookup_object(sha1))
232 die("object not found: %s", line);
233 /* make sure that it is parsed as shallow */
234 if (!parse_object(sha1))
235 die("error in object: %s", line);
236 if (unregister_shallow(sha1))
237 die("no shallow found: %s", line);
240 die("expected shallow/unshallow, got %s", line);
246 while ((sha1 = get_rev())) {
247 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
249 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
251 if (!(31 & ++count)) {
258 * We keep one window "ahead" of the other side, and
259 * will wait for an ACK only on the next one
265 ack = get_ack(fd[0], result_sha1);
266 if (args.verbose && ack)
267 fprintf(stderr, "got ack %d %s\n", ack,
268 sha1_to_hex(result_sha1));
274 } else if (ack == 2) {
275 struct commit *commit =
276 lookup_commit(result_sha1);
277 mark_common(commit, 0, 1);
284 if (got_continue && MAX_IN_VAIN < in_vain) {
286 fprintf(stderr, "giving up\n");
292 packet_write(fd[1], "done\n");
294 fprintf(stderr, "done\n");
299 while (flushes || multi_ack) {
300 int ack = get_ack(fd[0], result_sha1);
303 fprintf(stderr, "got ack (%d) %s\n", ack,
304 sha1_to_hex(result_sha1));
312 /* it is no error to fetch into a completely empty repo */
313 return count ? retval : 0;
316 static struct commit_list *complete;
318 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
320 struct object *o = parse_object(sha1);
322 while (o && o->type == OBJ_TAG) {
323 struct tag *t = (struct tag *) o;
325 break; /* broken repository */
326 o->flags |= COMPLETE;
327 o = parse_object(t->tagged->sha1);
329 if (o && o->type == OBJ_COMMIT) {
330 struct commit *commit = (struct commit *)o;
331 commit->object.flags |= COMPLETE;
332 insert_by_date(commit, &complete);
337 static void mark_recent_complete_commits(unsigned long cutoff)
339 while (complete && cutoff <= complete->item->date) {
341 fprintf(stderr, "Marking %s as complete\n",
342 sha1_to_hex(complete->item->object.sha1));
343 pop_most_recent_commit(&complete, COMPLETE);
347 static void filter_refs(struct ref **refs, int nr_match, char **match)
349 struct ref **return_refs;
350 struct ref *newlist = NULL;
351 struct ref **newtail = &newlist;
352 struct ref *ref, *next;
353 struct ref *fastarray[32];
355 if (nr_match && !args.fetch_all) {
356 if (ARRAY_SIZE(fastarray) < nr_match)
357 return_refs = xcalloc(nr_match, sizeof(struct ref *));
359 return_refs = fastarray;
360 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
366 for (ref = *refs; ref; ref = next) {
368 if (!memcmp(ref->name, "refs/", 5) &&
369 check_ref_format(ref->name + 5))
371 else if (args.fetch_all &&
372 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
375 newtail = &ref->next;
379 int order = path_match(ref->name, nr_match, match);
381 return_refs[order-1] = ref;
382 continue; /* we will link it later */
388 if (!args.fetch_all) {
390 for (i = 0; i < nr_match; i++) {
391 ref = return_refs[i];
395 newtail = &ref->next;
398 if (return_refs != fastarray)
404 static int everything_local(struct ref **refs, int nr_match, char **match)
408 unsigned long cutoff = 0;
410 save_commit_buffer = 0;
412 for (ref = *refs; ref; ref = ref->next) {
415 o = parse_object(ref->old_sha1);
419 /* We already have it -- which may mean that we were
420 * in sync with the other side at some time after
421 * that (it is OK if we guess wrong here).
423 if (o->type == OBJ_COMMIT) {
424 struct commit *commit = (struct commit *)o;
425 if (!cutoff || cutoff < commit->date)
426 cutoff = commit->date;
431 for_each_ref(mark_complete, NULL);
433 mark_recent_complete_commits(cutoff);
437 * Mark all complete remote refs as common refs.
438 * Don't mark them common yet; the server has to be told so first.
440 for (ref = *refs; ref; ref = ref->next) {
441 struct object *o = deref_tag(lookup_object(ref->old_sha1),
444 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
447 if (!(o->flags & SEEN)) {
448 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
450 mark_common((struct commit *)o, 1, 1);
454 filter_refs(refs, nr_match, match);
456 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
457 const unsigned char *remote = ref->old_sha1;
458 unsigned char local[20];
461 o = lookup_object(remote);
462 if (!o || !(o->flags & COMPLETE)) {
467 "want %s (%s)\n", sha1_to_hex(remote),
472 hashcpy(ref->new_sha1, local);
476 "already have %s (%s)\n", sha1_to_hex(remote),
482 static int sideband_demux(int fd, void *data)
486 return recv_sideband("fetch-pack", xd[0], fd, 2);
489 static int get_pack(int xd[2], char **pack_lockfile)
492 const char *argv[20];
496 int do_keep = args.keep_pack;
497 struct child_process cmd;
499 memset(&demux, 0, sizeof(demux));
501 /* xd[] is talking with upload-pack; subprocess reads from
502 * xd[0], spits out band#2 to stderr, and feeds us band#1
503 * through demux->out.
505 demux.proc = sideband_demux;
507 if (start_async(&demux))
508 die("fetch-pack: unable to fork off sideband"
514 memset(&cmd, 0, sizeof(cmd));
518 if (!args.keep_pack && unpack_limit) {
519 struct pack_header header;
521 if (read_pack_header(demux.out, &header))
522 die("protocol error: bad pack header");
523 snprintf(hdr_arg, sizeof(hdr_arg),
524 "--pack_header=%"PRIu32",%"PRIu32,
525 ntohl(header.hdr_version), ntohl(header.hdr_entries));
526 if (ntohl(header.hdr_entries) < unpack_limit)
535 *av++ = "index-pack";
537 if (!args.quiet && !args.no_progress)
539 if (args.use_thin_pack)
540 *av++ = "--fix-thin";
541 if (args.lock_pack || unpack_limit) {
542 int s = sprintf(keep_arg,
543 "--keep=fetch-pack %d on ", getpid());
544 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
545 strcpy(keep_arg + s, "localhost");
550 *av++ = "unpack-objects";
560 if (start_command(&cmd))
561 die("fetch-pack: unable to fork off %s", argv[0]);
562 if (do_keep && pack_lockfile) {
563 *pack_lockfile = index_pack_lockfile(cmd.out);
567 if (finish_command(&cmd))
568 die("%s failed", argv[0]);
569 if (use_sideband && finish_async(&demux))
570 die("error in sideband demultiplexer");
574 static struct ref *do_fetch_pack(int fd[2],
575 const struct ref *orig_ref,
578 char **pack_lockfile)
580 struct ref *ref = copy_ref_list(orig_ref);
581 unsigned char sha1[20];
583 if (is_repository_shallow() && !server_supports("shallow"))
584 die("Server does not support shallow clients");
585 if (server_supports("multi_ack")) {
587 fprintf(stderr, "Server supports multi_ack\n");
590 if (server_supports("side-band-64k")) {
592 fprintf(stderr, "Server supports side-band-64k\n");
595 else if (server_supports("side-band")) {
597 fprintf(stderr, "Server supports side-band\n");
600 if (everything_local(&ref, nr_match, match)) {
604 if (find_common(fd, sha1, ref) < 0)
606 /* When cloning, it is not unusual to have
609 fprintf(stderr, "warning: no common commits\n");
611 if (get_pack(fd, pack_lockfile))
612 die("git-fetch-pack: fetch failed.");
618 static int remove_duplicates(int nr_heads, char **heads)
622 for (src = dst = 0; src < nr_heads; src++) {
623 /* If heads[src] is different from any of
624 * heads[0..dst], push it in.
627 for (i = 0; i < dst; i++) {
628 if (!strcmp(heads[i], heads[src]))
634 heads[dst] = heads[src];
640 static int fetch_pack_config(const char *var, const char *value, void *cb)
642 if (strcmp(var, "fetch.unpacklimit") == 0) {
643 fetch_unpack_limit = git_config_int(var, value);
647 if (strcmp(var, "transfer.unpacklimit") == 0) {
648 transfer_unpack_limit = git_config_int(var, value);
652 return git_default_config(var, value, cb);
655 static struct lock_file lock;
657 static void fetch_pack_setup(void)
659 static int did_setup;
662 git_config(fetch_pack_config, NULL);
663 if (0 <= transfer_unpack_limit)
664 unpack_limit = transfer_unpack_limit;
665 else if (0 <= fetch_unpack_limit)
666 unpack_limit = fetch_unpack_limit;
670 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
672 int i, ret, nr_heads;
673 struct ref *ref = NULL;
674 char *dest = NULL, **heads;
676 struct child_process *conn;
680 for (i = 1; i < argc; i++) {
681 const char *arg = argv[i];
684 if (!prefixcmp(arg, "--upload-pack=")) {
685 args.uploadpack = arg + 14;
688 if (!prefixcmp(arg, "--exec=")) {
689 args.uploadpack = arg + 7;
692 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
696 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
697 args.lock_pack = args.keep_pack;
701 if (!strcmp("--thin", arg)) {
702 args.use_thin_pack = 1;
705 if (!strcmp("--include-tag", arg)) {
706 args.include_tag = 1;
709 if (!strcmp("--all", arg)) {
713 if (!strcmp("-v", arg)) {
717 if (!prefixcmp(arg, "--depth=")) {
718 args.depth = strtol(arg + 8, NULL, 0);
721 if (!strcmp("--no-progress", arg)) {
722 args.no_progress = 1;
725 usage(fetch_pack_usage);
728 heads = (char **)(argv + i + 1);
729 nr_heads = argc - i - 1;
733 usage(fetch_pack_usage);
735 conn = git_connect(fd, (char *)dest, args.uploadpack,
736 args.verbose ? CONNECT_VERBOSE : 0);
738 get_remote_heads(fd[0], &ref, 0, NULL, 0);
740 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
743 if (finish_connect(conn))
750 if (!ret && nr_heads) {
751 /* If the heads to pull were given, we should have
752 * consumed all of them by matching the remote.
753 * Otherwise, 'git-fetch remote no-such-ref' would
754 * silently succeed without issuing an error.
756 for (i = 0; i < nr_heads; i++)
757 if (heads[i] && heads[i][0]) {
758 error("no such remote ref %s", heads[i]);
764 sha1_to_hex(ref->old_sha1), ref->name);
771 struct ref *fetch_pack(struct fetch_pack_args *my_args,
772 int fd[], struct child_process *conn,
773 const struct ref *ref,
777 char **pack_lockfile)
783 memcpy(&args, my_args, sizeof(args));
784 if (args.depth > 0) {
785 if (stat(git_path("shallow"), &st))
789 if (heads && nr_heads)
790 nr_heads = remove_duplicates(nr_heads, heads);
793 die("no matching remote head");
795 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
797 if (args.depth > 0) {
798 struct cache_time mtime;
799 char *shallow = git_path("shallow");
802 mtime.sec = st.st_mtime;
804 mtime.usec = st.st_mtim.usec;
806 if (stat(shallow, &st)) {
808 die("shallow file was removed during fetch");
809 } else if (st.st_mtime != mtime.sec
811 || st.st_mtim.usec != mtime.usec
814 die("shallow file was changed during fetch");
816 fd = hold_lock_file_for_update(&lock, shallow, 1);
817 if (!write_shallow_commits(fd, 0)) {
819 rollback_lock_file(&lock);
821 commit_lock_file(&lock);
825 reprepare_packed_git();