9 #include "fetch-pack.h"
11 #include "run-command.h"
13 static int transfer_unpack_limit = -1;
14 static int fetch_unpack_limit = -1;
15 static int unpack_limit = 100;
16 static struct fetch_pack_args args = {
17 /* .uploadpack = */ "git-upload-pack",
20 static const char fetch_pack_usage[] =
21 "git-fetch-pack [--all] [--quiet|-q] [--keep|-k] [--thin] [--upload-pack=<git-upload-pack>] [--depth=<n>] [--no-progress] [-v] [<host>:]<directory> [<refs>...]";
23 #define COMPLETE (1U << 0)
24 #define COMMON (1U << 1)
25 #define COMMON_REF (1U << 2)
26 #define SEEN (1U << 3)
27 #define POPPED (1U << 4)
30 * After sending this many "have"s if we do not get any new ACK , we
31 * give up traversing our history.
33 #define MAX_IN_VAIN 256
35 static struct commit_list *rev_list;
36 static int non_common_revs, multi_ack, use_sideband;
38 static void rev_list_push(struct commit *commit, int mark)
40 if (!(commit->object.flags & mark)) {
41 commit->object.flags |= mark;
43 if (!(commit->object.parsed))
44 if (parse_commit(commit))
47 insert_by_date(commit, &rev_list);
49 if (!(commit->object.flags & COMMON))
54 static int rev_list_insert_ref(const char *path, const unsigned char *sha1, int flag, void *cb_data)
56 struct object *o = deref_tag(parse_object(sha1), path, 0);
58 if (o && o->type == OBJ_COMMIT)
59 rev_list_push((struct commit *)o, SEEN);
65 This function marks a rev and its ancestors as common.
66 In some cases, it is desirable to mark only the ancestors (for example
67 when only the server does not yet know that they are common).
70 static void mark_common(struct commit *commit,
71 int ancestors_only, int dont_parse)
73 if (commit != NULL && !(commit->object.flags & COMMON)) {
74 struct object *o = (struct object *)commit;
79 if (!(o->flags & SEEN))
80 rev_list_push(commit, SEEN);
82 struct commit_list *parents;
84 if (!ancestors_only && !(o->flags & POPPED))
86 if (!o->parsed && !dont_parse)
87 if (parse_commit(commit))
90 for (parents = commit->parents;
92 parents = parents->next)
93 mark_common(parents->item, 0, dont_parse);
99 Get the next rev to send, ignoring the common.
102 static const unsigned char* get_rev(void)
104 struct commit *commit = NULL;
106 while (commit == NULL) {
108 struct commit_list *parents = NULL;
110 if (rev_list == NULL || non_common_revs == 0)
113 commit = rev_list->item;
114 if (!(commit->object.parsed))
115 if (!parse_commit(commit))
116 parents = commit->parents;
118 commit->object.flags |= POPPED;
119 if (!(commit->object.flags & COMMON))
122 if (commit->object.flags & COMMON) {
123 /* do not send "have", and ignore ancestors */
125 mark = COMMON | SEEN;
126 } else if (commit->object.flags & COMMON_REF)
127 /* send "have", and ignore ancestors */
128 mark = COMMON | SEEN;
130 /* send "have", also for its ancestors */
134 if (!(parents->item->object.flags & SEEN))
135 rev_list_push(parents->item, mark);
137 mark_common(parents->item, 1, 0);
138 parents = parents->next;
141 rev_list = rev_list->next;
144 return commit->object.sha1;
147 static int find_common(int fd[2], unsigned char *result_sha1,
151 int count = 0, flushes = 0, retval;
152 const unsigned char *sha1;
153 unsigned in_vain = 0;
154 int got_continue = 0;
156 for_each_ref(rev_list_insert_ref, NULL);
159 for ( ; refs ; refs = refs->next) {
160 unsigned char *remote = refs->old_sha1;
164 * If that object is complete (i.e. it is an ancestor of a
165 * local ref), we tell them we have it but do not have to
166 * tell them about its ancestors, which they already know
169 * We use lookup_object here because we are only
170 * interested in the case we *know* the object is
171 * reachable and we have already scanned it.
173 if (((o = lookup_object(remote)) != NULL) &&
174 (o->flags & COMPLETE)) {
179 packet_write(fd[1], "want %s%s%s%s%s%s%s\n",
181 (multi_ack ? " multi_ack" : ""),
182 (use_sideband == 2 ? " side-band-64k" : ""),
183 (use_sideband == 1 ? " side-band" : ""),
184 (args.use_thin_pack ? " thin-pack" : ""),
185 (args.no_progress ? " no-progress" : ""),
188 packet_write(fd[1], "want %s\n", sha1_to_hex(remote));
191 if (is_repository_shallow())
192 write_shallow_commits(fd[1], 1);
194 packet_write(fd[1], "deepen %d", args.depth);
199 if (args.depth > 0) {
201 unsigned char sha1[20];
204 while ((len = packet_read_line(fd[0], line, sizeof(line)))) {
205 if (!prefixcmp(line, "shallow ")) {
206 if (get_sha1_hex(line + 8, sha1))
207 die("invalid shallow line: %s", line);
208 register_shallow(sha1);
211 if (!prefixcmp(line, "unshallow ")) {
212 if (get_sha1_hex(line + 10, sha1))
213 die("invalid unshallow line: %s", line);
214 if (!lookup_object(sha1))
215 die("object not found: %s", line);
216 /* make sure that it is parsed as shallow */
217 if (!parse_object(sha1))
218 die("error in object: %s", line);
219 if (unregister_shallow(sha1))
220 die("no shallow found: %s", line);
223 die("expected shallow/unshallow, got %s", line);
229 while ((sha1 = get_rev())) {
230 packet_write(fd[1], "have %s\n", sha1_to_hex(sha1));
232 fprintf(stderr, "have %s\n", sha1_to_hex(sha1));
234 if (!(31 & ++count)) {
241 * We keep one window "ahead" of the other side, and
242 * will wait for an ACK only on the next one
248 ack = get_ack(fd[0], result_sha1);
249 if (args.verbose && ack)
250 fprintf(stderr, "got ack %d %s\n", ack,
251 sha1_to_hex(result_sha1));
257 } else if (ack == 2) {
258 struct commit *commit =
259 lookup_commit(result_sha1);
260 mark_common(commit, 0, 1);
267 if (got_continue && MAX_IN_VAIN < in_vain) {
269 fprintf(stderr, "giving up\n");
275 packet_write(fd[1], "done\n");
277 fprintf(stderr, "done\n");
282 while (flushes || multi_ack) {
283 int ack = get_ack(fd[0], result_sha1);
286 fprintf(stderr, "got ack (%d) %s\n", ack,
287 sha1_to_hex(result_sha1));
298 static struct commit_list *complete;
300 static int mark_complete(const char *path, const unsigned char *sha1, int flag, void *cb_data)
302 struct object *o = parse_object(sha1);
304 while (o && o->type == OBJ_TAG) {
305 struct tag *t = (struct tag *) o;
307 break; /* broken repository */
308 o->flags |= COMPLETE;
309 o = parse_object(t->tagged->sha1);
311 if (o && o->type == OBJ_COMMIT) {
312 struct commit *commit = (struct commit *)o;
313 commit->object.flags |= COMPLETE;
314 insert_by_date(commit, &complete);
319 static void mark_recent_complete_commits(unsigned long cutoff)
321 while (complete && cutoff <= complete->item->date) {
323 fprintf(stderr, "Marking %s as complete\n",
324 sha1_to_hex(complete->item->object.sha1));
325 pop_most_recent_commit(&complete, COMPLETE);
329 static void filter_refs(struct ref **refs, int nr_match, char **match)
331 struct ref **return_refs;
332 struct ref *newlist = NULL;
333 struct ref **newtail = &newlist;
334 struct ref *ref, *next;
335 struct ref *fastarray[32];
337 if (nr_match && !args.fetch_all) {
338 if (ARRAY_SIZE(fastarray) < nr_match)
339 return_refs = xcalloc(nr_match, sizeof(struct ref *));
341 return_refs = fastarray;
342 memset(return_refs, 0, sizeof(struct ref *) * nr_match);
348 for (ref = *refs; ref; ref = next) {
350 if (!memcmp(ref->name, "refs/", 5) &&
351 check_ref_format(ref->name + 5))
353 else if (args.fetch_all &&
354 (!args.depth || prefixcmp(ref->name, "refs/tags/") )) {
357 newtail = &ref->next;
361 int order = path_match(ref->name, nr_match, match);
363 return_refs[order-1] = ref;
364 continue; /* we will link it later */
370 if (!args.fetch_all) {
372 for (i = 0; i < nr_match; i++) {
373 ref = return_refs[i];
377 newtail = &ref->next;
380 if (return_refs != fastarray)
386 static int everything_local(struct ref **refs, int nr_match, char **match)
390 unsigned long cutoff = 0;
392 save_commit_buffer = 0;
394 for (ref = *refs; ref; ref = ref->next) {
397 o = parse_object(ref->old_sha1);
401 /* We already have it -- which may mean that we were
402 * in sync with the other side at some time after
403 * that (it is OK if we guess wrong here).
405 if (o->type == OBJ_COMMIT) {
406 struct commit *commit = (struct commit *)o;
407 if (!cutoff || cutoff < commit->date)
408 cutoff = commit->date;
413 for_each_ref(mark_complete, NULL);
415 mark_recent_complete_commits(cutoff);
419 * Mark all complete remote refs as common refs.
420 * Don't mark them common yet; the server has to be told so first.
422 for (ref = *refs; ref; ref = ref->next) {
423 struct object *o = deref_tag(lookup_object(ref->old_sha1),
426 if (!o || o->type != OBJ_COMMIT || !(o->flags & COMPLETE))
429 if (!(o->flags & SEEN)) {
430 rev_list_push((struct commit *)o, COMMON_REF | SEEN);
432 mark_common((struct commit *)o, 1, 1);
436 filter_refs(refs, nr_match, match);
438 for (retval = 1, ref = *refs; ref ; ref = ref->next) {
439 const unsigned char *remote = ref->old_sha1;
440 unsigned char local[20];
443 o = lookup_object(remote);
444 if (!o || !(o->flags & COMPLETE)) {
449 "want %s (%s)\n", sha1_to_hex(remote),
454 hashcpy(ref->new_sha1, local);
458 "already have %s (%s)\n", sha1_to_hex(remote),
464 static int sideband_demux(int fd, void *data)
468 return recv_sideband("fetch-pack", xd[0], fd, 2);
471 static int get_pack(int xd[2], char **pack_lockfile)
474 const char *argv[20];
478 int do_keep = args.keep_pack;
479 struct child_process cmd;
481 memset(&demux, 0, sizeof(demux));
483 /* xd[] is talking with upload-pack; subprocess reads from
484 * xd[0], spits out band#2 to stderr, and feeds us band#1
485 * through demux->out.
487 demux.proc = sideband_demux;
489 if (start_async(&demux))
490 die("fetch-pack: unable to fork off sideband"
496 memset(&cmd, 0, sizeof(cmd));
500 if (!args.keep_pack && unpack_limit) {
501 struct pack_header header;
503 if (read_pack_header(demux.out, &header))
504 die("protocol error: bad pack header");
505 snprintf(hdr_arg, sizeof(hdr_arg), "--pack_header=%u,%u",
506 ntohl(header.hdr_version), ntohl(header.hdr_entries));
507 if (ntohl(header.hdr_entries) < unpack_limit)
516 *av++ = "index-pack";
518 if (!args.quiet && !args.no_progress)
520 if (args.use_thin_pack)
521 *av++ = "--fix-thin";
522 if (args.lock_pack || unpack_limit) {
523 int s = sprintf(keep_arg,
524 "--keep=fetch-pack %d on ", getpid());
525 if (gethostname(keep_arg + s, sizeof(keep_arg) - s))
526 strcpy(keep_arg + s, "localhost");
531 *av++ = "unpack-objects";
541 if (start_command(&cmd))
542 die("fetch-pack: unable to fork off %s", argv[0]);
543 if (do_keep && pack_lockfile) {
544 *pack_lockfile = index_pack_lockfile(cmd.out);
548 if (finish_command(&cmd))
549 die("%s failed", argv[0]);
550 if (use_sideband && finish_async(&demux))
551 die("error in sideband demultiplexer");
555 static struct ref *do_fetch_pack(int fd[2],
556 const struct ref *orig_ref,
559 char **pack_lockfile)
561 struct ref *ref = copy_ref_list(orig_ref);
562 unsigned char sha1[20];
564 if (is_repository_shallow() && !server_supports("shallow"))
565 die("Server does not support shallow clients");
566 if (server_supports("multi_ack")) {
568 fprintf(stderr, "Server supports multi_ack\n");
571 if (server_supports("side-band-64k")) {
573 fprintf(stderr, "Server supports side-band-64k\n");
576 else if (server_supports("side-band")) {
578 fprintf(stderr, "Server supports side-band\n");
581 if (everything_local(&ref, nr_match, match)) {
585 if (find_common(fd, sha1, ref) < 0)
587 /* When cloning, it is not unusual to have
590 fprintf(stderr, "warning: no common commits\n");
592 if (get_pack(fd, pack_lockfile))
593 die("git-fetch-pack: fetch failed.");
599 static int remove_duplicates(int nr_heads, char **heads)
603 for (src = dst = 0; src < nr_heads; src++) {
604 /* If heads[src] is different from any of
605 * heads[0..dst], push it in.
608 for (i = 0; i < dst; i++) {
609 if (!strcmp(heads[i], heads[src]))
615 heads[dst] = heads[src];
621 static int fetch_pack_config(const char *var, const char *value)
623 if (strcmp(var, "fetch.unpacklimit") == 0) {
624 fetch_unpack_limit = git_config_int(var, value);
628 if (strcmp(var, "transfer.unpacklimit") == 0) {
629 transfer_unpack_limit = git_config_int(var, value);
633 return git_default_config(var, value);
636 static struct lock_file lock;
638 static void fetch_pack_setup(void)
640 static int did_setup;
643 git_config(fetch_pack_config);
644 if (0 <= transfer_unpack_limit)
645 unpack_limit = transfer_unpack_limit;
646 else if (0 <= fetch_unpack_limit)
647 unpack_limit = fetch_unpack_limit;
651 int cmd_fetch_pack(int argc, const char **argv, const char *prefix)
653 int i, ret, nr_heads;
654 struct ref *ref = NULL;
655 char *dest = NULL, **heads;
657 struct child_process *conn;
661 for (i = 1; i < argc; i++) {
662 const char *arg = argv[i];
665 if (!prefixcmp(arg, "--upload-pack=")) {
666 args.uploadpack = arg + 14;
669 if (!prefixcmp(arg, "--exec=")) {
670 args.uploadpack = arg + 7;
673 if (!strcmp("--quiet", arg) || !strcmp("-q", arg)) {
677 if (!strcmp("--keep", arg) || !strcmp("-k", arg)) {
678 args.lock_pack = args.keep_pack;
682 if (!strcmp("--thin", arg)) {
683 args.use_thin_pack = 1;
686 if (!strcmp("--all", arg)) {
690 if (!strcmp("-v", arg)) {
694 if (!prefixcmp(arg, "--depth=")) {
695 args.depth = strtol(arg + 8, NULL, 0);
698 if (!strcmp("--no-progress", arg)) {
699 args.no_progress = 1;
702 usage(fetch_pack_usage);
705 heads = (char **)(argv + i + 1);
706 nr_heads = argc - i - 1;
710 usage(fetch_pack_usage);
712 conn = git_connect(fd, (char *)dest, args.uploadpack,
713 args.verbose ? CONNECT_VERBOSE : 0);
715 get_remote_heads(fd[0], &ref, 0, NULL, 0);
717 ref = fetch_pack(&args, fd, conn, ref, dest, nr_heads, heads, NULL);
720 if (finish_connect(conn))
727 if (!ret && nr_heads) {
728 /* If the heads to pull were given, we should have
729 * consumed all of them by matching the remote.
730 * Otherwise, 'git-fetch remote no-such-ref' would
731 * silently succeed without issuing an error.
733 for (i = 0; i < nr_heads; i++)
734 if (heads[i] && heads[i][0]) {
735 error("no such remote ref %s", heads[i]);
741 sha1_to_hex(ref->old_sha1), ref->name);
748 struct ref *fetch_pack(struct fetch_pack_args *my_args,
749 int fd[], struct child_process *conn,
750 const struct ref *ref,
754 char **pack_lockfile)
760 memcpy(&args, my_args, sizeof(args));
761 if (args.depth > 0) {
762 if (stat(git_path("shallow"), &st))
766 if (heads && nr_heads)
767 nr_heads = remove_duplicates(nr_heads, heads);
770 die("no matching remote head");
772 ref_cpy = do_fetch_pack(fd, ref, nr_heads, heads, pack_lockfile);
774 if (args.depth > 0) {
775 struct cache_time mtime;
776 char *shallow = git_path("shallow");
779 mtime.sec = st.st_mtime;
781 mtime.usec = st.st_mtim.usec;
783 if (stat(shallow, &st)) {
785 die("shallow file was removed during fetch");
786 } else if (st.st_mtime != mtime.sec
788 || st.st_mtim.usec != mtime.usec
791 die("shallow file was changed during fetch");
793 fd = hold_lock_file_for_update(&lock, shallow, 1);
794 if (!write_shallow_commits(fd, 0)) {
796 rollback_lock_file(&lock);
798 commit_lock_file(&lock);