Merge branch 'jk/safe-pipe-capture' into maint-2.10
[git] / builtin / fast-export.c
1 /*
2  * "git fast-export" builtin command
3  *
4  * Copyright (C) 2007 Johannes E. Schindelin
5  */
6 #include "builtin.h"
7 #include "cache.h"
8 #include "refs.h"
9 #include "commit.h"
10 #include "object.h"
11 #include "tag.h"
12 #include "diff.h"
13 #include "diffcore.h"
14 #include "log-tree.h"
15 #include "revision.h"
16 #include "decorate.h"
17 #include "string-list.h"
18 #include "utf8.h"
19 #include "parse-options.h"
20 #include "quote.h"
21 #include "remote.h"
22 #include "blob.h"
23
24 static const char *fast_export_usage[] = {
25         N_("git fast-export [rev-list-opts]"),
26         NULL
27 };
28
29 static int progress;
30 static enum { ABORT, VERBATIM, WARN, WARN_STRIP, STRIP } signed_tag_mode = ABORT;
31 static enum { ERROR, DROP, REWRITE } tag_of_filtered_mode = ERROR;
32 static int fake_missing_tagger;
33 static int use_done_feature;
34 static int no_data;
35 static int full_tree;
36 static struct string_list extra_refs = STRING_LIST_INIT_NODUP;
37 static struct refspec *refspecs;
38 static int refspecs_nr;
39 static int anonymize;
40
41 static int parse_opt_signed_tag_mode(const struct option *opt,
42                                      const char *arg, int unset)
43 {
44         if (unset || !strcmp(arg, "abort"))
45                 signed_tag_mode = ABORT;
46         else if (!strcmp(arg, "verbatim") || !strcmp(arg, "ignore"))
47                 signed_tag_mode = VERBATIM;
48         else if (!strcmp(arg, "warn"))
49                 signed_tag_mode = WARN;
50         else if (!strcmp(arg, "warn-strip"))
51                 signed_tag_mode = WARN_STRIP;
52         else if (!strcmp(arg, "strip"))
53                 signed_tag_mode = STRIP;
54         else
55                 return error("Unknown signed-tags mode: %s", arg);
56         return 0;
57 }
58
59 static int parse_opt_tag_of_filtered_mode(const struct option *opt,
60                                           const char *arg, int unset)
61 {
62         if (unset || !strcmp(arg, "abort"))
63                 tag_of_filtered_mode = ERROR;
64         else if (!strcmp(arg, "drop"))
65                 tag_of_filtered_mode = DROP;
66         else if (!strcmp(arg, "rewrite"))
67                 tag_of_filtered_mode = REWRITE;
68         else
69                 return error("Unknown tag-of-filtered mode: %s", arg);
70         return 0;
71 }
72
73 static struct decoration idnums;
74 static uint32_t last_idnum;
75
76 static int has_unshown_parent(struct commit *commit)
77 {
78         struct commit_list *parent;
79
80         for (parent = commit->parents; parent; parent = parent->next)
81                 if (!(parent->item->object.flags & SHOWN) &&
82                     !(parent->item->object.flags & UNINTERESTING))
83                         return 1;
84         return 0;
85 }
86
87 struct anonymized_entry {
88         struct hashmap_entry hash;
89         const char *orig;
90         size_t orig_len;
91         const char *anon;
92         size_t anon_len;
93 };
94
95 static int anonymized_entry_cmp(const void *va, const void *vb,
96                                 const void *data)
97 {
98         const struct anonymized_entry *a = va, *b = vb;
99         return a->orig_len != b->orig_len ||
100                 memcmp(a->orig, b->orig, a->orig_len);
101 }
102
103 /*
104  * Basically keep a cache of X->Y so that we can repeatedly replace
105  * the same anonymized string with another. The actual generation
106  * is farmed out to the generate function.
107  */
108 static const void *anonymize_mem(struct hashmap *map,
109                                  void *(*generate)(const void *, size_t *),
110                                  const void *orig, size_t *len)
111 {
112         struct anonymized_entry key, *ret;
113
114         if (!map->cmpfn)
115                 hashmap_init(map, anonymized_entry_cmp, 0);
116
117         hashmap_entry_init(&key, memhash(orig, *len));
118         key.orig = orig;
119         key.orig_len = *len;
120         ret = hashmap_get(map, &key, NULL);
121
122         if (!ret) {
123                 ret = xmalloc(sizeof(*ret));
124                 hashmap_entry_init(&ret->hash, key.hash.hash);
125                 ret->orig = xstrdup(orig);
126                 ret->orig_len = *len;
127                 ret->anon = generate(orig, len);
128                 ret->anon_len = *len;
129                 hashmap_put(map, ret);
130         }
131
132         *len = ret->anon_len;
133         return ret->anon;
134 }
135
136 /*
137  * We anonymize each component of a path individually,
138  * so that paths a/b and a/c will share a common root.
139  * The paths are cached via anonymize_mem so that repeated
140  * lookups for "a" will yield the same value.
141  */
142 static void anonymize_path(struct strbuf *out, const char *path,
143                            struct hashmap *map,
144                            void *(*generate)(const void *, size_t *))
145 {
146         while (*path) {
147                 const char *end_of_component = strchrnul(path, '/');
148                 size_t len = end_of_component - path;
149                 const char *c = anonymize_mem(map, generate, path, &len);
150                 strbuf_add(out, c, len);
151                 path = end_of_component;
152                 if (*path)
153                         strbuf_addch(out, *path++);
154         }
155 }
156
157 /* Since intptr_t is C99, we do not use it here */
158 static inline uint32_t *mark_to_ptr(uint32_t mark)
159 {
160         return ((uint32_t *)NULL) + mark;
161 }
162
163 static inline uint32_t ptr_to_mark(void * mark)
164 {
165         return (uint32_t *)mark - (uint32_t *)NULL;
166 }
167
168 static inline void mark_object(struct object *object, uint32_t mark)
169 {
170         add_decoration(&idnums, object, mark_to_ptr(mark));
171 }
172
173 static inline void mark_next_object(struct object *object)
174 {
175         mark_object(object, ++last_idnum);
176 }
177
178 static int get_object_mark(struct object *object)
179 {
180         void *decoration = lookup_decoration(&idnums, object);
181         if (!decoration)
182                 return 0;
183         return ptr_to_mark(decoration);
184 }
185
186 static void show_progress(void)
187 {
188         static int counter = 0;
189         if (!progress)
190                 return;
191         if ((++counter % progress) == 0)
192                 printf("progress %d objects\n", counter);
193 }
194
195 /*
196  * Ideally we would want some transformation of the blob data here
197  * that is unreversible, but would still be the same size and have
198  * the same data relationship to other blobs (so that we get the same
199  * delta and packing behavior as the original). But the first and last
200  * requirements there are probably mutually exclusive, so let's take
201  * the easy way out for now, and just generate arbitrary content.
202  *
203  * There's no need to cache this result with anonymize_mem, since
204  * we already handle blob content caching with marks.
205  */
206 static char *anonymize_blob(unsigned long *size)
207 {
208         static int counter;
209         struct strbuf out = STRBUF_INIT;
210         strbuf_addf(&out, "anonymous blob %d", counter++);
211         *size = out.len;
212         return strbuf_detach(&out, NULL);
213 }
214
215 static void export_blob(const unsigned char *sha1)
216 {
217         unsigned long size;
218         enum object_type type;
219         char *buf;
220         struct object *object;
221         int eaten;
222
223         if (no_data)
224                 return;
225
226         if (is_null_sha1(sha1))
227                 return;
228
229         object = lookup_object(sha1);
230         if (object && object->flags & SHOWN)
231                 return;
232
233         if (anonymize) {
234                 buf = anonymize_blob(&size);
235                 object = (struct object *)lookup_blob(sha1);
236                 eaten = 0;
237         } else {
238                 buf = read_sha1_file(sha1, &type, &size);
239                 if (!buf)
240                         die ("Could not read blob %s", sha1_to_hex(sha1));
241                 if (check_sha1_signature(sha1, buf, size, typename(type)) < 0)
242                         die("sha1 mismatch in blob %s", sha1_to_hex(sha1));
243                 object = parse_object_buffer(sha1, type, size, buf, &eaten);
244         }
245
246         if (!object)
247                 die("Could not read blob %s", sha1_to_hex(sha1));
248
249         mark_next_object(object);
250
251         printf("blob\nmark :%"PRIu32"\ndata %lu\n", last_idnum, size);
252         if (size && fwrite(buf, size, 1, stdout) != 1)
253                 die_errno ("Could not write blob '%s'", sha1_to_hex(sha1));
254         printf("\n");
255
256         show_progress();
257
258         object->flags |= SHOWN;
259         if (!eaten)
260                 free(buf);
261 }
262
263 static int depth_first(const void *a_, const void *b_)
264 {
265         const struct diff_filepair *a = *((const struct diff_filepair **)a_);
266         const struct diff_filepair *b = *((const struct diff_filepair **)b_);
267         const char *name_a, *name_b;
268         int len_a, len_b, len;
269         int cmp;
270
271         name_a = a->one ? a->one->path : a->two->path;
272         name_b = b->one ? b->one->path : b->two->path;
273
274         len_a = strlen(name_a);
275         len_b = strlen(name_b);
276         len = (len_a < len_b) ? len_a : len_b;
277
278         /* strcmp will sort 'd' before 'd/e', we want 'd/e' before 'd' */
279         cmp = memcmp(name_a, name_b, len);
280         if (cmp)
281                 return cmp;
282         cmp = len_b - len_a;
283         if (cmp)
284                 return cmp;
285         /*
286          * Move 'R'ename entries last so that all references of the file
287          * appear in the output before it is renamed (e.g., when a file
288          * was copied and renamed in the same commit).
289          */
290         return (a->status == 'R') - (b->status == 'R');
291 }
292
293 static void print_path_1(const char *path)
294 {
295         int need_quote = quote_c_style(path, NULL, NULL, 0);
296         if (need_quote)
297                 quote_c_style(path, NULL, stdout, 0);
298         else if (strchr(path, ' '))
299                 printf("\"%s\"", path);
300         else
301                 printf("%s", path);
302 }
303
304 static void *anonymize_path_component(const void *path, size_t *len)
305 {
306         static int counter;
307         struct strbuf out = STRBUF_INIT;
308         strbuf_addf(&out, "path%d", counter++);
309         return strbuf_detach(&out, len);
310 }
311
312 static void print_path(const char *path)
313 {
314         if (!anonymize)
315                 print_path_1(path);
316         else {
317                 static struct hashmap paths;
318                 static struct strbuf anon = STRBUF_INIT;
319
320                 anonymize_path(&anon, path, &paths, anonymize_path_component);
321                 print_path_1(anon.buf);
322                 strbuf_reset(&anon);
323         }
324 }
325
326 static void *generate_fake_sha1(const void *old, size_t *len)
327 {
328         static uint32_t counter = 1; /* avoid null sha1 */
329         unsigned char *out = xcalloc(20, 1);
330         put_be32(out + 16, counter++);
331         return out;
332 }
333
334 static const unsigned char *anonymize_sha1(const unsigned char *sha1)
335 {
336         static struct hashmap sha1s;
337         size_t len = 20;
338         return anonymize_mem(&sha1s, generate_fake_sha1, sha1, &len);
339 }
340
341 static void show_filemodify(struct diff_queue_struct *q,
342                             struct diff_options *options, void *data)
343 {
344         int i;
345
346         /*
347          * Handle files below a directory first, in case they are all deleted
348          * and the directory changes to a file or symlink.
349          */
350         qsort(q->queue, q->nr, sizeof(q->queue[0]), depth_first);
351
352         for (i = 0; i < q->nr; i++) {
353                 struct diff_filespec *ospec = q->queue[i]->one;
354                 struct diff_filespec *spec = q->queue[i]->two;
355
356                 switch (q->queue[i]->status) {
357                 case DIFF_STATUS_DELETED:
358                         printf("D ");
359                         print_path(spec->path);
360                         putchar('\n');
361                         break;
362
363                 case DIFF_STATUS_COPIED:
364                 case DIFF_STATUS_RENAMED:
365                         printf("%c ", q->queue[i]->status);
366                         print_path(ospec->path);
367                         putchar(' ');
368                         print_path(spec->path);
369                         putchar('\n');
370
371                         if (!oidcmp(&ospec->oid, &spec->oid) &&
372                             ospec->mode == spec->mode)
373                                 break;
374                         /* fallthrough */
375
376                 case DIFF_STATUS_TYPE_CHANGED:
377                 case DIFF_STATUS_MODIFIED:
378                 case DIFF_STATUS_ADDED:
379                         /*
380                          * Links refer to objects in another repositories;
381                          * output the SHA-1 verbatim.
382                          */
383                         if (no_data || S_ISGITLINK(spec->mode))
384                                 printf("M %06o %s ", spec->mode,
385                                        sha1_to_hex(anonymize ?
386                                                    anonymize_sha1(spec->oid.hash) :
387                                                    spec->oid.hash));
388                         else {
389                                 struct object *object = lookup_object(spec->oid.hash);
390                                 printf("M %06o :%d ", spec->mode,
391                                        get_object_mark(object));
392                         }
393                         print_path(spec->path);
394                         putchar('\n');
395                         break;
396
397                 default:
398                         die("Unexpected comparison status '%c' for %s, %s",
399                                 q->queue[i]->status,
400                                 ospec->path ? ospec->path : "none",
401                                 spec->path ? spec->path : "none");
402                 }
403         }
404 }
405
406 static const char *find_encoding(const char *begin, const char *end)
407 {
408         const char *needle = "\nencoding ";
409         char *bol, *eol;
410
411         bol = memmem(begin, end ? end - begin : strlen(begin),
412                      needle, strlen(needle));
413         if (!bol)
414                 return git_commit_encoding;
415         bol += strlen(needle);
416         eol = strchrnul(bol, '\n');
417         *eol = '\0';
418         return bol;
419 }
420
421 static void *anonymize_ref_component(const void *old, size_t *len)
422 {
423         static int counter;
424         struct strbuf out = STRBUF_INIT;
425         strbuf_addf(&out, "ref%d", counter++);
426         return strbuf_detach(&out, len);
427 }
428
429 static const char *anonymize_refname(const char *refname)
430 {
431         /*
432          * If any of these prefixes is found, we will leave it intact
433          * so that tags remain tags and so forth.
434          */
435         static const char *prefixes[] = {
436                 "refs/heads/",
437                 "refs/tags/",
438                 "refs/remotes/",
439                 "refs/"
440         };
441         static struct hashmap refs;
442         static struct strbuf anon = STRBUF_INIT;
443         int i;
444
445         /*
446          * We also leave "master" as a special case, since it does not reveal
447          * anything interesting.
448          */
449         if (!strcmp(refname, "refs/heads/master"))
450                 return refname;
451
452         strbuf_reset(&anon);
453         for (i = 0; i < ARRAY_SIZE(prefixes); i++) {
454                 if (skip_prefix(refname, prefixes[i], &refname)) {
455                         strbuf_addstr(&anon, prefixes[i]);
456                         break;
457                 }
458         }
459
460         anonymize_path(&anon, refname, &refs, anonymize_ref_component);
461         return anon.buf;
462 }
463
464 /*
465  * We do not even bother to cache commit messages, as they are unlikely
466  * to be repeated verbatim, and it is not that interesting when they are.
467  */
468 static char *anonymize_commit_message(const char *old)
469 {
470         static int counter;
471         return xstrfmt("subject %d\n\nbody\n", counter++);
472 }
473
474 static struct hashmap idents;
475 static void *anonymize_ident(const void *old, size_t *len)
476 {
477         static int counter;
478         struct strbuf out = STRBUF_INIT;
479         strbuf_addf(&out, "User %d <user%d@example.com>", counter, counter);
480         counter++;
481         return strbuf_detach(&out, len);
482 }
483
484 /*
485  * Our strategy here is to anonymize the names and email addresses,
486  * but keep timestamps intact, as they influence things like traversal
487  * order (and by themselves should not be too revealing).
488  */
489 static void anonymize_ident_line(const char **beg, const char **end)
490 {
491         static struct strbuf buffers[] = { STRBUF_INIT, STRBUF_INIT };
492         static unsigned which_buffer;
493
494         struct strbuf *out;
495         struct ident_split split;
496         const char *end_of_header;
497
498         out = &buffers[which_buffer++];
499         which_buffer %= ARRAY_SIZE(buffers);
500         strbuf_reset(out);
501
502         /* skip "committer", "author", "tagger", etc */
503         end_of_header = strchr(*beg, ' ');
504         if (!end_of_header)
505                 die("BUG: malformed line fed to anonymize_ident_line: %.*s",
506                     (int)(*end - *beg), *beg);
507         end_of_header++;
508         strbuf_add(out, *beg, end_of_header - *beg);
509
510         if (!split_ident_line(&split, end_of_header, *end - end_of_header) &&
511             split.date_begin) {
512                 const char *ident;
513                 size_t len;
514
515                 len = split.mail_end - split.name_begin;
516                 ident = anonymize_mem(&idents, anonymize_ident,
517                                       split.name_begin, &len);
518                 strbuf_add(out, ident, len);
519                 strbuf_addch(out, ' ');
520                 strbuf_add(out, split.date_begin, split.tz_end - split.date_begin);
521         } else {
522                 strbuf_addstr(out, "Malformed Ident <malformed@example.com> 0 -0000");
523         }
524
525         *beg = out->buf;
526         *end = out->buf + out->len;
527 }
528
529 static void handle_commit(struct commit *commit, struct rev_info *rev)
530 {
531         int saved_output_format = rev->diffopt.output_format;
532         const char *commit_buffer;
533         const char *author, *author_end, *committer, *committer_end;
534         const char *encoding, *message;
535         char *reencoded = NULL;
536         struct commit_list *p;
537         const char *refname;
538         int i;
539
540         rev->diffopt.output_format = DIFF_FORMAT_CALLBACK;
541
542         parse_commit_or_die(commit);
543         commit_buffer = get_commit_buffer(commit, NULL);
544         author = strstr(commit_buffer, "\nauthor ");
545         if (!author)
546                 die ("Could not find author in commit %s",
547                      oid_to_hex(&commit->object.oid));
548         author++;
549         author_end = strchrnul(author, '\n');
550         committer = strstr(author_end, "\ncommitter ");
551         if (!committer)
552                 die ("Could not find committer in commit %s",
553                      oid_to_hex(&commit->object.oid));
554         committer++;
555         committer_end = strchrnul(committer, '\n');
556         message = strstr(committer_end, "\n\n");
557         encoding = find_encoding(committer_end, message);
558         if (message)
559                 message += 2;
560
561         if (commit->parents &&
562             get_object_mark(&commit->parents->item->object) != 0 &&
563             !full_tree) {
564                 parse_commit_or_die(commit->parents->item);
565                 diff_tree_sha1(commit->parents->item->tree->object.oid.hash,
566                                commit->tree->object.oid.hash, "", &rev->diffopt);
567         }
568         else
569                 diff_root_tree_sha1(commit->tree->object.oid.hash,
570                                     "", &rev->diffopt);
571
572         /* Export the referenced blobs, and remember the marks. */
573         for (i = 0; i < diff_queued_diff.nr; i++)
574                 if (!S_ISGITLINK(diff_queued_diff.queue[i]->two->mode))
575                         export_blob(diff_queued_diff.queue[i]->two->oid.hash);
576
577         refname = commit->util;
578         if (anonymize) {
579                 refname = anonymize_refname(refname);
580                 anonymize_ident_line(&committer, &committer_end);
581                 anonymize_ident_line(&author, &author_end);
582         }
583
584         mark_next_object(&commit->object);
585         if (anonymize)
586                 reencoded = anonymize_commit_message(message);
587         else if (!is_encoding_utf8(encoding))
588                 reencoded = reencode_string(message, "UTF-8", encoding);
589         if (!commit->parents)
590                 printf("reset %s\n", refname);
591         printf("commit %s\nmark :%"PRIu32"\n%.*s\n%.*s\ndata %u\n%s",
592                refname, last_idnum,
593                (int)(author_end - author), author,
594                (int)(committer_end - committer), committer,
595                (unsigned)(reencoded
596                           ? strlen(reencoded) : message
597                           ? strlen(message) : 0),
598                reencoded ? reencoded : message ? message : "");
599         free(reencoded);
600         unuse_commit_buffer(commit, commit_buffer);
601
602         for (i = 0, p = commit->parents; p; p = p->next) {
603                 int mark = get_object_mark(&p->item->object);
604                 if (!mark)
605                         continue;
606                 if (i == 0)
607                         printf("from :%d\n", mark);
608                 else
609                         printf("merge :%d\n", mark);
610                 i++;
611         }
612
613         if (full_tree)
614                 printf("deleteall\n");
615         log_tree_diff_flush(rev);
616         rev->diffopt.output_format = saved_output_format;
617
618         printf("\n");
619
620         show_progress();
621 }
622
623 static void *anonymize_tag(const void *old, size_t *len)
624 {
625         static int counter;
626         struct strbuf out = STRBUF_INIT;
627         strbuf_addf(&out, "tag message %d", counter++);
628         return strbuf_detach(&out, len);
629 }
630
631 static void handle_tail(struct object_array *commits, struct rev_info *revs)
632 {
633         struct commit *commit;
634         while (commits->nr) {
635                 commit = (struct commit *)commits->objects[commits->nr - 1].item;
636                 if (has_unshown_parent(commit))
637                         return;
638                 handle_commit(commit, revs);
639                 commits->nr--;
640         }
641 }
642
643 static void handle_tag(const char *name, struct tag *tag)
644 {
645         unsigned long size;
646         enum object_type type;
647         char *buf;
648         const char *tagger, *tagger_end, *message;
649         size_t message_size = 0;
650         struct object *tagged;
651         int tagged_mark;
652         struct commit *p;
653
654         /* Trees have no identifier in fast-export output, thus we have no way
655          * to output tags of trees, tags of tags of trees, etc.  Simply omit
656          * such tags.
657          */
658         tagged = tag->tagged;
659         while (tagged->type == OBJ_TAG) {
660                 tagged = ((struct tag *)tagged)->tagged;
661         }
662         if (tagged->type == OBJ_TREE) {
663                 warning("Omitting tag %s,\nsince tags of trees (or tags of tags of trees, etc.) are not supported.",
664                         oid_to_hex(&tag->object.oid));
665                 return;
666         }
667
668         buf = read_sha1_file(tag->object.oid.hash, &type, &size);
669         if (!buf)
670                 die ("Could not read tag %s", oid_to_hex(&tag->object.oid));
671         message = memmem(buf, size, "\n\n", 2);
672         if (message) {
673                 message += 2;
674                 message_size = strlen(message);
675         }
676         tagger = memmem(buf, message ? message - buf : size, "\ntagger ", 8);
677         if (!tagger) {
678                 if (fake_missing_tagger)
679                         tagger = "tagger Unspecified Tagger "
680                                 "<unspecified-tagger> 0 +0000";
681                 else
682                         tagger = "";
683                 tagger_end = tagger + strlen(tagger);
684         } else {
685                 tagger++;
686                 tagger_end = strchrnul(tagger, '\n');
687                 if (anonymize)
688                         anonymize_ident_line(&tagger, &tagger_end);
689         }
690
691         if (anonymize) {
692                 name = anonymize_refname(name);
693                 if (message) {
694                         static struct hashmap tags;
695                         message = anonymize_mem(&tags, anonymize_tag,
696                                                 message, &message_size);
697                 }
698         }
699
700         /* handle signed tags */
701         if (message) {
702                 const char *signature = strstr(message,
703                                                "\n-----BEGIN PGP SIGNATURE-----\n");
704                 if (signature)
705                         switch(signed_tag_mode) {
706                         case ABORT:
707                                 die ("Encountered signed tag %s; use "
708                                      "--signed-tags=<mode> to handle it.",
709                                      oid_to_hex(&tag->object.oid));
710                         case WARN:
711                                 warning ("Exporting signed tag %s",
712                                          oid_to_hex(&tag->object.oid));
713                                 /* fallthru */
714                         case VERBATIM:
715                                 break;
716                         case WARN_STRIP:
717                                 warning ("Stripping signature from tag %s",
718                                          oid_to_hex(&tag->object.oid));
719                                 /* fallthru */
720                         case STRIP:
721                                 message_size = signature + 1 - message;
722                                 break;
723                         }
724         }
725
726         /* handle tag->tagged having been filtered out due to paths specified */
727         tagged = tag->tagged;
728         tagged_mark = get_object_mark(tagged);
729         if (!tagged_mark) {
730                 switch(tag_of_filtered_mode) {
731                 case ABORT:
732                         die ("Tag %s tags unexported object; use "
733                              "--tag-of-filtered-object=<mode> to handle it.",
734                              oid_to_hex(&tag->object.oid));
735                 case DROP:
736                         /* Ignore this tag altogether */
737                         return;
738                 case REWRITE:
739                         if (tagged->type != OBJ_COMMIT) {
740                                 die ("Tag %s tags unexported %s!",
741                                      oid_to_hex(&tag->object.oid),
742                                      typename(tagged->type));
743                         }
744                         p = (struct commit *)tagged;
745                         for (;;) {
746                                 if (p->parents && p->parents->next)
747                                         break;
748                                 if (p->object.flags & UNINTERESTING)
749                                         break;
750                                 if (!(p->object.flags & TREESAME))
751                                         break;
752                                 if (!p->parents)
753                                         die ("Can't find replacement commit for tag %s\n",
754                                              oid_to_hex(&tag->object.oid));
755                                 p = p->parents->item;
756                         }
757                         tagged_mark = get_object_mark(&p->object);
758                 }
759         }
760
761         if (starts_with(name, "refs/tags/"))
762                 name += 10;
763         printf("tag %s\nfrom :%d\n%.*s%sdata %d\n%.*s\n",
764                name, tagged_mark,
765                (int)(tagger_end - tagger), tagger,
766                tagger == tagger_end ? "" : "\n",
767                (int)message_size, (int)message_size, message ? message : "");
768 }
769
770 static struct commit *get_commit(struct rev_cmdline_entry *e, char *full_name)
771 {
772         switch (e->item->type) {
773         case OBJ_COMMIT:
774                 return (struct commit *)e->item;
775         case OBJ_TAG: {
776                 struct tag *tag = (struct tag *)e->item;
777
778                 /* handle nested tags */
779                 while (tag && tag->object.type == OBJ_TAG) {
780                         parse_object(tag->object.oid.hash);
781                         string_list_append(&extra_refs, full_name)->util = tag;
782                         tag = (struct tag *)tag->tagged;
783                 }
784                 if (!tag)
785                         die("Tag %s points nowhere?", e->name);
786                 return (struct commit *)tag;
787                 break;
788         }
789         default:
790                 return NULL;
791         }
792 }
793
794 static void get_tags_and_duplicates(struct rev_cmdline_info *info)
795 {
796         int i;
797
798         for (i = 0; i < info->nr; i++) {
799                 struct rev_cmdline_entry *e = info->rev + i;
800                 unsigned char sha1[20];
801                 struct commit *commit;
802                 char *full_name;
803
804                 if (e->flags & UNINTERESTING)
805                         continue;
806
807                 if (dwim_ref(e->name, strlen(e->name), sha1, &full_name) != 1)
808                         continue;
809
810                 if (refspecs) {
811                         char *private;
812                         private = apply_refspecs(refspecs, refspecs_nr, full_name);
813                         if (private) {
814                                 free(full_name);
815                                 full_name = private;
816                         }
817                 }
818
819                 commit = get_commit(e, full_name);
820                 if (!commit) {
821                         warning("%s: Unexpected object of type %s, skipping.",
822                                 e->name,
823                                 typename(e->item->type));
824                         continue;
825                 }
826
827                 switch(commit->object.type) {
828                 case OBJ_COMMIT:
829                         break;
830                 case OBJ_BLOB:
831                         export_blob(commit->object.oid.hash);
832                         continue;
833                 default: /* OBJ_TAG (nested tags) is already handled */
834                         warning("Tag points to object of unexpected type %s, skipping.",
835                                 typename(commit->object.type));
836                         continue;
837                 }
838
839                 /*
840                  * This ref will not be updated through a commit, lets make
841                  * sure it gets properly updated eventually.
842                  */
843                 if (commit->util || commit->object.flags & SHOWN)
844                         string_list_append(&extra_refs, full_name)->util = commit;
845                 if (!commit->util)
846                         commit->util = full_name;
847         }
848 }
849
850 static void handle_tags_and_duplicates(void)
851 {
852         struct commit *commit;
853         int i;
854
855         for (i = extra_refs.nr - 1; i >= 0; i--) {
856                 const char *name = extra_refs.items[i].string;
857                 struct object *object = extra_refs.items[i].util;
858                 switch (object->type) {
859                 case OBJ_TAG:
860                         handle_tag(name, (struct tag *)object);
861                         break;
862                 case OBJ_COMMIT:
863                         if (anonymize)
864                                 name = anonymize_refname(name);
865                         /* create refs pointing to already seen commits */
866                         commit = (struct commit *)object;
867                         printf("reset %s\nfrom :%d\n\n", name,
868                                get_object_mark(&commit->object));
869                         show_progress();
870                         break;
871                 }
872         }
873 }
874
875 static void export_marks(char *file)
876 {
877         unsigned int i;
878         uint32_t mark;
879         struct object_decoration *deco = idnums.hash;
880         FILE *f;
881         int e = 0;
882
883         f = fopen_for_writing(file);
884         if (!f)
885                 die_errno("Unable to open marks file %s for writing.", file);
886
887         for (i = 0; i < idnums.size; i++) {
888                 if (deco->base && deco->base->type == 1) {
889                         mark = ptr_to_mark(deco->decoration);
890                         if (fprintf(f, ":%"PRIu32" %s\n", mark,
891                                 oid_to_hex(&deco->base->oid)) < 0) {
892                             e = 1;
893                             break;
894                         }
895                 }
896                 deco++;
897         }
898
899         e |= ferror(f);
900         e |= fclose(f);
901         if (e)
902                 error("Unable to write marks file %s.", file);
903 }
904
905 static void import_marks(char *input_file)
906 {
907         char line[512];
908         FILE *f = fopen(input_file, "r");
909         if (!f)
910                 die_errno("cannot read '%s'", input_file);
911
912         while (fgets(line, sizeof(line), f)) {
913                 uint32_t mark;
914                 char *line_end, *mark_end;
915                 unsigned char sha1[20];
916                 struct object *object;
917                 struct commit *commit;
918                 enum object_type type;
919
920                 line_end = strchr(line, '\n');
921                 if (line[0] != ':' || !line_end)
922                         die("corrupt mark line: %s", line);
923                 *line_end = '\0';
924
925                 mark = strtoumax(line + 1, &mark_end, 10);
926                 if (!mark || mark_end == line + 1
927                         || *mark_end != ' ' || get_sha1_hex(mark_end + 1, sha1))
928                         die("corrupt mark line: %s", line);
929
930                 if (last_idnum < mark)
931                         last_idnum = mark;
932
933                 type = sha1_object_info(sha1, NULL);
934                 if (type < 0)
935                         die("object not found: %s", sha1_to_hex(sha1));
936
937                 if (type != OBJ_COMMIT)
938                         /* only commits */
939                         continue;
940
941                 commit = lookup_commit(sha1);
942                 if (!commit)
943                         die("not a commit? can't happen: %s", sha1_to_hex(sha1));
944
945                 object = &commit->object;
946
947                 if (object->flags & SHOWN)
948                         error("Object %s already has a mark", sha1_to_hex(sha1));
949
950                 mark_object(object, mark);
951
952                 object->flags |= SHOWN;
953         }
954         fclose(f);
955 }
956
957 static void handle_deletes(void)
958 {
959         int i;
960         for (i = 0; i < refspecs_nr; i++) {
961                 struct refspec *refspec = &refspecs[i];
962                 if (*refspec->src)
963                         continue;
964
965                 printf("reset %s\nfrom %s\n\n",
966                                 refspec->dst, sha1_to_hex(null_sha1));
967         }
968 }
969
970 int cmd_fast_export(int argc, const char **argv, const char *prefix)
971 {
972         struct rev_info revs;
973         struct object_array commits = OBJECT_ARRAY_INIT;
974         struct commit *commit;
975         char *export_filename = NULL, *import_filename = NULL;
976         uint32_t lastimportid;
977         struct string_list refspecs_list = STRING_LIST_INIT_NODUP;
978         struct option options[] = {
979                 OPT_INTEGER(0, "progress", &progress,
980                             N_("show progress after <n> objects")),
981                 OPT_CALLBACK(0, "signed-tags", &signed_tag_mode, N_("mode"),
982                              N_("select handling of signed tags"),
983                              parse_opt_signed_tag_mode),
984                 OPT_CALLBACK(0, "tag-of-filtered-object", &tag_of_filtered_mode, N_("mode"),
985                              N_("select handling of tags that tag filtered objects"),
986                              parse_opt_tag_of_filtered_mode),
987                 OPT_STRING(0, "export-marks", &export_filename, N_("file"),
988                              N_("Dump marks to this file")),
989                 OPT_STRING(0, "import-marks", &import_filename, N_("file"),
990                              N_("Import marks from this file")),
991                 OPT_BOOL(0, "fake-missing-tagger", &fake_missing_tagger,
992                          N_("Fake a tagger when tags lack one")),
993                 OPT_BOOL(0, "full-tree", &full_tree,
994                          N_("Output full tree for each commit")),
995                 OPT_BOOL(0, "use-done-feature", &use_done_feature,
996                              N_("Use the done feature to terminate the stream")),
997                 OPT_BOOL(0, "no-data", &no_data, N_("Skip output of blob data")),
998                 OPT_STRING_LIST(0, "refspec", &refspecs_list, N_("refspec"),
999                              N_("Apply refspec to exported refs")),
1000                 OPT_BOOL(0, "anonymize", &anonymize, N_("anonymize output")),
1001                 OPT_END()
1002         };
1003
1004         if (argc == 1)
1005                 usage_with_options (fast_export_usage, options);
1006
1007         /* we handle encodings */
1008         git_config(git_default_config, NULL);
1009
1010         init_revisions(&revs, prefix);
1011         revs.topo_order = 1;
1012         revs.show_source = 1;
1013         revs.rewrite_parents = 1;
1014         argc = parse_options(argc, argv, prefix, options, fast_export_usage,
1015                         PARSE_OPT_KEEP_ARGV0 | PARSE_OPT_KEEP_UNKNOWN);
1016         argc = setup_revisions(argc, argv, &revs, NULL);
1017         if (argc > 1)
1018                 usage_with_options (fast_export_usage, options);
1019
1020         if (refspecs_list.nr) {
1021                 const char **refspecs_str;
1022                 int i;
1023
1024                 ALLOC_ARRAY(refspecs_str, refspecs_list.nr);
1025                 for (i = 0; i < refspecs_list.nr; i++)
1026                         refspecs_str[i] = refspecs_list.items[i].string;
1027
1028                 refspecs_nr = refspecs_list.nr;
1029                 refspecs = parse_fetch_refspec(refspecs_nr, refspecs_str);
1030
1031                 string_list_clear(&refspecs_list, 1);
1032                 free(refspecs_str);
1033         }
1034
1035         if (use_done_feature)
1036                 printf("feature done\n");
1037
1038         if (import_filename)
1039                 import_marks(import_filename);
1040         lastimportid = last_idnum;
1041
1042         if (import_filename && revs.prune_data.nr)
1043                 full_tree = 1;
1044
1045         get_tags_and_duplicates(&revs.cmdline);
1046
1047         if (prepare_revision_walk(&revs))
1048                 die("revision walk setup failed");
1049         revs.diffopt.format_callback = show_filemodify;
1050         DIFF_OPT_SET(&revs.diffopt, RECURSIVE);
1051         while ((commit = get_revision(&revs))) {
1052                 if (has_unshown_parent(commit)) {
1053                         add_object_array(&commit->object, NULL, &commits);
1054                 }
1055                 else {
1056                         handle_commit(commit, &revs);
1057                         handle_tail(&commits, &revs);
1058                 }
1059         }
1060
1061         handle_tags_and_duplicates();
1062         handle_deletes();
1063
1064         if (export_filename && lastimportid != last_idnum)
1065                 export_marks(export_filename);
1066
1067         if (use_done_feature)
1068                 printf("done\n");
1069
1070         free_refspec(refspecs_nr, refspecs);
1071
1072         return 0;
1073 }