Merge branch 'maint'
[git] / builtin-unpack-objects.c
1 #include "builtin.h"
2 #include "cache.h"
3 #include "object.h"
4 #include "delta.h"
5 #include "pack.h"
6 #include "blob.h"
7 #include "commit.h"
8 #include "tag.h"
9 #include "tree.h"
10
11 #include <sys/time.h>
12
13 static int dry_run, quiet, recover, has_errors;
14 static const char unpack_usage[] = "git-unpack-objects [-n] [-q] [-r] < pack-file";
15
16 /* We always read in 4kB chunks. */
17 static unsigned char buffer[4096];
18 static unsigned long offset, len, consumed_bytes;
19 static SHA_CTX ctx;
20
21 /*
22  * Make sure at least "min" bytes are available in the buffer, and
23  * return the pointer to the buffer.
24  */
25 static void *fill(int min)
26 {
27         if (min <= len)
28                 return buffer + offset;
29         if (min > sizeof(buffer))
30                 die("cannot fill %d bytes", min);
31         if (offset) {
32                 SHA1_Update(&ctx, buffer, offset);
33                 memmove(buffer, buffer + offset, len);
34                 offset = 0;
35         }
36         do {
37                 int ret = xread(0, buffer + len, sizeof(buffer) - len);
38                 if (ret <= 0) {
39                         if (!ret)
40                                 die("early EOF");
41                         die("read error on input: %s", strerror(errno));
42                 }
43                 len += ret;
44         } while (len < min);
45         return buffer;
46 }
47
48 static void use(int bytes)
49 {
50         if (bytes > len)
51                 die("used more bytes than were available");
52         len -= bytes;
53         offset += bytes;
54         consumed_bytes += bytes;
55 }
56
57 static void *get_data(unsigned long size)
58 {
59         z_stream stream;
60         void *buf = xmalloc(size);
61
62         memset(&stream, 0, sizeof(stream));
63
64         stream.next_out = buf;
65         stream.avail_out = size;
66         stream.next_in = fill(1);
67         stream.avail_in = len;
68         inflateInit(&stream);
69
70         for (;;) {
71                 int ret = inflate(&stream, 0);
72                 use(len - stream.avail_in);
73                 if (stream.total_out == size && ret == Z_STREAM_END)
74                         break;
75                 if (ret != Z_OK) {
76                         error("inflate returned %d\n", ret);
77                         free(buf);
78                         buf = NULL;
79                         if (!recover)
80                                 exit(1);
81                         has_errors = 1;
82                         break;
83                 }
84                 stream.next_in = fill(1);
85                 stream.avail_in = len;
86         }
87         inflateEnd(&stream);
88         return buf;
89 }
90
91 struct delta_info {
92         unsigned char base_sha1[20];
93         unsigned long base_offset;
94         unsigned long size;
95         void *delta;
96         unsigned nr;
97         struct delta_info *next;
98 };
99
100 static struct delta_info *delta_list;
101
102 static void add_delta_to_list(unsigned nr, unsigned const char *base_sha1,
103                               unsigned long base_offset,
104                               void *delta, unsigned long size)
105 {
106         struct delta_info *info = xmalloc(sizeof(*info));
107
108         hashcpy(info->base_sha1, base_sha1);
109         info->base_offset = base_offset;
110         info->size = size;
111         info->delta = delta;
112         info->nr = nr;
113         info->next = delta_list;
114         delta_list = info;
115 }
116
117 struct obj_info {
118         unsigned long offset;
119         unsigned char sha1[20];
120 };
121
122 static struct obj_info *obj_list;
123
124 static void added_object(unsigned nr, const char *type, void *data,
125                          unsigned long size);
126
127 static void write_object(unsigned nr, void *buf, unsigned long size,
128                          const char *type)
129 {
130         if (write_sha1_file(buf, size, type, obj_list[nr].sha1) < 0)
131                 die("failed to write object");
132         added_object(nr, type, buf, size);
133 }
134
135 static void resolve_delta(unsigned nr, const char *type,
136                           void *base, unsigned long base_size,
137                           void *delta, unsigned long delta_size)
138 {
139         void *result;
140         unsigned long result_size;
141
142         result = patch_delta(base, base_size,
143                              delta, delta_size,
144                              &result_size);
145         if (!result)
146                 die("failed to apply delta");
147         free(delta);
148         write_object(nr, result, result_size, type);
149         free(result);
150 }
151
152 static void added_object(unsigned nr, const char *type, void *data,
153                          unsigned long size)
154 {
155         struct delta_info **p = &delta_list;
156         struct delta_info *info;
157
158         while ((info = *p) != NULL) {
159                 if (!hashcmp(info->base_sha1, obj_list[nr].sha1) ||
160                     info->base_offset == obj_list[nr].offset) {
161                         *p = info->next;
162                         p = &delta_list;
163                         resolve_delta(info->nr, type, data, size,
164                                       info->delta, info->size);
165                         free(info);
166                         continue;
167                 }
168                 p = &info->next;
169         }
170 }
171
172 static void unpack_non_delta_entry(enum object_type kind, unsigned long size,
173                                    unsigned nr)
174 {
175         void *buf = get_data(size);
176         const char *type;
177
178         switch (kind) {
179         case OBJ_COMMIT: type = commit_type; break;
180         case OBJ_TREE:   type = tree_type; break;
181         case OBJ_BLOB:   type = blob_type; break;
182         case OBJ_TAG:    type = tag_type; break;
183         default: die("bad type %d", kind);
184         }
185         if (!dry_run && buf)
186                 write_object(nr, buf, size, type);
187         free(buf);
188 }
189
190 static void unpack_delta_entry(enum object_type kind, unsigned long delta_size,
191                                unsigned nr)
192 {
193         void *delta_data, *base;
194         unsigned long base_size;
195         char type[20];
196         unsigned char base_sha1[20];
197
198         if (kind == OBJ_REF_DELTA) {
199                 hashcpy(base_sha1, fill(20));
200                 use(20);
201                 delta_data = get_data(delta_size);
202                 if (dry_run || !delta_data) {
203                         free(delta_data);
204                         return;
205                 }
206                 if (!has_sha1_file(base_sha1)) {
207                         hashcpy(obj_list[nr].sha1, null_sha1);
208                         add_delta_to_list(nr, base_sha1, 0, delta_data, delta_size);
209                         return;
210                 }
211         } else {
212                 unsigned base_found = 0;
213                 unsigned char *pack, c;
214                 unsigned long base_offset;
215                 unsigned lo, mid, hi;
216
217                 pack = fill(1);
218                 c = *pack;
219                 use(1);
220                 base_offset = c & 127;
221                 while (c & 128) {
222                         base_offset += 1;
223                         if (!base_offset || base_offset & ~(~0UL >> 7))
224                                 die("offset value overflow for delta base object");
225                         pack = fill(1);
226                         c = *pack;
227                         use(1);
228                         base_offset = (base_offset << 7) + (c & 127);
229                 }
230                 base_offset = obj_list[nr].offset - base_offset;
231
232                 delta_data = get_data(delta_size);
233                 if (dry_run || !delta_data) {
234                         free(delta_data);
235                         return;
236                 }
237                 lo = 0;
238                 hi = nr;
239                 while (lo < hi) {
240                         mid = (lo + hi)/2;
241                         if (base_offset < obj_list[mid].offset) {
242                                 hi = mid;
243                         } else if (base_offset > obj_list[mid].offset) {
244                                 lo = mid + 1;
245                         } else {
246                                 hashcpy(base_sha1, obj_list[mid].sha1);
247                                 base_found = !is_null_sha1(base_sha1);
248                                 break;
249                         }
250                 }
251                 if (!base_found) {
252                         /* The delta base object is itself a delta that
253                            has not been resolved yet. */
254                         hashcpy(obj_list[nr].sha1, null_sha1);
255                         add_delta_to_list(nr, null_sha1, base_offset, delta_data, delta_size);
256                         return;
257                 }
258         }
259
260         base = read_sha1_file(base_sha1, type, &base_size);
261         if (!base) {
262                 error("failed to read delta-pack base object %s",
263                       sha1_to_hex(base_sha1));
264                 if (!recover)
265                         exit(1);
266                 has_errors = 1;
267                 return;
268         }
269         resolve_delta(nr, type, base, base_size, delta_data, delta_size);
270         free(base);
271 }
272
273 static void unpack_one(unsigned nr, unsigned total)
274 {
275         unsigned shift;
276         unsigned char *pack, c;
277         unsigned long size;
278         enum object_type type;
279
280         obj_list[nr].offset = consumed_bytes;
281
282         pack = fill(1);
283         c = *pack;
284         use(1);
285         type = (c >> 4) & 7;
286         size = (c & 15);
287         shift = 4;
288         while (c & 0x80) {
289                 pack = fill(1);
290                 c = *pack;
291                 use(1);
292                 size += (c & 0x7f) << shift;
293                 shift += 7;
294         }
295         if (!quiet) {
296                 static unsigned long last_sec;
297                 static unsigned last_percent;
298                 struct timeval now;
299                 unsigned percentage = ((nr+1) * 100) / total;
300
301                 gettimeofday(&now, NULL);
302                 if (percentage != last_percent || now.tv_sec != last_sec) {
303                         last_sec = now.tv_sec;
304                         last_percent = percentage;
305                         fprintf(stderr, "%4u%% (%u/%u) done\r",
306                                         percentage, (nr+1), total);
307                 }
308         }
309         switch (type) {
310         case OBJ_COMMIT:
311         case OBJ_TREE:
312         case OBJ_BLOB:
313         case OBJ_TAG:
314                 unpack_non_delta_entry(type, size, nr);
315                 return;
316         case OBJ_REF_DELTA:
317         case OBJ_OFS_DELTA:
318                 unpack_delta_entry(type, size, nr);
319                 return;
320         default:
321                 error("bad object type %d", type);
322                 has_errors = 1;
323                 if (recover)
324                         return;
325                 exit(1);
326         }
327 }
328
329 static void unpack_all(void)
330 {
331         int i;
332         struct pack_header *hdr = fill(sizeof(struct pack_header));
333         unsigned nr_objects = ntohl(hdr->hdr_entries);
334
335         if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
336                 die("bad pack file");
337         if (!pack_version_ok(hdr->hdr_version))
338                 die("unknown pack file version %d", ntohl(hdr->hdr_version));
339         fprintf(stderr, "Unpacking %d objects\n", nr_objects);
340
341         obj_list = xmalloc(nr_objects * sizeof(*obj_list));
342         use(sizeof(struct pack_header));
343         for (i = 0; i < nr_objects; i++)
344                 unpack_one(i, nr_objects);
345         if (delta_list)
346                 die("unresolved deltas left after unpacking");
347 }
348
349 int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
350 {
351         int i;
352         unsigned char sha1[20];
353
354         git_config(git_default_config);
355
356         quiet = !isatty(2);
357
358         for (i = 1 ; i < argc; i++) {
359                 const char *arg = argv[i];
360
361                 if (*arg == '-') {
362                         if (!strcmp(arg, "-n")) {
363                                 dry_run = 1;
364                                 continue;
365                         }
366                         if (!strcmp(arg, "-q")) {
367                                 quiet = 1;
368                                 continue;
369                         }
370                         if (!strcmp(arg, "-r")) {
371                                 recover = 1;
372                                 continue;
373                         }
374                         if (!strncmp(arg, "--pack_header=", 14)) {
375                                 struct pack_header *hdr;
376                                 char *c;
377
378                                 hdr = (struct pack_header *)buffer;
379                                 hdr->hdr_signature = htonl(PACK_SIGNATURE);
380                                 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
381                                 if (*c != ',')
382                                         die("bad %s", arg);
383                                 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
384                                 if (*c)
385                                         die("bad %s", arg);
386                                 len = sizeof(*hdr);
387                                 continue;
388                         }
389                         usage(unpack_usage);
390                 }
391
392                 /* We don't take any non-flag arguments now.. Maybe some day */
393                 usage(unpack_usage);
394         }
395         SHA1_Init(&ctx);
396         unpack_all();
397         SHA1_Update(&ctx, buffer, offset);
398         SHA1_Final(sha1, &ctx);
399         if (hashcmp(fill(20), sha1))
400                 die("final sha1 did not match");
401         use(20);
402
403         /* Write the last part of the buffer to stdout */
404         while (len) {
405                 int ret = xwrite(1, buffer + offset, len);
406                 if (ret <= 0)
407                         break;
408                 len -= ret;
409                 offset += ret;
410         }
411
412         /* All done */
413         if (!quiet)
414                 fprintf(stderr, "\n");
415         return has_errors;
416 }