Merge branch 'sb/submodule-recursive-fetch-gets-the-tip'
[git] / builtin / unpack-objects.c
1 #include "builtin.h"
2 #include "cache.h"
3 #include "config.h"
4 #include "object-store.h"
5 #include "object.h"
6 #include "delta.h"
7 #include "pack.h"
8 #include "blob.h"
9 #include "commit.h"
10 #include "tag.h"
11 #include "tree.h"
12 #include "tree-walk.h"
13 #include "progress.h"
14 #include "decorate.h"
15 #include "fsck.h"
16
17 static int dry_run, quiet, recover, has_errors, strict;
18 static const char unpack_usage[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
19
20 /* We always read in 4kB chunks. */
21 static unsigned char buffer[4096];
22 static unsigned int offset, len;
23 static off_t consumed_bytes;
24 static off_t max_input_size;
25 static git_hash_ctx ctx;
26 static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
27
28 /*
29  * When running under --strict mode, objects whose reachability are
30  * suspect are kept in core without getting written in the object
31  * store.
32  */
33 struct obj_buffer {
34         char *buffer;
35         unsigned long size;
36 };
37
38 static struct decoration obj_decorate;
39
40 static struct obj_buffer *lookup_object_buffer(struct object *base)
41 {
42         return lookup_decoration(&obj_decorate, base);
43 }
44
45 static void add_object_buffer(struct object *object, char *buffer, unsigned long size)
46 {
47         struct obj_buffer *obj;
48         obj = xcalloc(1, sizeof(struct obj_buffer));
49         obj->buffer = buffer;
50         obj->size = size;
51         if (add_decoration(&obj_decorate, object, obj))
52                 die("object %s tried to add buffer twice!", oid_to_hex(&object->oid));
53 }
54
55 /*
56  * Make sure at least "min" bytes are available in the buffer, and
57  * return the pointer to the buffer.
58  */
59 static void *fill(int min)
60 {
61         if (min <= len)
62                 return buffer + offset;
63         if (min > sizeof(buffer))
64                 die("cannot fill %d bytes", min);
65         if (offset) {
66                 the_hash_algo->update_fn(&ctx, buffer, offset);
67                 memmove(buffer, buffer + offset, len);
68                 offset = 0;
69         }
70         do {
71                 ssize_t ret = xread(0, buffer + len, sizeof(buffer) - len);
72                 if (ret <= 0) {
73                         if (!ret)
74                                 die("early EOF");
75                         die_errno("read error on input");
76                 }
77                 len += ret;
78         } while (len < min);
79         return buffer;
80 }
81
82 static void use(int bytes)
83 {
84         if (bytes > len)
85                 die("used more bytes than were available");
86         len -= bytes;
87         offset += bytes;
88
89         /* make sure off_t is sufficiently large not to wrap */
90         if (signed_add_overflows(consumed_bytes, bytes))
91                 die("pack too large for current definition of off_t");
92         consumed_bytes += bytes;
93         if (max_input_size && consumed_bytes > max_input_size)
94                 die(_("pack exceeds maximum allowed size"));
95 }
96
97 static void *get_data(unsigned long size)
98 {
99         git_zstream stream;
100         void *buf = xmallocz(size);
101
102         memset(&stream, 0, sizeof(stream));
103
104         stream.next_out = buf;
105         stream.avail_out = size;
106         stream.next_in = fill(1);
107         stream.avail_in = len;
108         git_inflate_init(&stream);
109
110         for (;;) {
111                 int ret = git_inflate(&stream, 0);
112                 use(len - stream.avail_in);
113                 if (stream.total_out == size && ret == Z_STREAM_END)
114                         break;
115                 if (ret != Z_OK) {
116                         error("inflate returned %d", ret);
117                         FREE_AND_NULL(buf);
118                         if (!recover)
119                                 exit(1);
120                         has_errors = 1;
121                         break;
122                 }
123                 stream.next_in = fill(1);
124                 stream.avail_in = len;
125         }
126         git_inflate_end(&stream);
127         return buf;
128 }
129
130 struct delta_info {
131         struct object_id base_oid;
132         unsigned nr;
133         off_t base_offset;
134         unsigned long size;
135         void *delta;
136         struct delta_info *next;
137 };
138
139 static struct delta_info *delta_list;
140
141 static void add_delta_to_list(unsigned nr, const struct object_id *base_oid,
142                               off_t base_offset,
143                               void *delta, unsigned long size)
144 {
145         struct delta_info *info = xmalloc(sizeof(*info));
146
147         oidcpy(&info->base_oid, base_oid);
148         info->base_offset = base_offset;
149         info->size = size;
150         info->delta = delta;
151         info->nr = nr;
152         info->next = delta_list;
153         delta_list = info;
154 }
155
156 struct obj_info {
157         off_t offset;
158         struct object_id oid;
159         struct object *obj;
160 };
161
162 /* Remember to update object flag allocation in object.h */
163 #define FLAG_OPEN (1u<<20)
164 #define FLAG_WRITTEN (1u<<21)
165
166 static struct obj_info *obj_list;
167 static unsigned nr_objects;
168
169 /*
170  * Called only from check_object() after it verified this object
171  * is Ok.
172  */
173 static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)
174 {
175         struct object_id oid;
176
177         if (write_object_file(obj_buf->buffer, obj_buf->size,
178                               type_name(obj->type), &oid) < 0)
179                 die("failed to write object %s", oid_to_hex(&obj->oid));
180         obj->flags |= FLAG_WRITTEN;
181 }
182
183 /*
184  * At the very end of the processing, write_rest() scans the objects
185  * that have reachability requirements and calls this function.
186  * Verify its reachability and validity recursively and write it out.
187  */
188 static int check_object(struct object *obj, int type, void *data, struct fsck_options *options)
189 {
190         struct obj_buffer *obj_buf;
191
192         if (!obj)
193                 return 1;
194
195         if (obj->flags & FLAG_WRITTEN)
196                 return 0;
197
198         if (type != OBJ_ANY && obj->type != type)
199                 die("object type mismatch");
200
201         if (!(obj->flags & FLAG_OPEN)) {
202                 unsigned long size;
203                 int type = oid_object_info(the_repository, &obj->oid, &size);
204                 if (type != obj->type || type <= 0)
205                         die("object of unexpected type");
206                 obj->flags |= FLAG_WRITTEN;
207                 return 0;
208         }
209
210         obj_buf = lookup_object_buffer(obj);
211         if (!obj_buf)
212                 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
213         if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
214                 die("fsck error in packed object");
215         fsck_options.walk = check_object;
216         if (fsck_walk(obj, NULL, &fsck_options))
217                 die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
218         write_cached_object(obj, obj_buf);
219         return 0;
220 }
221
222 static void write_rest(void)
223 {
224         unsigned i;
225         for (i = 0; i < nr_objects; i++) {
226                 if (obj_list[i].obj)
227                         check_object(obj_list[i].obj, OBJ_ANY, NULL, NULL);
228         }
229 }
230
231 static void added_object(unsigned nr, enum object_type type,
232                          void *data, unsigned long size);
233
234 /*
235  * Write out nr-th object from the list, now we know the contents
236  * of it.  Under --strict, this buffers structured objects in-core,
237  * to be checked at the end.
238  */
239 static void write_object(unsigned nr, enum object_type type,
240                          void *buf, unsigned long size)
241 {
242         if (!strict) {
243                 if (write_object_file(buf, size, type_name(type),
244                                       &obj_list[nr].oid) < 0)
245                         die("failed to write object");
246                 added_object(nr, type, buf, size);
247                 free(buf);
248                 obj_list[nr].obj = NULL;
249         } else if (type == OBJ_BLOB) {
250                 struct blob *blob;
251                 if (write_object_file(buf, size, type_name(type),
252                                       &obj_list[nr].oid) < 0)
253                         die("failed to write object");
254                 added_object(nr, type, buf, size);
255                 free(buf);
256
257                 blob = lookup_blob(the_repository, &obj_list[nr].oid);
258                 if (blob)
259                         blob->object.flags |= FLAG_WRITTEN;
260                 else
261                         die("invalid blob object");
262                 obj_list[nr].obj = NULL;
263         } else {
264                 struct object *obj;
265                 int eaten;
266                 hash_object_file(buf, size, type_name(type), &obj_list[nr].oid);
267                 added_object(nr, type, buf, size);
268                 obj = parse_object_buffer(the_repository, &obj_list[nr].oid,
269                                           type, size, buf,
270                                           &eaten);
271                 if (!obj)
272                         die("invalid %s", type_name(type));
273                 add_object_buffer(obj, buf, size);
274                 obj->flags |= FLAG_OPEN;
275                 obj_list[nr].obj = obj;
276         }
277 }
278
279 static void resolve_delta(unsigned nr, enum object_type type,
280                           void *base, unsigned long base_size,
281                           void *delta, unsigned long delta_size)
282 {
283         void *result;
284         unsigned long result_size;
285
286         result = patch_delta(base, base_size,
287                              delta, delta_size,
288                              &result_size);
289         if (!result)
290                 die("failed to apply delta");
291         free(delta);
292         write_object(nr, type, result, result_size);
293 }
294
295 /*
296  * We now know the contents of an object (which is nr-th in the pack);
297  * resolve all the deltified objects that are based on it.
298  */
299 static void added_object(unsigned nr, enum object_type type,
300                          void *data, unsigned long size)
301 {
302         struct delta_info **p = &delta_list;
303         struct delta_info *info;
304
305         while ((info = *p) != NULL) {
306                 if (oideq(&info->base_oid, &obj_list[nr].oid) ||
307                     info->base_offset == obj_list[nr].offset) {
308                         *p = info->next;
309                         p = &delta_list;
310                         resolve_delta(info->nr, type, data, size,
311                                       info->delta, info->size);
312                         free(info);
313                         continue;
314                 }
315                 p = &info->next;
316         }
317 }
318
319 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
320                                    unsigned nr)
321 {
322         void *buf = get_data(size);
323
324         if (!dry_run && buf)
325                 write_object(nr, type, buf, size);
326         else
327                 free(buf);
328 }
329
330 static int resolve_against_held(unsigned nr, const struct object_id *base,
331                                 void *delta_data, unsigned long delta_size)
332 {
333         struct object *obj;
334         struct obj_buffer *obj_buffer;
335         obj = lookup_object(the_repository, base->hash);
336         if (!obj)
337                 return 0;
338         obj_buffer = lookup_object_buffer(obj);
339         if (!obj_buffer)
340                 return 0;
341         resolve_delta(nr, obj->type, obj_buffer->buffer,
342                       obj_buffer->size, delta_data, delta_size);
343         return 1;
344 }
345
346 static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
347                                unsigned nr)
348 {
349         void *delta_data, *base;
350         unsigned long base_size;
351         struct object_id base_oid;
352
353         if (type == OBJ_REF_DELTA) {
354                 hashcpy(base_oid.hash, fill(the_hash_algo->rawsz));
355                 use(the_hash_algo->rawsz);
356                 delta_data = get_data(delta_size);
357                 if (dry_run || !delta_data) {
358                         free(delta_data);
359                         return;
360                 }
361                 if (has_object_file(&base_oid))
362                         ; /* Ok we have this one */
363                 else if (resolve_against_held(nr, &base_oid,
364                                               delta_data, delta_size))
365                         return; /* we are done */
366                 else {
367                         /* cannot resolve yet --- queue it */
368                         oidclr(&obj_list[nr].oid);
369                         add_delta_to_list(nr, &base_oid, 0, delta_data, delta_size);
370                         return;
371                 }
372         } else {
373                 unsigned base_found = 0;
374                 unsigned char *pack, c;
375                 off_t base_offset;
376                 unsigned lo, mid, hi;
377
378                 pack = fill(1);
379                 c = *pack;
380                 use(1);
381                 base_offset = c & 127;
382                 while (c & 128) {
383                         base_offset += 1;
384                         if (!base_offset || MSB(base_offset, 7))
385                                 die("offset value overflow for delta base object");
386                         pack = fill(1);
387                         c = *pack;
388                         use(1);
389                         base_offset = (base_offset << 7) + (c & 127);
390                 }
391                 base_offset = obj_list[nr].offset - base_offset;
392                 if (base_offset <= 0 || base_offset >= obj_list[nr].offset)
393                         die("offset value out of bound for delta base object");
394
395                 delta_data = get_data(delta_size);
396                 if (dry_run || !delta_data) {
397                         free(delta_data);
398                         return;
399                 }
400                 lo = 0;
401                 hi = nr;
402                 while (lo < hi) {
403                         mid = lo + (hi - lo) / 2;
404                         if (base_offset < obj_list[mid].offset) {
405                                 hi = mid;
406                         } else if (base_offset > obj_list[mid].offset) {
407                                 lo = mid + 1;
408                         } else {
409                                 oidcpy(&base_oid, &obj_list[mid].oid);
410                                 base_found = !is_null_oid(&base_oid);
411                                 break;
412                         }
413                 }
414                 if (!base_found) {
415                         /*
416                          * The delta base object is itself a delta that
417                          * has not been resolved yet.
418                          */
419                         oidclr(&obj_list[nr].oid);
420                         add_delta_to_list(nr, &null_oid, base_offset, delta_data, delta_size);
421                         return;
422                 }
423         }
424
425         if (resolve_against_held(nr, &base_oid, delta_data, delta_size))
426                 return;
427
428         base = read_object_file(&base_oid, &type, &base_size);
429         if (!base) {
430                 error("failed to read delta-pack base object %s",
431                       oid_to_hex(&base_oid));
432                 if (!recover)
433                         exit(1);
434                 has_errors = 1;
435                 return;
436         }
437         resolve_delta(nr, type, base, base_size, delta_data, delta_size);
438         free(base);
439 }
440
441 static void unpack_one(unsigned nr)
442 {
443         unsigned shift;
444         unsigned char *pack;
445         unsigned long size, c;
446         enum object_type type;
447
448         obj_list[nr].offset = consumed_bytes;
449
450         pack = fill(1);
451         c = *pack;
452         use(1);
453         type = (c >> 4) & 7;
454         size = (c & 15);
455         shift = 4;
456         while (c & 0x80) {
457                 pack = fill(1);
458                 c = *pack;
459                 use(1);
460                 size += (c & 0x7f) << shift;
461                 shift += 7;
462         }
463
464         switch (type) {
465         case OBJ_COMMIT:
466         case OBJ_TREE:
467         case OBJ_BLOB:
468         case OBJ_TAG:
469                 unpack_non_delta_entry(type, size, nr);
470                 return;
471         case OBJ_REF_DELTA:
472         case OBJ_OFS_DELTA:
473                 unpack_delta_entry(type, size, nr);
474                 return;
475         default:
476                 error("bad object type %d", type);
477                 has_errors = 1;
478                 if (recover)
479                         return;
480                 exit(1);
481         }
482 }
483
484 static void unpack_all(void)
485 {
486         int i;
487         struct progress *progress = NULL;
488         struct pack_header *hdr = fill(sizeof(struct pack_header));
489
490         nr_objects = ntohl(hdr->hdr_entries);
491
492         if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
493                 die("bad pack file");
494         if (!pack_version_ok(hdr->hdr_version))
495                 die("unknown pack file version %"PRIu32,
496                         ntohl(hdr->hdr_version));
497         use(sizeof(struct pack_header));
498
499         if (!quiet)
500                 progress = start_progress(_("Unpacking objects"), nr_objects);
501         obj_list = xcalloc(nr_objects, sizeof(*obj_list));
502         for (i = 0; i < nr_objects; i++) {
503                 unpack_one(i);
504                 display_progress(progress, i + 1);
505         }
506         stop_progress(&progress);
507
508         if (delta_list)
509                 die("unresolved deltas left after unpacking");
510 }
511
512 int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
513 {
514         int i;
515         struct object_id oid;
516
517         read_replace_refs = 0;
518
519         git_config(git_default_config, NULL);
520
521         quiet = !isatty(2);
522
523         for (i = 1 ; i < argc; i++) {
524                 const char *arg = argv[i];
525
526                 if (*arg == '-') {
527                         if (!strcmp(arg, "-n")) {
528                                 dry_run = 1;
529                                 continue;
530                         }
531                         if (!strcmp(arg, "-q")) {
532                                 quiet = 1;
533                                 continue;
534                         }
535                         if (!strcmp(arg, "-r")) {
536                                 recover = 1;
537                                 continue;
538                         }
539                         if (!strcmp(arg, "--strict")) {
540                                 strict = 1;
541                                 continue;
542                         }
543                         if (skip_prefix(arg, "--strict=", &arg)) {
544                                 strict = 1;
545                                 fsck_set_msg_types(&fsck_options, arg);
546                                 continue;
547                         }
548                         if (starts_with(arg, "--pack_header=")) {
549                                 struct pack_header *hdr;
550                                 char *c;
551
552                                 hdr = (struct pack_header *)buffer;
553                                 hdr->hdr_signature = htonl(PACK_SIGNATURE);
554                                 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
555                                 if (*c != ',')
556                                         die("bad %s", arg);
557                                 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
558                                 if (*c)
559                                         die("bad %s", arg);
560                                 len = sizeof(*hdr);
561                                 continue;
562                         }
563                         if (skip_prefix(arg, "--max-input-size=", &arg)) {
564                                 max_input_size = strtoumax(arg, NULL, 10);
565                                 continue;
566                         }
567                         usage(unpack_usage);
568                 }
569
570                 /* We don't take any non-flag arguments now.. Maybe some day */
571                 usage(unpack_usage);
572         }
573         the_hash_algo->init_fn(&ctx);
574         unpack_all();
575         the_hash_algo->update_fn(&ctx, buffer, offset);
576         the_hash_algo->final_fn(oid.hash, &ctx);
577         if (strict) {
578                 write_rest();
579                 if (fsck_finish(&fsck_options))
580                         die(_("fsck error in pack objects"));
581         }
582         if (!hasheq(fill(the_hash_algo->rawsz), oid.hash))
583                 die("final sha1 did not match");
584         use(the_hash_algo->rawsz);
585
586         /* Write the last part of the buffer to stdout */
587         while (len) {
588                 int ret = xwrite(1, buffer + offset, len);
589                 if (ret <= 0)
590                         break;
591                 len -= ret;
592                 offset += ret;
593         }
594
595         /* All done */
596         return has_errors;
597 }