Merge branch 'jt/lazy-fetch'
[git] / builtin / unpack-objects.c
1 #include "builtin.h"
2 #include "cache.h"
3 #include "config.h"
4 #include "object-store.h"
5 #include "object.h"
6 #include "delta.h"
7 #include "pack.h"
8 #include "blob.h"
9 #include "commit.h"
10 #include "tag.h"
11 #include "tree.h"
12 #include "tree-walk.h"
13 #include "progress.h"
14 #include "decorate.h"
15 #include "fsck.h"
16
17 static int dry_run, quiet, recover, has_errors, strict;
18 static const char unpack_usage[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
19
20 /* We always read in 4kB chunks. */
21 static unsigned char buffer[4096];
22 static unsigned int offset, len;
23 static off_t consumed_bytes;
24 static off_t max_input_size;
25 static git_hash_ctx ctx;
26 static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
27 static struct progress *progress;
28
29 /*
30  * When running under --strict mode, objects whose reachability are
31  * suspect are kept in core without getting written in the object
32  * store.
33  */
34 struct obj_buffer {
35         char *buffer;
36         unsigned long size;
37 };
38
39 static struct decoration obj_decorate;
40
41 static struct obj_buffer *lookup_object_buffer(struct object *base)
42 {
43         return lookup_decoration(&obj_decorate, base);
44 }
45
46 static void add_object_buffer(struct object *object, char *buffer, unsigned long size)
47 {
48         struct obj_buffer *obj;
49         obj = xcalloc(1, sizeof(struct obj_buffer));
50         obj->buffer = buffer;
51         obj->size = size;
52         if (add_decoration(&obj_decorate, object, obj))
53                 die("object %s tried to add buffer twice!", oid_to_hex(&object->oid));
54 }
55
56 /*
57  * Make sure at least "min" bytes are available in the buffer, and
58  * return the pointer to the buffer.
59  */
60 static void *fill(int min)
61 {
62         if (min <= len)
63                 return buffer + offset;
64         if (min > sizeof(buffer))
65                 die("cannot fill %d bytes", min);
66         if (offset) {
67                 the_hash_algo->update_fn(&ctx, buffer, offset);
68                 memmove(buffer, buffer + offset, len);
69                 offset = 0;
70         }
71         do {
72                 ssize_t ret = xread(0, buffer + len, sizeof(buffer) - len);
73                 if (ret <= 0) {
74                         if (!ret)
75                                 die("early EOF");
76                         die_errno("read error on input");
77                 }
78                 len += ret;
79         } while (len < min);
80         return buffer;
81 }
82
83 static void use(int bytes)
84 {
85         if (bytes > len)
86                 die("used more bytes than were available");
87         len -= bytes;
88         offset += bytes;
89
90         /* make sure off_t is sufficiently large not to wrap */
91         if (signed_add_overflows(consumed_bytes, bytes))
92                 die("pack too large for current definition of off_t");
93         consumed_bytes += bytes;
94         if (max_input_size && consumed_bytes > max_input_size)
95                 die(_("pack exceeds maximum allowed size"));
96         display_throughput(progress, consumed_bytes);
97 }
98
99 static void *get_data(unsigned long size)
100 {
101         git_zstream stream;
102         void *buf = xmallocz(size);
103
104         memset(&stream, 0, sizeof(stream));
105
106         stream.next_out = buf;
107         stream.avail_out = size;
108         stream.next_in = fill(1);
109         stream.avail_in = len;
110         git_inflate_init(&stream);
111
112         for (;;) {
113                 int ret = git_inflate(&stream, 0);
114                 use(len - stream.avail_in);
115                 if (stream.total_out == size && ret == Z_STREAM_END)
116                         break;
117                 if (ret != Z_OK) {
118                         error("inflate returned %d", ret);
119                         FREE_AND_NULL(buf);
120                         if (!recover)
121                                 exit(1);
122                         has_errors = 1;
123                         break;
124                 }
125                 stream.next_in = fill(1);
126                 stream.avail_in = len;
127         }
128         git_inflate_end(&stream);
129         return buf;
130 }
131
132 struct delta_info {
133         struct object_id base_oid;
134         unsigned nr;
135         off_t base_offset;
136         unsigned long size;
137         void *delta;
138         struct delta_info *next;
139 };
140
141 static struct delta_info *delta_list;
142
143 static void add_delta_to_list(unsigned nr, const struct object_id *base_oid,
144                               off_t base_offset,
145                               void *delta, unsigned long size)
146 {
147         struct delta_info *info = xmalloc(sizeof(*info));
148
149         oidcpy(&info->base_oid, base_oid);
150         info->base_offset = base_offset;
151         info->size = size;
152         info->delta = delta;
153         info->nr = nr;
154         info->next = delta_list;
155         delta_list = info;
156 }
157
158 struct obj_info {
159         off_t offset;
160         struct object_id oid;
161         struct object *obj;
162 };
163
164 /* Remember to update object flag allocation in object.h */
165 #define FLAG_OPEN (1u<<20)
166 #define FLAG_WRITTEN (1u<<21)
167
168 static struct obj_info *obj_list;
169 static unsigned nr_objects;
170
171 /*
172  * Called only from check_object() after it verified this object
173  * is Ok.
174  */
175 static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)
176 {
177         struct object_id oid;
178
179         if (write_object_file(obj_buf->buffer, obj_buf->size,
180                               type_name(obj->type), &oid) < 0)
181                 die("failed to write object %s", oid_to_hex(&obj->oid));
182         obj->flags |= FLAG_WRITTEN;
183 }
184
185 /*
186  * At the very end of the processing, write_rest() scans the objects
187  * that have reachability requirements and calls this function.
188  * Verify its reachability and validity recursively and write it out.
189  */
190 static int check_object(struct object *obj, int type, void *data, struct fsck_options *options)
191 {
192         struct obj_buffer *obj_buf;
193
194         if (!obj)
195                 return 1;
196
197         if (obj->flags & FLAG_WRITTEN)
198                 return 0;
199
200         if (type != OBJ_ANY && obj->type != type)
201                 die("object type mismatch");
202
203         if (!(obj->flags & FLAG_OPEN)) {
204                 unsigned long size;
205                 int type = oid_object_info(the_repository, &obj->oid, &size);
206                 if (type != obj->type || type <= 0)
207                         die("object of unexpected type");
208                 obj->flags |= FLAG_WRITTEN;
209                 return 0;
210         }
211
212         obj_buf = lookup_object_buffer(obj);
213         if (!obj_buf)
214                 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
215         if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
216                 die("fsck error in packed object");
217         fsck_options.walk = check_object;
218         if (fsck_walk(obj, NULL, &fsck_options))
219                 die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
220         write_cached_object(obj, obj_buf);
221         return 0;
222 }
223
224 static void write_rest(void)
225 {
226         unsigned i;
227         for (i = 0; i < nr_objects; i++) {
228                 if (obj_list[i].obj)
229                         check_object(obj_list[i].obj, OBJ_ANY, NULL, NULL);
230         }
231 }
232
233 static void added_object(unsigned nr, enum object_type type,
234                          void *data, unsigned long size);
235
236 /*
237  * Write out nr-th object from the list, now we know the contents
238  * of it.  Under --strict, this buffers structured objects in-core,
239  * to be checked at the end.
240  */
241 static void write_object(unsigned nr, enum object_type type,
242                          void *buf, unsigned long size)
243 {
244         if (!strict) {
245                 if (write_object_file(buf, size, type_name(type),
246                                       &obj_list[nr].oid) < 0)
247                         die("failed to write object");
248                 added_object(nr, type, buf, size);
249                 free(buf);
250                 obj_list[nr].obj = NULL;
251         } else if (type == OBJ_BLOB) {
252                 struct blob *blob;
253                 if (write_object_file(buf, size, type_name(type),
254                                       &obj_list[nr].oid) < 0)
255                         die("failed to write object");
256                 added_object(nr, type, buf, size);
257                 free(buf);
258
259                 blob = lookup_blob(the_repository, &obj_list[nr].oid);
260                 if (blob)
261                         blob->object.flags |= FLAG_WRITTEN;
262                 else
263                         die("invalid blob object");
264                 obj_list[nr].obj = NULL;
265         } else {
266                 struct object *obj;
267                 int eaten;
268                 hash_object_file(the_hash_algo, buf, size, type_name(type),
269                                  &obj_list[nr].oid);
270                 added_object(nr, type, buf, size);
271                 obj = parse_object_buffer(the_repository, &obj_list[nr].oid,
272                                           type, size, buf,
273                                           &eaten);
274                 if (!obj)
275                         die("invalid %s", type_name(type));
276                 add_object_buffer(obj, buf, size);
277                 obj->flags |= FLAG_OPEN;
278                 obj_list[nr].obj = obj;
279         }
280 }
281
282 static void resolve_delta(unsigned nr, enum object_type type,
283                           void *base, unsigned long base_size,
284                           void *delta, unsigned long delta_size)
285 {
286         void *result;
287         unsigned long result_size;
288
289         result = patch_delta(base, base_size,
290                              delta, delta_size,
291                              &result_size);
292         if (!result)
293                 die("failed to apply delta");
294         free(delta);
295         write_object(nr, type, result, result_size);
296 }
297
298 /*
299  * We now know the contents of an object (which is nr-th in the pack);
300  * resolve all the deltified objects that are based on it.
301  */
302 static void added_object(unsigned nr, enum object_type type,
303                          void *data, unsigned long size)
304 {
305         struct delta_info **p = &delta_list;
306         struct delta_info *info;
307
308         while ((info = *p) != NULL) {
309                 if (oideq(&info->base_oid, &obj_list[nr].oid) ||
310                     info->base_offset == obj_list[nr].offset) {
311                         *p = info->next;
312                         p = &delta_list;
313                         resolve_delta(info->nr, type, data, size,
314                                       info->delta, info->size);
315                         free(info);
316                         continue;
317                 }
318                 p = &info->next;
319         }
320 }
321
322 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
323                                    unsigned nr)
324 {
325         void *buf = get_data(size);
326
327         if (!dry_run && buf)
328                 write_object(nr, type, buf, size);
329         else
330                 free(buf);
331 }
332
333 static int resolve_against_held(unsigned nr, const struct object_id *base,
334                                 void *delta_data, unsigned long delta_size)
335 {
336         struct object *obj;
337         struct obj_buffer *obj_buffer;
338         obj = lookup_object(the_repository, base);
339         if (!obj)
340                 return 0;
341         obj_buffer = lookup_object_buffer(obj);
342         if (!obj_buffer)
343                 return 0;
344         resolve_delta(nr, obj->type, obj_buffer->buffer,
345                       obj_buffer->size, delta_data, delta_size);
346         return 1;
347 }
348
349 static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
350                                unsigned nr)
351 {
352         void *delta_data, *base;
353         unsigned long base_size;
354         struct object_id base_oid;
355
356         if (type == OBJ_REF_DELTA) {
357                 hashcpy(base_oid.hash, fill(the_hash_algo->rawsz));
358                 use(the_hash_algo->rawsz);
359                 delta_data = get_data(delta_size);
360                 if (dry_run || !delta_data) {
361                         free(delta_data);
362                         return;
363                 }
364                 if (has_object_file(&base_oid))
365                         ; /* Ok we have this one */
366                 else if (resolve_against_held(nr, &base_oid,
367                                               delta_data, delta_size))
368                         return; /* we are done */
369                 else {
370                         /* cannot resolve yet --- queue it */
371                         oidclr(&obj_list[nr].oid);
372                         add_delta_to_list(nr, &base_oid, 0, delta_data, delta_size);
373                         return;
374                 }
375         } else {
376                 unsigned base_found = 0;
377                 unsigned char *pack, c;
378                 off_t base_offset;
379                 unsigned lo, mid, hi;
380
381                 pack = fill(1);
382                 c = *pack;
383                 use(1);
384                 base_offset = c & 127;
385                 while (c & 128) {
386                         base_offset += 1;
387                         if (!base_offset || MSB(base_offset, 7))
388                                 die("offset value overflow for delta base object");
389                         pack = fill(1);
390                         c = *pack;
391                         use(1);
392                         base_offset = (base_offset << 7) + (c & 127);
393                 }
394                 base_offset = obj_list[nr].offset - base_offset;
395                 if (base_offset <= 0 || base_offset >= obj_list[nr].offset)
396                         die("offset value out of bound for delta base object");
397
398                 delta_data = get_data(delta_size);
399                 if (dry_run || !delta_data) {
400                         free(delta_data);
401                         return;
402                 }
403                 lo = 0;
404                 hi = nr;
405                 while (lo < hi) {
406                         mid = lo + (hi - lo) / 2;
407                         if (base_offset < obj_list[mid].offset) {
408                                 hi = mid;
409                         } else if (base_offset > obj_list[mid].offset) {
410                                 lo = mid + 1;
411                         } else {
412                                 oidcpy(&base_oid, &obj_list[mid].oid);
413                                 base_found = !is_null_oid(&base_oid);
414                                 break;
415                         }
416                 }
417                 if (!base_found) {
418                         /*
419                          * The delta base object is itself a delta that
420                          * has not been resolved yet.
421                          */
422                         oidclr(&obj_list[nr].oid);
423                         add_delta_to_list(nr, &null_oid, base_offset, delta_data, delta_size);
424                         return;
425                 }
426         }
427
428         if (resolve_against_held(nr, &base_oid, delta_data, delta_size))
429                 return;
430
431         base = read_object_file(&base_oid, &type, &base_size);
432         if (!base) {
433                 error("failed to read delta-pack base object %s",
434                       oid_to_hex(&base_oid));
435                 if (!recover)
436                         exit(1);
437                 has_errors = 1;
438                 return;
439         }
440         resolve_delta(nr, type, base, base_size, delta_data, delta_size);
441         free(base);
442 }
443
444 static void unpack_one(unsigned nr)
445 {
446         unsigned shift;
447         unsigned char *pack;
448         unsigned long size, c;
449         enum object_type type;
450
451         obj_list[nr].offset = consumed_bytes;
452
453         pack = fill(1);
454         c = *pack;
455         use(1);
456         type = (c >> 4) & 7;
457         size = (c & 15);
458         shift = 4;
459         while (c & 0x80) {
460                 pack = fill(1);
461                 c = *pack;
462                 use(1);
463                 size += (c & 0x7f) << shift;
464                 shift += 7;
465         }
466
467         switch (type) {
468         case OBJ_COMMIT:
469         case OBJ_TREE:
470         case OBJ_BLOB:
471         case OBJ_TAG:
472                 unpack_non_delta_entry(type, size, nr);
473                 return;
474         case OBJ_REF_DELTA:
475         case OBJ_OFS_DELTA:
476                 unpack_delta_entry(type, size, nr);
477                 return;
478         default:
479                 error("bad object type %d", type);
480                 has_errors = 1;
481                 if (recover)
482                         return;
483                 exit(1);
484         }
485 }
486
487 static void unpack_all(void)
488 {
489         int i;
490         struct pack_header *hdr = fill(sizeof(struct pack_header));
491
492         nr_objects = ntohl(hdr->hdr_entries);
493
494         if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
495                 die("bad pack file");
496         if (!pack_version_ok(hdr->hdr_version))
497                 die("unknown pack file version %"PRIu32,
498                         ntohl(hdr->hdr_version));
499         use(sizeof(struct pack_header));
500
501         if (!quiet)
502                 progress = start_progress(_("Unpacking objects"), nr_objects);
503         obj_list = xcalloc(nr_objects, sizeof(*obj_list));
504         for (i = 0; i < nr_objects; i++) {
505                 unpack_one(i);
506                 display_progress(progress, i + 1);
507         }
508         stop_progress(&progress);
509
510         if (delta_list)
511                 die("unresolved deltas left after unpacking");
512 }
513
514 int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
515 {
516         int i;
517         struct object_id oid;
518
519         read_replace_refs = 0;
520
521         git_config(git_default_config, NULL);
522
523         quiet = !isatty(2);
524
525         for (i = 1 ; i < argc; i++) {
526                 const char *arg = argv[i];
527
528                 if (*arg == '-') {
529                         if (!strcmp(arg, "-n")) {
530                                 dry_run = 1;
531                                 continue;
532                         }
533                         if (!strcmp(arg, "-q")) {
534                                 quiet = 1;
535                                 continue;
536                         }
537                         if (!strcmp(arg, "-r")) {
538                                 recover = 1;
539                                 continue;
540                         }
541                         if (!strcmp(arg, "--strict")) {
542                                 strict = 1;
543                                 continue;
544                         }
545                         if (skip_prefix(arg, "--strict=", &arg)) {
546                                 strict = 1;
547                                 fsck_set_msg_types(&fsck_options, arg);
548                                 continue;
549                         }
550                         if (starts_with(arg, "--pack_header=")) {
551                                 struct pack_header *hdr;
552                                 char *c;
553
554                                 hdr = (struct pack_header *)buffer;
555                                 hdr->hdr_signature = htonl(PACK_SIGNATURE);
556                                 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
557                                 if (*c != ',')
558                                         die("bad %s", arg);
559                                 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
560                                 if (*c)
561                                         die("bad %s", arg);
562                                 len = sizeof(*hdr);
563                                 continue;
564                         }
565                         if (skip_prefix(arg, "--max-input-size=", &arg)) {
566                                 max_input_size = strtoumax(arg, NULL, 10);
567                                 continue;
568                         }
569                         usage(unpack_usage);
570                 }
571
572                 /* We don't take any non-flag arguments now.. Maybe some day */
573                 usage(unpack_usage);
574         }
575         the_hash_algo->init_fn(&ctx);
576         unpack_all();
577         the_hash_algo->update_fn(&ctx, buffer, offset);
578         the_hash_algo->final_fn(oid.hash, &ctx);
579         if (strict) {
580                 write_rest();
581                 if (fsck_finish(&fsck_options))
582                         die(_("fsck error in pack objects"));
583         }
584         if (!hasheq(fill(the_hash_algo->rawsz), oid.hash))
585                 die("final sha1 did not match");
586         use(the_hash_algo->rawsz);
587
588         /* Write the last part of the buffer to stdout */
589         while (len) {
590                 int ret = xwrite(1, buffer + offset, len);
591                 if (ret <= 0)
592                         break;
593                 len -= ret;
594                 offset += ret;
595         }
596
597         /* All done */
598         return has_errors;
599 }