4 #include "object-store.h"
12 #include "tree-walk.h"
17 static int dry_run, quiet, recover, has_errors, strict;
18 static const char unpack_usage[] = "git unpack-objects [-n] [-q] [-r] [--strict]";
20 /* We always read in 4kB chunks. */
21 static unsigned char buffer[4096];
22 static unsigned int offset, len;
23 static off_t consumed_bytes;
24 static off_t max_input_size;
25 static git_hash_ctx ctx;
26 static struct fsck_options fsck_options = FSCK_OPTIONS_STRICT;
29 * When running under --strict mode, objects whose reachability are
30 * suspect are kept in core without getting written in the object
38 static struct decoration obj_decorate;
40 static struct obj_buffer *lookup_object_buffer(struct object *base)
42 return lookup_decoration(&obj_decorate, base);
45 static void add_object_buffer(struct object *object, char *buffer, unsigned long size)
47 struct obj_buffer *obj;
48 obj = xcalloc(1, sizeof(struct obj_buffer));
51 if (add_decoration(&obj_decorate, object, obj))
52 die("object %s tried to add buffer twice!", oid_to_hex(&object->oid));
56 * Make sure at least "min" bytes are available in the buffer, and
57 * return the pointer to the buffer.
59 static void *fill(int min)
62 return buffer + offset;
63 if (min > sizeof(buffer))
64 die("cannot fill %d bytes", min);
66 the_hash_algo->update_fn(&ctx, buffer, offset);
67 memmove(buffer, buffer + offset, len);
71 ssize_t ret = xread(0, buffer + len, sizeof(buffer) - len);
75 die_errno("read error on input");
82 static void use(int bytes)
85 die("used more bytes than were available");
89 /* make sure off_t is sufficiently large not to wrap */
90 if (signed_add_overflows(consumed_bytes, bytes))
91 die("pack too large for current definition of off_t");
92 consumed_bytes += bytes;
93 if (max_input_size && consumed_bytes > max_input_size)
94 die(_("pack exceeds maximum allowed size"));
97 static void *get_data(unsigned long size)
100 void *buf = xmallocz(size);
102 memset(&stream, 0, sizeof(stream));
104 stream.next_out = buf;
105 stream.avail_out = size;
106 stream.next_in = fill(1);
107 stream.avail_in = len;
108 git_inflate_init(&stream);
111 int ret = git_inflate(&stream, 0);
112 use(len - stream.avail_in);
113 if (stream.total_out == size && ret == Z_STREAM_END)
116 error("inflate returned %d", ret);
123 stream.next_in = fill(1);
124 stream.avail_in = len;
126 git_inflate_end(&stream);
131 struct object_id base_oid;
136 struct delta_info *next;
139 static struct delta_info *delta_list;
141 static void add_delta_to_list(unsigned nr, const struct object_id *base_oid,
143 void *delta, unsigned long size)
145 struct delta_info *info = xmalloc(sizeof(*info));
147 oidcpy(&info->base_oid, base_oid);
148 info->base_offset = base_offset;
152 info->next = delta_list;
158 struct object_id oid;
162 /* Remember to update object flag allocation in object.h */
163 #define FLAG_OPEN (1u<<20)
164 #define FLAG_WRITTEN (1u<<21)
166 static struct obj_info *obj_list;
167 static unsigned nr_objects;
170 * Called only from check_object() after it verified this object
173 static void write_cached_object(struct object *obj, struct obj_buffer *obj_buf)
175 struct object_id oid;
177 if (write_object_file(obj_buf->buffer, obj_buf->size,
178 type_name(obj->type), &oid) < 0)
179 die("failed to write object %s", oid_to_hex(&obj->oid));
180 obj->flags |= FLAG_WRITTEN;
184 * At the very end of the processing, write_rest() scans the objects
185 * that have reachability requirements and calls this function.
186 * Verify its reachability and validity recursively and write it out.
188 static int check_object(struct object *obj, int type, void *data, struct fsck_options *options)
190 struct obj_buffer *obj_buf;
195 if (obj->flags & FLAG_WRITTEN)
198 if (type != OBJ_ANY && obj->type != type)
199 die("object type mismatch");
201 if (!(obj->flags & FLAG_OPEN)) {
203 int type = oid_object_info(the_repository, &obj->oid, &size);
204 if (type != obj->type || type <= 0)
205 die("object of unexpected type");
206 obj->flags |= FLAG_WRITTEN;
210 obj_buf = lookup_object_buffer(obj);
212 die("Whoops! Cannot find object '%s'", oid_to_hex(&obj->oid));
213 if (fsck_object(obj, obj_buf->buffer, obj_buf->size, &fsck_options))
214 die("fsck error in packed object");
215 fsck_options.walk = check_object;
216 if (fsck_walk(obj, NULL, &fsck_options))
217 die("Error on reachable objects of %s", oid_to_hex(&obj->oid));
218 write_cached_object(obj, obj_buf);
222 static void write_rest(void)
225 for (i = 0; i < nr_objects; i++) {
227 check_object(obj_list[i].obj, OBJ_ANY, NULL, NULL);
231 static void added_object(unsigned nr, enum object_type type,
232 void *data, unsigned long size);
235 * Write out nr-th object from the list, now we know the contents
236 * of it. Under --strict, this buffers structured objects in-core,
237 * to be checked at the end.
239 static void write_object(unsigned nr, enum object_type type,
240 void *buf, unsigned long size)
243 if (write_object_file(buf, size, type_name(type),
244 &obj_list[nr].oid) < 0)
245 die("failed to write object");
246 added_object(nr, type, buf, size);
248 obj_list[nr].obj = NULL;
249 } else if (type == OBJ_BLOB) {
251 if (write_object_file(buf, size, type_name(type),
252 &obj_list[nr].oid) < 0)
253 die("failed to write object");
254 added_object(nr, type, buf, size);
257 blob = lookup_blob(&obj_list[nr].oid);
259 blob->object.flags |= FLAG_WRITTEN;
261 die("invalid blob object");
262 obj_list[nr].obj = NULL;
266 hash_object_file(buf, size, type_name(type), &obj_list[nr].oid);
267 added_object(nr, type, buf, size);
268 obj = parse_object_buffer(&obj_list[nr].oid, type, size, buf,
271 die("invalid %s", type_name(type));
272 add_object_buffer(obj, buf, size);
273 obj->flags |= FLAG_OPEN;
274 obj_list[nr].obj = obj;
278 static void resolve_delta(unsigned nr, enum object_type type,
279 void *base, unsigned long base_size,
280 void *delta, unsigned long delta_size)
283 unsigned long result_size;
285 result = patch_delta(base, base_size,
289 die("failed to apply delta");
291 write_object(nr, type, result, result_size);
295 * We now know the contents of an object (which is nr-th in the pack);
296 * resolve all the deltified objects that are based on it.
298 static void added_object(unsigned nr, enum object_type type,
299 void *data, unsigned long size)
301 struct delta_info **p = &delta_list;
302 struct delta_info *info;
304 while ((info = *p) != NULL) {
305 if (!oidcmp(&info->base_oid, &obj_list[nr].oid) ||
306 info->base_offset == obj_list[nr].offset) {
309 resolve_delta(info->nr, type, data, size,
310 info->delta, info->size);
318 static void unpack_non_delta_entry(enum object_type type, unsigned long size,
321 void *buf = get_data(size);
324 write_object(nr, type, buf, size);
329 static int resolve_against_held(unsigned nr, const struct object_id *base,
330 void *delta_data, unsigned long delta_size)
333 struct obj_buffer *obj_buffer;
334 obj = lookup_object(base->hash);
337 obj_buffer = lookup_object_buffer(obj);
340 resolve_delta(nr, obj->type, obj_buffer->buffer,
341 obj_buffer->size, delta_data, delta_size);
345 static void unpack_delta_entry(enum object_type type, unsigned long delta_size,
348 void *delta_data, *base;
349 unsigned long base_size;
350 struct object_id base_oid;
352 if (type == OBJ_REF_DELTA) {
353 hashcpy(base_oid.hash, fill(the_hash_algo->rawsz));
354 use(the_hash_algo->rawsz);
355 delta_data = get_data(delta_size);
356 if (dry_run || !delta_data) {
360 if (has_object_file(&base_oid))
361 ; /* Ok we have this one */
362 else if (resolve_against_held(nr, &base_oid,
363 delta_data, delta_size))
364 return; /* we are done */
366 /* cannot resolve yet --- queue it */
367 oidclr(&obj_list[nr].oid);
368 add_delta_to_list(nr, &base_oid, 0, delta_data, delta_size);
372 unsigned base_found = 0;
373 unsigned char *pack, c;
375 unsigned lo, mid, hi;
380 base_offset = c & 127;
383 if (!base_offset || MSB(base_offset, 7))
384 die("offset value overflow for delta base object");
388 base_offset = (base_offset << 7) + (c & 127);
390 base_offset = obj_list[nr].offset - base_offset;
391 if (base_offset <= 0 || base_offset >= obj_list[nr].offset)
392 die("offset value out of bound for delta base object");
394 delta_data = get_data(delta_size);
395 if (dry_run || !delta_data) {
402 mid = lo + (hi - lo) / 2;
403 if (base_offset < obj_list[mid].offset) {
405 } else if (base_offset > obj_list[mid].offset) {
408 oidcpy(&base_oid, &obj_list[mid].oid);
409 base_found = !is_null_oid(&base_oid);
415 * The delta base object is itself a delta that
416 * has not been resolved yet.
418 oidclr(&obj_list[nr].oid);
419 add_delta_to_list(nr, &null_oid, base_offset, delta_data, delta_size);
424 if (resolve_against_held(nr, &base_oid, delta_data, delta_size))
427 base = read_object_file(&base_oid, &type, &base_size);
429 error("failed to read delta-pack base object %s",
430 oid_to_hex(&base_oid));
436 resolve_delta(nr, type, base, base_size, delta_data, delta_size);
440 static void unpack_one(unsigned nr)
444 unsigned long size, c;
445 enum object_type type;
447 obj_list[nr].offset = consumed_bytes;
459 size += (c & 0x7f) << shift;
468 unpack_non_delta_entry(type, size, nr);
472 unpack_delta_entry(type, size, nr);
475 error("bad object type %d", type);
483 static void unpack_all(void)
486 struct progress *progress = NULL;
487 struct pack_header *hdr = fill(sizeof(struct pack_header));
489 nr_objects = ntohl(hdr->hdr_entries);
491 if (ntohl(hdr->hdr_signature) != PACK_SIGNATURE)
492 die("bad pack file");
493 if (!pack_version_ok(hdr->hdr_version))
494 die("unknown pack file version %"PRIu32,
495 ntohl(hdr->hdr_version));
496 use(sizeof(struct pack_header));
499 progress = start_progress(_("Unpacking objects"), nr_objects);
500 obj_list = xcalloc(nr_objects, sizeof(*obj_list));
501 for (i = 0; i < nr_objects; i++) {
503 display_progress(progress, i + 1);
505 stop_progress(&progress);
508 die("unresolved deltas left after unpacking");
511 int cmd_unpack_objects(int argc, const char **argv, const char *prefix)
514 struct object_id oid;
516 check_replace_refs = 0;
518 git_config(git_default_config, NULL);
522 for (i = 1 ; i < argc; i++) {
523 const char *arg = argv[i];
526 if (!strcmp(arg, "-n")) {
530 if (!strcmp(arg, "-q")) {
534 if (!strcmp(arg, "-r")) {
538 if (!strcmp(arg, "--strict")) {
542 if (skip_prefix(arg, "--strict=", &arg)) {
544 fsck_set_msg_types(&fsck_options, arg);
547 if (starts_with(arg, "--pack_header=")) {
548 struct pack_header *hdr;
551 hdr = (struct pack_header *)buffer;
552 hdr->hdr_signature = htonl(PACK_SIGNATURE);
553 hdr->hdr_version = htonl(strtoul(arg + 14, &c, 10));
556 hdr->hdr_entries = htonl(strtoul(c + 1, &c, 10));
562 if (skip_prefix(arg, "--max-input-size=", &arg)) {
563 max_input_size = strtoumax(arg, NULL, 10);
569 /* We don't take any non-flag arguments now.. Maybe some day */
572 the_hash_algo->init_fn(&ctx);
574 the_hash_algo->update_fn(&ctx, buffer, offset);
575 the_hash_algo->final_fn(oid.hash, &ctx);
578 if (fsck_finish(&fsck_options))
579 die(_("fsck error in pack objects"));
581 if (hashcmp(fill(the_hash_algo->rawsz), oid.hash))
582 die("final sha1 did not match");
583 use(the_hash_algo->rawsz);
585 /* Write the last part of the buffer to stdout */
587 int ret = xwrite(1, buffer + offset, len);