10 #include "tree-walk.h"
13 #include "list-objects.h"
17 #include "pack-bitmap.h"
18 #include "pack-objects.h"
19 #include "delta-islands.h"
20 #include "oid-array.h"
23 KHASH_INIT(str, const char *, void *, 1, kh_str_hash_func, kh_str_hash_equal)
25 static kh_oid_map_t *island_marks;
26 static unsigned island_counter;
27 static unsigned island_counter_core;
29 static kh_str_t *remote_islands;
31 struct remote_island {
33 struct oid_array oids;
36 struct island_bitmap {
38 uint32_t bits[FLEX_ARRAY];
41 static uint32_t island_bitmap_size;
44 * Allocate a new bitmap; if "old" is not NULL, the new bitmap will be a copy
45 * of "old". Otherwise, the new bitmap is empty.
47 static struct island_bitmap *island_bitmap_new(const struct island_bitmap *old)
49 size_t size = sizeof(struct island_bitmap) + (island_bitmap_size * 4);
50 struct island_bitmap *b = xcalloc(1, size);
59 static void island_bitmap_or(struct island_bitmap *a, const struct island_bitmap *b)
63 for (i = 0; i < island_bitmap_size; ++i)
64 a->bits[i] |= b->bits[i];
67 static int island_bitmap_is_subset(struct island_bitmap *self,
68 struct island_bitmap *super)
75 for (i = 0; i < island_bitmap_size; ++i) {
76 if ((self->bits[i] & super->bits[i]) != self->bits[i])
83 #define ISLAND_BITMAP_BLOCK(x) (x / 32)
84 #define ISLAND_BITMAP_MASK(x) (1 << (x % 32))
86 static void island_bitmap_set(struct island_bitmap *self, uint32_t i)
88 self->bits[ISLAND_BITMAP_BLOCK(i)] |= ISLAND_BITMAP_MASK(i);
91 static int island_bitmap_get(struct island_bitmap *self, uint32_t i)
93 return (self->bits[ISLAND_BITMAP_BLOCK(i)] & ISLAND_BITMAP_MASK(i)) != 0;
96 int in_same_island(const struct object_id *trg_oid, const struct object_id *src_oid)
98 khiter_t trg_pos, src_pos;
100 /* If we aren't using islands, assume everything goes together. */
105 * If we don't have a bitmap for the target, we can delta it
106 * against anything -- it's not an important object
108 trg_pos = kh_get_oid_map(island_marks, *trg_oid);
109 if (trg_pos >= kh_end(island_marks))
113 * if the source (our delta base) doesn't have a bitmap,
114 * we don't want to base any deltas on it!
116 src_pos = kh_get_oid_map(island_marks, *src_oid);
117 if (src_pos >= kh_end(island_marks))
120 return island_bitmap_is_subset(kh_value(island_marks, trg_pos),
121 kh_value(island_marks, src_pos));
124 int island_delta_cmp(const struct object_id *a, const struct object_id *b)
126 khiter_t a_pos, b_pos;
127 struct island_bitmap *a_bitmap = NULL, *b_bitmap = NULL;
132 a_pos = kh_get_oid_map(island_marks, *a);
133 if (a_pos < kh_end(island_marks))
134 a_bitmap = kh_value(island_marks, a_pos);
136 b_pos = kh_get_oid_map(island_marks, *b);
137 if (b_pos < kh_end(island_marks))
138 b_bitmap = kh_value(island_marks, b_pos);
141 if (!b_bitmap || !island_bitmap_is_subset(a_bitmap, b_bitmap))
145 if (!a_bitmap || !island_bitmap_is_subset(b_bitmap, a_bitmap))
152 static struct island_bitmap *create_or_get_island_marks(struct object *obj)
157 pos = kh_put_oid_map(island_marks, obj->oid, &hash_ret);
159 kh_value(island_marks, pos) = island_bitmap_new(NULL);
161 return kh_value(island_marks, pos);
164 static void set_island_marks(struct object *obj, struct island_bitmap *marks)
166 struct island_bitmap *b;
170 pos = kh_put_oid_map(island_marks, obj->oid, &hash_ret);
173 * We don't have one yet; make a copy-on-write of the
177 kh_value(island_marks, pos) = marks;
182 * We do have it. Make sure we split any copy-on-write before
185 b = kh_value(island_marks, pos);
186 if (b->refcount > 1) {
188 b = kh_value(island_marks, pos) = island_bitmap_new(b);
190 island_bitmap_or(b, marks);
193 static void mark_remote_island_1(struct repository *r,
194 struct remote_island *rl,
199 for (i = 0; i < rl->oids.nr; ++i) {
200 struct island_bitmap *marks;
201 struct object *obj = parse_object(r, &rl->oids.oid[i]);
206 marks = create_or_get_island_marks(obj);
207 island_bitmap_set(marks, island_counter);
209 if (is_core_island && obj->type == OBJ_COMMIT)
210 obj->flags |= NEEDS_BITMAP;
212 /* If it was a tag, also make sure we hit the underlying object. */
213 while (obj && obj->type == OBJ_TAG) {
214 obj = ((struct tag *)obj)->tagged;
216 parse_object(r, &obj->oid);
217 marks = create_or_get_island_marks(obj);
218 island_bitmap_set(marks, island_counter);
224 island_counter_core = island_counter;
229 struct tree_islands_todo {
230 struct object_entry *entry;
234 static int tree_depth_compare(const void *a, const void *b)
236 const struct tree_islands_todo *todo_a = a;
237 const struct tree_islands_todo *todo_b = b;
239 return todo_a->depth - todo_b->depth;
242 void resolve_tree_islands(struct repository *r,
244 struct packing_data *to_pack)
246 struct progress *progress_state = NULL;
247 struct tree_islands_todo *todo;
255 * We process only trees, as commits and tags have already been handled
256 * (and passed their marks on to root trees, as well. We must make sure
257 * to process them in descending tree-depth order so that marks
258 * propagate down the tree properly, even if a sub-tree is found in
259 * multiple parent trees.
261 ALLOC_ARRAY(todo, to_pack->nr_objects);
262 for (i = 0; i < to_pack->nr_objects; i++) {
263 if (oe_type(&to_pack->objects[i]) == OBJ_TREE) {
264 todo[nr].entry = &to_pack->objects[i];
265 todo[nr].depth = oe_tree_depth(to_pack, &to_pack->objects[i]);
269 QSORT(todo, nr, tree_depth_compare);
272 progress_state = start_progress(_("Propagating island marks"), nr);
274 for (i = 0; i < nr; i++) {
275 struct object_entry *ent = todo[i].entry;
276 struct island_bitmap *root_marks;
278 struct tree_desc desc;
279 struct name_entry entry;
282 pos = kh_get_oid_map(island_marks, ent->idx.oid);
283 if (pos >= kh_end(island_marks))
286 root_marks = kh_value(island_marks, pos);
288 tree = lookup_tree(r, &ent->idx.oid);
289 if (!tree || parse_tree(tree) < 0)
290 die(_("bad tree object %s"), oid_to_hex(&ent->idx.oid));
292 init_tree_desc(&desc, tree->buffer, tree->size);
293 while (tree_entry(&desc, &entry)) {
296 if (S_ISGITLINK(entry.mode))
299 obj = lookup_object(r, &entry.oid);
303 set_island_marks(obj, root_marks);
306 free_tree_buffer(tree);
308 display_progress(progress_state, i+1);
311 stop_progress(&progress_state);
315 static regex_t *island_regexes;
316 static unsigned int island_regexes_alloc, island_regexes_nr;
317 static const char *core_island_name;
319 static int island_config_callback(const char *k, const char *v, void *cb)
321 if (!strcmp(k, "pack.island")) {
322 struct strbuf re = STRBUF_INIT;
325 return config_error_nonbool(k);
327 ALLOC_GROW(island_regexes, island_regexes_nr + 1, island_regexes_alloc);
330 strbuf_addch(&re, '^');
331 strbuf_addstr(&re, v);
333 if (regcomp(&island_regexes[island_regexes_nr], re.buf, REG_EXTENDED))
334 die(_("failed to load island regex for '%s': %s"), k, re.buf);
341 if (!strcmp(k, "pack.islandcore"))
342 return git_config_string(&core_island_name, k, v);
347 static void add_ref_to_island(const char *island_name, const struct object_id *oid)
350 struct remote_island *rl = NULL;
353 khiter_t pos = kh_put_str(remote_islands, island_name, &hash_ret);
356 kh_key(remote_islands, pos) = xstrdup(island_name);
357 kh_value(remote_islands, pos) = xcalloc(1, sizeof(struct remote_island));
360 rl = kh_value(remote_islands, pos);
361 oid_array_append(&rl->oids, oid);
363 memcpy(&sha_core, oid->hash, sizeof(uint64_t));
364 rl->hash += sha_core;
367 static int find_island_for_ref(const char *refname, const struct object_id *oid,
368 int flags, void *data)
371 * We should advertise 'ARRAY_SIZE(matches) - 2' as the max,
372 * so we can diagnose below a config with more capture groups
375 regmatch_t matches[16];
377 struct strbuf island_name = STRBUF_INIT;
379 /* walk backwards to get last-one-wins ordering */
380 for (i = island_regexes_nr - 1; i >= 0; i--) {
381 if (!regexec(&island_regexes[i], refname,
382 ARRAY_SIZE(matches), matches, 0))
389 if (matches[ARRAY_SIZE(matches) - 1].rm_so != -1)
390 warning(_("island regex from config has "
391 "too many capture groups (max=%d)"),
392 (int)ARRAY_SIZE(matches) - 2);
394 for (m = 1; m < ARRAY_SIZE(matches); m++) {
395 regmatch_t *match = &matches[m];
397 if (match->rm_so == -1)
401 strbuf_addch(&island_name, '-');
403 strbuf_add(&island_name, refname + match->rm_so, match->rm_eo - match->rm_so);
406 add_ref_to_island(island_name.buf, oid);
407 strbuf_release(&island_name);
411 static struct remote_island *get_core_island(void)
413 if (core_island_name) {
414 khiter_t pos = kh_get_str(remote_islands, core_island_name);
415 if (pos < kh_end(remote_islands))
416 return kh_value(remote_islands, pos);
422 static void deduplicate_islands(struct repository *r)
424 struct remote_island *island, *core = NULL, **list;
425 unsigned int island_count, dst, src, ref, i = 0;
427 island_count = kh_size(remote_islands);
428 ALLOC_ARRAY(list, island_count);
430 kh_foreach_value(remote_islands, island, {
434 for (ref = 0; ref + 1 < island_count; ref++) {
435 for (src = ref + 1, dst = src; src < island_count; src++) {
436 if (list[ref]->hash == list[src]->hash)
440 list[dst] = list[src];
447 island_bitmap_size = (island_count / 32) + 1;
448 core = get_core_island();
450 for (i = 0; i < island_count; ++i) {
451 mark_remote_island_1(r, list[i], core && list[i]->hash == core->hash);
457 void load_delta_islands(struct repository *r, int progress)
459 island_marks = kh_init_oid_map();
460 remote_islands = kh_init_str();
462 git_config(island_config_callback, NULL);
463 for_each_ref(find_island_for_ref, NULL);
464 deduplicate_islands(r);
467 fprintf(stderr, _("Marked %d islands, done.\n"), island_counter);
470 void propagate_island_marks(struct commit *commit)
472 khiter_t pos = kh_get_oid_map(island_marks, commit->object.oid);
474 if (pos < kh_end(island_marks)) {
475 struct commit_list *p;
476 struct island_bitmap *root_marks = kh_value(island_marks, pos);
478 parse_commit(commit);
479 set_island_marks(&get_commit_tree(commit)->object, root_marks);
480 for (p = commit->parents; p; p = p->next)
481 set_island_marks(&p->item->object, root_marks);
485 int compute_pack_layers(struct packing_data *to_pack)
489 if (!core_island_name || !island_marks)
492 for (i = 0; i < to_pack->nr_objects; ++i) {
493 struct object_entry *entry = &to_pack->objects[i];
494 khiter_t pos = kh_get_oid_map(island_marks, entry->idx.oid);
496 oe_set_layer(to_pack, entry, 1);
498 if (pos < kh_end(island_marks)) {
499 struct island_bitmap *bitmap = kh_value(island_marks, pos);
501 if (island_bitmap_get(bitmap, island_counter_core))
502 oe_set_layer(to_pack, entry, 0);