4 * Hashing names in the index state
6 * Copyright (C) 2008 Linus Torvalds
9 #include "thread-utils.h"
13 struct hashmap_entry ent;
14 struct dir_entry *parent;
17 char name[FLEX_ARRAY];
20 static int dir_entry_cmp(const void *unused_cmp_data,
21 const struct hashmap_entry *eptr,
22 const struct hashmap_entry *entry_or_key,
25 const struct dir_entry *e1, *e2;
26 const char *name = keydata;
28 e1 = container_of(eptr, const struct dir_entry, ent);
29 e2 = container_of(entry_or_key, const struct dir_entry, ent);
31 return e1->namelen != e2->namelen || strncasecmp(e1->name,
32 name ? name : e2->name, e1->namelen);
35 static struct dir_entry *find_dir_entry__hash(struct index_state *istate,
36 const char *name, unsigned int namelen, unsigned int hash)
39 hashmap_entry_init(&key.ent, hash);
40 key.namelen = namelen;
41 return hashmap_get_entry(&istate->dir_hash, &key, ent, name);
44 static struct dir_entry *find_dir_entry(struct index_state *istate,
45 const char *name, unsigned int namelen)
47 return find_dir_entry__hash(istate, name, namelen, memihash(name, namelen));
50 static struct dir_entry *hash_dir_entry(struct index_state *istate,
51 struct cache_entry *ce, int namelen)
54 * Throw each directory component in the hash for quick lookup
55 * during a git status. Directory components are stored without their
56 * closing slash. Despite submodules being a directory, they never
57 * reach this point, because they are stored
58 * in index_state.name_hash (as ordinary cache_entries).
60 struct dir_entry *dir;
62 /* get length of parent directory */
63 while (namelen > 0 && !is_dir_sep(ce->name[namelen - 1]))
69 /* lookup existing entry for that directory */
70 dir = find_dir_entry(istate, ce->name, namelen);
72 /* not found, create it and add to hash table */
73 FLEX_ALLOC_MEM(dir, name, ce->name, namelen);
74 hashmap_entry_init(&dir->ent, memihash(ce->name, namelen));
75 dir->namelen = namelen;
76 hashmap_add(&istate->dir_hash, &dir->ent);
78 /* recursively add missing parent directories */
79 dir->parent = hash_dir_entry(istate, ce, namelen);
84 static void add_dir_entry(struct index_state *istate, struct cache_entry *ce)
86 /* Add reference to the directory entry (and parents if 0). */
87 struct dir_entry *dir = hash_dir_entry(istate, ce, ce_namelen(ce));
88 while (dir && !(dir->nr++))
92 static void remove_dir_entry(struct index_state *istate, struct cache_entry *ce)
95 * Release reference to the directory entry. If 0, remove and continue
96 * with parent directory.
98 struct dir_entry *dir = hash_dir_entry(istate, ce, ce_namelen(ce));
99 while (dir && !(--dir->nr)) {
100 struct dir_entry *parent = dir->parent;
101 hashmap_remove(&istate->dir_hash, &dir->ent, NULL);
107 static void hash_index_entry(struct index_state *istate, struct cache_entry *ce)
109 if (ce->ce_flags & CE_HASHED)
111 ce->ce_flags |= CE_HASHED;
113 if (!S_ISSPARSEDIR(ce->ce_mode)) {
114 hashmap_entry_init(&ce->ent, memihash(ce->name, ce_namelen(ce)));
115 hashmap_add(&istate->name_hash, &ce->ent);
119 add_dir_entry(istate, ce);
122 static int cache_entry_cmp(const void *unused_cmp_data,
123 const struct hashmap_entry *eptr,
124 const struct hashmap_entry *entry_or_key,
127 const struct cache_entry *ce1, *ce2;
129 ce1 = container_of(eptr, const struct cache_entry, ent);
130 ce2 = container_of(entry_or_key, const struct cache_entry, ent);
133 * For remove_name_hash, find the exact entry (pointer equality); for
134 * index_file_exists, find all entries with matching hash code and
135 * decide whether the entry matches in same_name.
137 return remove ? !(ce1 == ce2) : 0;
140 static int lazy_try_threaded = 1;
141 static int lazy_nr_dir_threads;
144 * Set a minimum number of cache_entries that we will handle per
145 * thread and use that to decide how many threads to run (up to
146 * the number on the system).
148 * For guidance setting the lower per-thread bound, see:
149 * t/helper/test-lazy-init-name-hash --analyze
151 #define LAZY_THREAD_COST (2000)
154 * We use n mutexes to guard n partitions of the "istate->dir_hash"
155 * hashtable. Since "find" and "insert" operations will hash to a
156 * particular bucket and modify/search a single chain, we can say
157 * that "all chains mod n" are guarded by the same mutex -- rather
158 * than having a single mutex to guard the entire table. (This does
159 * require that we disable "rehashing" on the hashtable.)
161 * So, a larger value here decreases the probability of a collision
162 * and the time that each thread must wait for the mutex.
164 #define LAZY_MAX_MUTEX (32)
166 static pthread_mutex_t *lazy_dir_mutex_array;
169 * An array of lazy_entry items is used by the n threads in
170 * the directory parse (first) phase to (lock-free) store the
171 * intermediate results. These values are then referenced by
172 * the 2 threads in the second phase.
175 struct dir_entry *dir;
176 unsigned int hash_dir;
177 unsigned int hash_name;
181 * Decide if we want to use threads (if available) to load
182 * the hash tables. We set "lazy_nr_dir_threads" to zero when
183 * it is not worth it.
185 static int lookup_lazy_params(struct index_state *istate)
189 lazy_nr_dir_threads = 0;
191 if (!lazy_try_threaded)
195 * If we are respecting case, just use the original
196 * code to build the "istate->name_hash". We don't
197 * need the complexity here.
202 nr_cpus = online_cpus();
206 if (istate->cache_nr < 2 * LAZY_THREAD_COST)
209 if (istate->cache_nr < nr_cpus * LAZY_THREAD_COST)
210 nr_cpus = istate->cache_nr / LAZY_THREAD_COST;
211 lazy_nr_dir_threads = nr_cpus;
212 return lazy_nr_dir_threads;
216 * Initialize n mutexes for use when searching and inserting
217 * into "istate->dir_hash". All "dir" threads are trying
218 * to insert partial pathnames into the hash as they iterate
219 * over their portions of the index, so lock contention is
222 * However, the hashmap is going to put items into bucket
223 * chains based on their hash values. Use that to create n
224 * mutexes and lock on mutex[bucket(hash) % n]. This will
225 * decrease the collision rate by (hopefully) a factor of n.
227 static void init_dir_mutex(void)
231 lazy_dir_mutex_array = xcalloc(LAZY_MAX_MUTEX, sizeof(pthread_mutex_t));
233 for (j = 0; j < LAZY_MAX_MUTEX; j++)
234 init_recursive_mutex(&lazy_dir_mutex_array[j]);
237 static void cleanup_dir_mutex(void)
241 for (j = 0; j < LAZY_MAX_MUTEX; j++)
242 pthread_mutex_destroy(&lazy_dir_mutex_array[j]);
244 free(lazy_dir_mutex_array);
247 static void lock_dir_mutex(int j)
249 pthread_mutex_lock(&lazy_dir_mutex_array[j]);
252 static void unlock_dir_mutex(int j)
254 pthread_mutex_unlock(&lazy_dir_mutex_array[j]);
257 static inline int compute_dir_lock_nr(
258 const struct hashmap *map,
261 return hashmap_bucket(map, hash) % LAZY_MAX_MUTEX;
264 static struct dir_entry *hash_dir_entry_with_parent_and_prefix(
265 struct index_state *istate,
266 struct dir_entry *parent,
267 struct strbuf *prefix)
269 struct dir_entry *dir;
274 * Either we have a parent directory and path with slash(es)
275 * or the directory is an immediate child of the root directory.
277 assert((parent != NULL) ^ (strchr(prefix->buf, '/') == NULL));
280 hash = memihash_cont(parent->ent.hash,
281 prefix->buf + parent->namelen,
282 prefix->len - parent->namelen);
284 hash = memihash(prefix->buf, prefix->len);
286 lock_nr = compute_dir_lock_nr(&istate->dir_hash, hash);
287 lock_dir_mutex(lock_nr);
289 dir = find_dir_entry__hash(istate, prefix->buf, prefix->len, hash);
291 FLEX_ALLOC_MEM(dir, name, prefix->buf, prefix->len);
292 hashmap_entry_init(&dir->ent, hash);
293 dir->namelen = prefix->len;
294 dir->parent = parent;
295 hashmap_add(&istate->dir_hash, &dir->ent);
298 unlock_dir_mutex(lock_nr);
300 /* All I really need here is an InterlockedIncrement(&(parent->nr)) */
301 lock_nr = compute_dir_lock_nr(&istate->dir_hash, parent->ent.hash);
302 lock_dir_mutex(lock_nr);
307 unlock_dir_mutex(lock_nr);
313 * handle_range_1() and handle_range_dir() are derived from
314 * clear_ce_flags_1() and clear_ce_flags_dir() in unpack-trees.c
315 * and handle the iteration over the entire array of index entries.
316 * They use recursion for adjacent entries in the same parent
319 static int handle_range_1(
320 struct index_state *istate,
323 struct dir_entry *parent,
324 struct strbuf *prefix,
325 struct lazy_entry *lazy_entries);
327 static int handle_range_dir(
328 struct index_state *istate,
331 struct dir_entry *parent,
332 struct strbuf *prefix,
333 struct lazy_entry *lazy_entries,
334 struct dir_entry **dir_new_out)
337 int input_prefix_len = prefix->len;
338 struct dir_entry *dir_new;
340 dir_new = hash_dir_entry_with_parent_and_prefix(istate, parent, prefix);
342 strbuf_addch(prefix, '/');
345 * Scan forward in the index array for index entries having the same
346 * path prefix (that are also in this directory).
348 if (k_start + 1 >= k_end)
350 else if (strncmp(istate->cache[k_start + 1]->name, prefix->buf, prefix->len) > 0)
352 else if (strncmp(istate->cache[k_end - 1]->name, prefix->buf, prefix->len) == 0)
358 while (begin < end) {
359 int mid = begin + ((end - begin) >> 1);
360 int cmp = strncmp(istate->cache[mid]->name, prefix->buf, prefix->len);
361 if (cmp == 0) /* mid has same prefix; look in second part */
363 else if (cmp > 0) /* mid is past group; look in first part */
366 die("cache entry out of order");
372 * Recurse and process what we can of this subset [k_start, k).
374 rc = handle_range_1(istate, k_start, k, dir_new, prefix, lazy_entries);
376 strbuf_setlen(prefix, input_prefix_len);
378 *dir_new_out = dir_new;
382 static int handle_range_1(
383 struct index_state *istate,
386 struct dir_entry *parent,
387 struct strbuf *prefix,
388 struct lazy_entry *lazy_entries)
390 int input_prefix_len = prefix->len;
394 struct cache_entry *ce_k = istate->cache[k];
395 const char *name, *slash;
397 if (prefix->len && strncmp(ce_k->name, prefix->buf, prefix->len))
400 name = ce_k->name + prefix->len;
401 slash = strchr(name, '/');
404 int len = slash - name;
406 struct dir_entry *dir_new;
408 strbuf_add(prefix, name, len);
409 processed = handle_range_dir(istate, k, k_end, parent, prefix, lazy_entries, &dir_new);
412 strbuf_setlen(prefix, input_prefix_len);
416 strbuf_addch(prefix, '/');
417 processed = handle_range_1(istate, k, k_end, dir_new, prefix, lazy_entries);
419 strbuf_setlen(prefix, input_prefix_len);
424 * It is too expensive to take a lock to insert "ce_k"
425 * into "istate->name_hash" and increment the ref-count
426 * on the "parent" dir. So we defer actually updating
427 * permanent data structures until phase 2 (where we
428 * can change the locking requirements) and simply
429 * accumulate our current results into the lazy_entries
432 * We do not need to lock the lazy_entries array because
433 * we have exclusive access to the cells in the range
434 * [k_start,k_end) that this thread was given.
436 lazy_entries[k].dir = parent;
438 lazy_entries[k].hash_name = memihash_cont(
440 ce_k->name + parent->namelen,
441 ce_namelen(ce_k) - parent->namelen);
442 lazy_entries[k].hash_dir = parent->ent.hash;
444 lazy_entries[k].hash_name = memihash(ce_k->name, ce_namelen(ce_k));
453 struct lazy_dir_thread_data {
455 struct index_state *istate;
456 struct lazy_entry *lazy_entries;
461 static void *lazy_dir_thread_proc(void *_data)
463 struct lazy_dir_thread_data *d = _data;
464 struct strbuf prefix = STRBUF_INIT;
465 handle_range_1(d->istate, d->k_start, d->k_end, NULL, &prefix, d->lazy_entries);
466 strbuf_release(&prefix);
470 struct lazy_name_thread_data {
472 struct index_state *istate;
473 struct lazy_entry *lazy_entries;
476 static void *lazy_name_thread_proc(void *_data)
478 struct lazy_name_thread_data *d = _data;
481 for (k = 0; k < d->istate->cache_nr; k++) {
482 struct cache_entry *ce_k = d->istate->cache[k];
483 ce_k->ce_flags |= CE_HASHED;
484 hashmap_entry_init(&ce_k->ent, d->lazy_entries[k].hash_name);
485 hashmap_add(&d->istate->name_hash, &ce_k->ent);
491 static inline void lazy_update_dir_ref_counts(
492 struct index_state *istate,
493 struct lazy_entry *lazy_entries)
497 for (k = 0; k < istate->cache_nr; k++) {
498 if (lazy_entries[k].dir)
499 lazy_entries[k].dir->nr++;
503 static void threaded_lazy_init_name_hash(
504 struct index_state *istate)
510 struct lazy_entry *lazy_entries;
511 struct lazy_dir_thread_data *td_dir;
512 struct lazy_name_thread_data *td_name;
518 nr_each = DIV_ROUND_UP(istate->cache_nr, lazy_nr_dir_threads);
520 lazy_entries = xcalloc(istate->cache_nr, sizeof(struct lazy_entry));
521 td_dir = xcalloc(lazy_nr_dir_threads, sizeof(struct lazy_dir_thread_data));
522 td_name = xcalloc(1, sizeof(struct lazy_name_thread_data));
528 * Build "istate->dir_hash" using n "dir" threads (and a read-only index).
530 for (t = 0; t < lazy_nr_dir_threads; t++) {
531 struct lazy_dir_thread_data *td_dir_t = td_dir + t;
532 td_dir_t->istate = istate;
533 td_dir_t->lazy_entries = lazy_entries;
534 td_dir_t->k_start = k_start;
536 if (k_start > istate->cache_nr)
537 k_start = istate->cache_nr;
538 td_dir_t->k_end = k_start;
539 err = pthread_create(&td_dir_t->pthread, NULL, lazy_dir_thread_proc, td_dir_t);
541 die(_("unable to create lazy_dir thread: %s"), strerror(err));
543 for (t = 0; t < lazy_nr_dir_threads; t++) {
544 struct lazy_dir_thread_data *td_dir_t = td_dir + t;
545 if (pthread_join(td_dir_t->pthread, NULL))
546 die("unable to join lazy_dir_thread");
551 * Iterate over all index entries and add them to the "istate->name_hash"
552 * using a single "name" background thread.
553 * (Testing showed it wasn't worth running more than 1 thread for this.)
555 * Meanwhile, finish updating the parent directory ref-counts for each
556 * index entry using the current thread. (This step is very fast and
557 * doesn't need threading.)
559 td_name->istate = istate;
560 td_name->lazy_entries = lazy_entries;
561 err = pthread_create(&td_name->pthread, NULL, lazy_name_thread_proc, td_name);
563 die(_("unable to create lazy_name thread: %s"), strerror(err));
565 lazy_update_dir_ref_counts(istate, lazy_entries);
567 err = pthread_join(td_name->pthread, NULL);
569 die(_("unable to join lazy_name thread: %s"), strerror(err));
578 static void lazy_init_name_hash(struct index_state *istate)
581 if (istate->name_hash_initialized)
583 trace_performance_enter();
584 trace2_region_enter("index", "name-hash-init", istate->repo);
585 hashmap_init(&istate->name_hash, cache_entry_cmp, NULL, istate->cache_nr);
586 hashmap_init(&istate->dir_hash, dir_entry_cmp, NULL, istate->cache_nr);
588 if (lookup_lazy_params(istate)) {
590 * Disable item counting and automatic rehashing because
591 * we do per-chain (mod n) locking rather than whole hashmap
592 * locking and we need to prevent the table-size from changing
593 * and bucket items from being redistributed.
595 hashmap_disable_item_counting(&istate->dir_hash);
596 threaded_lazy_init_name_hash(istate);
597 hashmap_enable_item_counting(&istate->dir_hash);
600 for (nr = 0; nr < istate->cache_nr; nr++)
601 hash_index_entry(istate, istate->cache[nr]);
604 istate->name_hash_initialized = 1;
605 trace2_region_leave("index", "name-hash-init", istate->repo);
606 trace_performance_leave("initialize name hash");
610 * A test routine for t/helper/ sources.
612 * Returns the number of threads used or 0 when
613 * the non-threaded code path was used.
615 * Requesting threading WILL NOT override guards
616 * in lookup_lazy_params().
618 int test_lazy_init_name_hash(struct index_state *istate, int try_threaded)
620 lazy_nr_dir_threads = 0;
621 lazy_try_threaded = try_threaded;
623 lazy_init_name_hash(istate);
625 return lazy_nr_dir_threads;
628 void add_name_hash(struct index_state *istate, struct cache_entry *ce)
630 if (istate->name_hash_initialized)
631 hash_index_entry(istate, ce);
634 void remove_name_hash(struct index_state *istate, struct cache_entry *ce)
636 if (!istate->name_hash_initialized || !(ce->ce_flags & CE_HASHED))
638 ce->ce_flags &= ~CE_HASHED;
639 hashmap_remove(&istate->name_hash, &ce->ent, ce);
642 remove_dir_entry(istate, ce);
645 static int slow_same_name(const char *name1, int len1, const char *name2, int len2)
651 unsigned char c1 = *name1++;
652 unsigned char c2 = *name2++;
664 static int same_name(const struct cache_entry *ce, const char *name, int namelen, int icase)
666 int len = ce_namelen(ce);
669 * Always do exact compare, even if we want a case-ignoring comparison;
670 * we do the quick exact one first, because it will be the common case.
672 if (len == namelen && !memcmp(name, ce->name, len))
678 return slow_same_name(name, namelen, ce->name, len);
681 int index_dir_exists(struct index_state *istate, const char *name, int namelen)
683 struct dir_entry *dir;
685 lazy_init_name_hash(istate);
686 dir = find_dir_entry(istate, name, namelen);
687 return dir && dir->nr;
690 void adjust_dirname_case(struct index_state *istate, char *name)
692 const char *startPtr = name;
693 const char *ptr = startPtr;
695 lazy_init_name_hash(istate);
697 while (*ptr && *ptr != '/')
701 struct dir_entry *dir;
703 dir = find_dir_entry(istate, name, ptr - name);
705 memcpy((void *)startPtr, dir->name + (startPtr - name), ptr - startPtr);
713 struct cache_entry *index_file_exists(struct index_state *istate, const char *name, int namelen, int icase)
715 struct cache_entry *ce;
716 unsigned int hash = memihash(name, namelen);
718 lazy_init_name_hash(istate);
720 ce = hashmap_get_entry_from_hash(&istate->name_hash, hash, NULL,
721 struct cache_entry, ent);
722 hashmap_for_each_entry_from(&istate->name_hash, ce, ent) {
723 if (same_name(ce, name, namelen, icase))
729 void free_name_hash(struct index_state *istate)
731 if (!istate->name_hash_initialized)
733 istate->name_hash_initialized = 0;
735 hashmap_clear(&istate->name_hash);
736 hashmap_clear_and_free(&istate->dir_hash, struct dir_entry, ent);