2 * Copyright 2013, GitHub, Inc
3 * Copyright 2009-2013, Daniel Lemire, Cliff Moon,
4 * David McIntosh, Robert Becho, Google Inc. and Veronika Zenz
6 * This program is free software; you can redistribute it and/or
7 * modify it under the terms of the GNU General Public License
8 * as published by the Free Software Foundation; either version 2
9 * of the License, or (at your option) any later version.
11 * This program is distributed in the hope that it will be useful,
12 * but WITHOUT ANY WARRANTY; without even the implied warranty of
13 * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
14 * GNU General Public License for more details.
16 * You should have received a copy of the GNU General Public License
17 * along with this program; if not, see <http://www.gnu.org/licenses/>.
19 #include "git-compat-util.h"
23 static inline size_t min_size(size_t a, size_t b)
28 static inline size_t max_size(size_t a, size_t b)
33 static inline void buffer_grow(struct ewah_bitmap *self, size_t new_size)
35 size_t rlw_offset = (uint8_t *)self->rlw - (uint8_t *)self->buffer;
37 if (self->alloc_size >= new_size)
40 self->alloc_size = new_size;
41 REALLOC_ARRAY(self->buffer, self->alloc_size);
42 self->rlw = self->buffer + (rlw_offset / sizeof(eword_t));
45 static inline void buffer_push(struct ewah_bitmap *self, eword_t value)
47 if (self->buffer_size + 1 >= self->alloc_size)
48 buffer_grow(self, self->buffer_size * 3 / 2);
50 self->buffer[self->buffer_size++] = value;
53 static void buffer_push_rlw(struct ewah_bitmap *self, eword_t value)
55 buffer_push(self, value);
56 self->rlw = self->buffer + self->buffer_size - 1;
59 static size_t add_empty_words(struct ewah_bitmap *self, int v, size_t number)
62 eword_t runlen, can_add;
64 if (rlw_get_run_bit(self->rlw) != v && rlw_size(self->rlw) == 0) {
65 rlw_set_run_bit(self->rlw, v);
66 } else if (rlw_get_literal_words(self->rlw) != 0 ||
67 rlw_get_run_bit(self->rlw) != v) {
68 buffer_push_rlw(self, 0);
69 if (v) rlw_set_run_bit(self->rlw, v);
73 runlen = rlw_get_running_len(self->rlw);
74 can_add = min_size(number, RLW_LARGEST_RUNNING_COUNT - runlen);
76 rlw_set_running_len(self->rlw, runlen + can_add);
79 while (number >= RLW_LARGEST_RUNNING_COUNT) {
80 buffer_push_rlw(self, 0);
82 if (v) rlw_set_run_bit(self->rlw, v);
83 rlw_set_running_len(self->rlw, RLW_LARGEST_RUNNING_COUNT);
84 number -= RLW_LARGEST_RUNNING_COUNT;
88 buffer_push_rlw(self, 0);
91 if (v) rlw_set_run_bit(self->rlw, v);
92 rlw_set_running_len(self->rlw, number);
98 size_t ewah_add_empty_words(struct ewah_bitmap *self, int v, size_t number)
103 self->bit_size += number * BITS_IN_EWORD;
104 return add_empty_words(self, v, number);
107 static size_t add_literal(struct ewah_bitmap *self, eword_t new_data)
109 eword_t current_num = rlw_get_literal_words(self->rlw);
111 if (current_num >= RLW_LARGEST_LITERAL_COUNT) {
112 buffer_push_rlw(self, 0);
114 rlw_set_literal_words(self->rlw, 1);
115 buffer_push(self, new_data);
119 rlw_set_literal_words(self->rlw, current_num + 1);
122 assert(rlw_get_literal_words(self->rlw) == current_num + 1);
124 buffer_push(self, new_data);
128 void ewah_add_dirty_words(
129 struct ewah_bitmap *self, const eword_t *buffer,
130 size_t number, int negate)
132 size_t literals, can_add;
135 literals = rlw_get_literal_words(self->rlw);
136 can_add = min_size(number, RLW_LARGEST_LITERAL_COUNT - literals);
138 rlw_set_literal_words(self->rlw, literals + can_add);
140 if (self->buffer_size + can_add >= self->alloc_size)
141 buffer_grow(self, (self->buffer_size + can_add) * 3 / 2);
145 for (i = 0; i < can_add; ++i)
146 self->buffer[self->buffer_size++] = ~buffer[i];
148 memcpy(self->buffer + self->buffer_size,
149 buffer, can_add * sizeof(eword_t));
150 self->buffer_size += can_add;
153 self->bit_size += can_add * BITS_IN_EWORD;
155 if (number - can_add == 0)
158 buffer_push_rlw(self, 0);
164 static size_t add_empty_word(struct ewah_bitmap *self, int v)
166 int no_literal = (rlw_get_literal_words(self->rlw) == 0);
167 eword_t run_len = rlw_get_running_len(self->rlw);
169 if (no_literal && run_len == 0) {
170 rlw_set_run_bit(self->rlw, v);
171 assert(rlw_get_run_bit(self->rlw) == v);
174 if (no_literal && rlw_get_run_bit(self->rlw) == v &&
175 run_len < RLW_LARGEST_RUNNING_COUNT) {
176 rlw_set_running_len(self->rlw, run_len + 1);
177 assert(rlw_get_running_len(self->rlw) == run_len + 1);
180 buffer_push_rlw(self, 0);
182 assert(rlw_get_running_len(self->rlw) == 0);
183 assert(rlw_get_run_bit(self->rlw) == 0);
184 assert(rlw_get_literal_words(self->rlw) == 0);
186 rlw_set_run_bit(self->rlw, v);
187 assert(rlw_get_run_bit(self->rlw) == v);
189 rlw_set_running_len(self->rlw, 1);
190 assert(rlw_get_running_len(self->rlw) == 1);
191 assert(rlw_get_literal_words(self->rlw) == 0);
196 size_t ewah_add(struct ewah_bitmap *self, eword_t word)
198 self->bit_size += BITS_IN_EWORD;
201 return add_empty_word(self, 0);
203 if (word == (eword_t)(~0))
204 return add_empty_word(self, 1);
206 return add_literal(self, word);
209 void ewah_set(struct ewah_bitmap *self, size_t i)
212 DIV_ROUND_UP(i + 1, BITS_IN_EWORD) -
213 DIV_ROUND_UP(self->bit_size, BITS_IN_EWORD);
215 assert(i >= self->bit_size);
217 self->bit_size = i + 1;
221 add_empty_words(self, 0, dist - 1);
223 add_literal(self, (eword_t)1 << (i % BITS_IN_EWORD));
227 if (rlw_get_literal_words(self->rlw) == 0) {
228 rlw_set_running_len(self->rlw,
229 rlw_get_running_len(self->rlw) - 1);
230 add_literal(self, (eword_t)1 << (i % BITS_IN_EWORD));
234 self->buffer[self->buffer_size - 1] |=
235 ((eword_t)1 << (i % BITS_IN_EWORD));
237 /* check if we just completed a stream of 1s */
238 if (self->buffer[self->buffer_size - 1] == (eword_t)(~0)) {
239 self->buffer[--self->buffer_size] = 0;
240 rlw_set_literal_words(self->rlw,
241 rlw_get_literal_words(self->rlw) - 1);
242 add_empty_word(self, 1);
246 void ewah_each_bit(struct ewah_bitmap *self, void (*callback)(size_t, void*), void *payload)
252 while (pointer < self->buffer_size) {
253 eword_t *word = &self->buffer[pointer];
255 if (rlw_get_run_bit(word)) {
256 size_t len = rlw_get_running_len(word) * BITS_IN_EWORD;
257 for (k = 0; k < len; ++k, ++pos)
258 callback(pos, payload);
260 pos += rlw_get_running_len(word) * BITS_IN_EWORD;
265 for (k = 0; k < rlw_get_literal_words(word); ++k) {
268 /* todo: zero count optimization */
269 for (c = 0; c < BITS_IN_EWORD; ++c, ++pos) {
270 if ((self->buffer[pointer] & ((eword_t)1 << c)) != 0)
271 callback(pos, payload);
279 struct ewah_bitmap *ewah_new(void)
281 struct ewah_bitmap *self;
283 self = xmalloc(sizeof(struct ewah_bitmap));
284 self->alloc_size = 32;
285 ALLOC_ARRAY(self->buffer, self->alloc_size);
291 void ewah_clear(struct ewah_bitmap *self)
293 self->buffer_size = 1;
296 self->rlw = self->buffer;
299 void ewah_free(struct ewah_bitmap *self)
304 if (self->alloc_size)
310 static void read_new_rlw(struct ewah_iterator *it)
312 const eword_t *word = NULL;
318 word = &it->buffer[it->pointer];
320 it->rl = rlw_get_running_len(word);
321 it->lw = rlw_get_literal_words(word);
322 it->b = rlw_get_run_bit(word);
324 if (it->rl || it->lw)
327 if (it->pointer < it->buffer_size - 1) {
330 it->pointer = it->buffer_size;
336 int ewah_iterator_next(eword_t *next, struct ewah_iterator *it)
338 if (it->pointer >= it->buffer_size)
341 if (it->compressed < it->rl) {
343 *next = it->b ? (eword_t)(~0) : 0;
345 assert(it->literals < it->lw);
350 assert(it->pointer < it->buffer_size);
352 *next = it->buffer[it->pointer];
355 if (it->compressed == it->rl && it->literals == it->lw) {
356 if (++it->pointer < it->buffer_size)
363 void ewah_iterator_init(struct ewah_iterator *it, struct ewah_bitmap *parent)
365 it->buffer = parent->buffer;
366 it->buffer_size = parent->buffer_size;
375 if (it->pointer < it->buffer_size)
379 void ewah_not(struct ewah_bitmap *self)
383 while (pointer < self->buffer_size) {
384 eword_t *word = &self->buffer[pointer];
387 rlw_xor_run_bit(word);
390 literals = rlw_get_literal_words(word);
391 for (k = 0; k < literals; ++k) {
392 self->buffer[pointer] = ~self->buffer[pointer];
399 struct ewah_bitmap *ewah_i,
400 struct ewah_bitmap *ewah_j,
401 struct ewah_bitmap *out)
403 struct rlw_iterator rlw_i;
404 struct rlw_iterator rlw_j;
407 rlwit_init(&rlw_i, ewah_i);
408 rlwit_init(&rlw_j, ewah_j);
410 while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
411 while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
412 struct rlw_iterator *prey, *predator;
416 if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
424 negate_words = !!predator->rlw.running_bit;
425 index = rlwit_discharge(prey, out,
426 predator->rlw.running_len, negate_words);
428 ewah_add_empty_words(out, negate_words,
429 predator->rlw.running_len - index);
431 rlwit_discard_first_words(predator,
432 predator->rlw.running_len);
436 rlw_i.rlw.literal_words,
437 rlw_j.rlw.literal_words);
442 for (k = 0; k < literals; ++k) {
444 rlw_i.buffer[rlw_i.literal_word_start + k] ^
445 rlw_j.buffer[rlw_j.literal_word_start + k]
449 rlwit_discard_first_words(&rlw_i, literals);
450 rlwit_discard_first_words(&rlw_j, literals);
454 if (rlwit_word_size(&rlw_i) > 0)
455 rlwit_discharge(&rlw_i, out, ~0, 0);
457 rlwit_discharge(&rlw_j, out, ~0, 0);
459 out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
463 struct ewah_bitmap *ewah_i,
464 struct ewah_bitmap *ewah_j,
465 struct ewah_bitmap *out)
467 struct rlw_iterator rlw_i;
468 struct rlw_iterator rlw_j;
471 rlwit_init(&rlw_i, ewah_i);
472 rlwit_init(&rlw_j, ewah_j);
474 while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
475 while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
476 struct rlw_iterator *prey, *predator;
478 if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
486 if (predator->rlw.running_bit == 0) {
487 ewah_add_empty_words(out, 0,
488 predator->rlw.running_len);
489 rlwit_discard_first_words(prey,
490 predator->rlw.running_len);
491 rlwit_discard_first_words(predator,
492 predator->rlw.running_len);
494 size_t index = rlwit_discharge(prey, out,
495 predator->rlw.running_len, 0);
496 ewah_add_empty_words(out, 0,
497 predator->rlw.running_len - index);
498 rlwit_discard_first_words(predator,
499 predator->rlw.running_len);
504 rlw_i.rlw.literal_words,
505 rlw_j.rlw.literal_words);
510 for (k = 0; k < literals; ++k) {
512 rlw_i.buffer[rlw_i.literal_word_start + k] &
513 rlw_j.buffer[rlw_j.literal_word_start + k]
517 rlwit_discard_first_words(&rlw_i, literals);
518 rlwit_discard_first_words(&rlw_j, literals);
522 if (rlwit_word_size(&rlw_i) > 0)
523 rlwit_discharge_empty(&rlw_i, out);
525 rlwit_discharge_empty(&rlw_j, out);
527 out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
531 struct ewah_bitmap *ewah_i,
532 struct ewah_bitmap *ewah_j,
533 struct ewah_bitmap *out)
535 struct rlw_iterator rlw_i;
536 struct rlw_iterator rlw_j;
539 rlwit_init(&rlw_i, ewah_i);
540 rlwit_init(&rlw_j, ewah_j);
542 while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
543 while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
544 struct rlw_iterator *prey, *predator;
546 if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
554 if ((predator->rlw.running_bit && prey == &rlw_i) ||
555 (!predator->rlw.running_bit && prey != &rlw_i)) {
556 ewah_add_empty_words(out, 0,
557 predator->rlw.running_len);
558 rlwit_discard_first_words(prey,
559 predator->rlw.running_len);
560 rlwit_discard_first_words(predator,
561 predator->rlw.running_len);
566 negate_words = (&rlw_i != prey);
567 index = rlwit_discharge(prey, out,
568 predator->rlw.running_len, negate_words);
569 ewah_add_empty_words(out, negate_words,
570 predator->rlw.running_len - index);
571 rlwit_discard_first_words(predator,
572 predator->rlw.running_len);
577 rlw_i.rlw.literal_words,
578 rlw_j.rlw.literal_words);
583 for (k = 0; k < literals; ++k) {
585 rlw_i.buffer[rlw_i.literal_word_start + k] &
586 ~(rlw_j.buffer[rlw_j.literal_word_start + k])
590 rlwit_discard_first_words(&rlw_i, literals);
591 rlwit_discard_first_words(&rlw_j, literals);
595 if (rlwit_word_size(&rlw_i) > 0)
596 rlwit_discharge(&rlw_i, out, ~0, 0);
598 rlwit_discharge_empty(&rlw_j, out);
600 out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
604 struct ewah_bitmap *ewah_i,
605 struct ewah_bitmap *ewah_j,
606 struct ewah_bitmap *out)
608 struct rlw_iterator rlw_i;
609 struct rlw_iterator rlw_j;
612 rlwit_init(&rlw_i, ewah_i);
613 rlwit_init(&rlw_j, ewah_j);
615 while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
616 while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
617 struct rlw_iterator *prey, *predator;
619 if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
627 if (predator->rlw.running_bit) {
628 ewah_add_empty_words(out, 0,
629 predator->rlw.running_len);
630 rlwit_discard_first_words(prey,
631 predator->rlw.running_len);
632 rlwit_discard_first_words(predator,
633 predator->rlw.running_len);
635 size_t index = rlwit_discharge(prey, out,
636 predator->rlw.running_len, 0);
637 ewah_add_empty_words(out, 0,
638 predator->rlw.running_len - index);
639 rlwit_discard_first_words(predator,
640 predator->rlw.running_len);
645 rlw_i.rlw.literal_words,
646 rlw_j.rlw.literal_words);
651 for (k = 0; k < literals; ++k) {
653 rlw_i.buffer[rlw_i.literal_word_start + k] |
654 rlw_j.buffer[rlw_j.literal_word_start + k]
658 rlwit_discard_first_words(&rlw_i, literals);
659 rlwit_discard_first_words(&rlw_j, literals);
663 if (rlwit_word_size(&rlw_i) > 0)
664 rlwit_discharge(&rlw_i, out, ~0, 0);
666 rlwit_discharge(&rlw_j, out, ~0, 0);
668 out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
672 #define BITMAP_POOL_MAX 16
673 static struct ewah_bitmap *bitmap_pool[BITMAP_POOL_MAX];
674 static size_t bitmap_pool_size;
676 struct ewah_bitmap *ewah_pool_new(void)
678 if (bitmap_pool_size)
679 return bitmap_pool[--bitmap_pool_size];
684 void ewah_pool_free(struct ewah_bitmap *self)
689 if (bitmap_pool_size == BITMAP_POOL_MAX ||
690 self->alloc_size == 0) {
696 bitmap_pool[bitmap_pool_size++] = self;
699 uint32_t ewah_checksum(struct ewah_bitmap *self)
701 const uint8_t *p = (uint8_t *)self->buffer;
702 uint32_t crc = (uint32_t)self->bit_size;
703 size_t size = self->buffer_size * sizeof(eword_t);
706 crc = (crc << 5) - crc + (uint32_t)*p++;