Merge branch 'sb/describe-blob' into next
[git] / ewah / ewah_bitmap.c
1 /**
2  * Copyright 2013, GitHub, Inc
3  * Copyright 2009-2013, Daniel Lemire, Cliff Moon,
4  *      David McIntosh, Robert Becho, Google Inc. and Veronika Zenz
5  *
6  * This program is free software; you can redistribute it and/or
7  * modify it under the terms of the GNU General Public License
8  * as published by the Free Software Foundation; either version 2
9  * of the License, or (at your option) any later version.
10  *
11  * This program is distributed in the hope that it will be useful,
12  * but WITHOUT ANY WARRANTY; without even the implied warranty of
13  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
14  * GNU General Public License for more details.
15  *
16  * You should have received a copy of the GNU General Public License
17  * along with this program; if not, see <http://www.gnu.org/licenses/>.
18  */
19 #include "git-compat-util.h"
20 #include "ewok.h"
21 #include "ewok_rlw.h"
22
23 static inline size_t min_size(size_t a, size_t b)
24 {
25         return a < b ? a : b;
26 }
27
28 static inline size_t max_size(size_t a, size_t b)
29 {
30         return a > b ? a : b;
31 }
32
33 static inline void buffer_grow(struct ewah_bitmap *self, size_t new_size)
34 {
35         size_t rlw_offset = (uint8_t *)self->rlw - (uint8_t *)self->buffer;
36
37         if (self->alloc_size >= new_size)
38                 return;
39
40         self->alloc_size = new_size;
41         REALLOC_ARRAY(self->buffer, self->alloc_size);
42         self->rlw = self->buffer + (rlw_offset / sizeof(eword_t));
43 }
44
45 static inline void buffer_push(struct ewah_bitmap *self, eword_t value)
46 {
47         if (self->buffer_size + 1 >= self->alloc_size)
48                 buffer_grow(self, self->buffer_size * 3 / 2);
49
50         self->buffer[self->buffer_size++] = value;
51 }
52
53 static void buffer_push_rlw(struct ewah_bitmap *self, eword_t value)
54 {
55         buffer_push(self, value);
56         self->rlw = self->buffer + self->buffer_size - 1;
57 }
58
59 static size_t add_empty_words(struct ewah_bitmap *self, int v, size_t number)
60 {
61         size_t added = 0;
62         eword_t runlen, can_add;
63
64         if (rlw_get_run_bit(self->rlw) != v && rlw_size(self->rlw) == 0) {
65                 rlw_set_run_bit(self->rlw, v);
66         } else if (rlw_get_literal_words(self->rlw) != 0 ||
67                         rlw_get_run_bit(self->rlw) != v) {
68                 buffer_push_rlw(self, 0);
69                 if (v) rlw_set_run_bit(self->rlw, v);
70                 added++;
71         }
72
73         runlen = rlw_get_running_len(self->rlw);
74         can_add = min_size(number, RLW_LARGEST_RUNNING_COUNT - runlen);
75
76         rlw_set_running_len(self->rlw, runlen + can_add);
77         number -= can_add;
78
79         while (number >= RLW_LARGEST_RUNNING_COUNT) {
80                 buffer_push_rlw(self, 0);
81                 added++;
82                 if (v) rlw_set_run_bit(self->rlw, v);
83                 rlw_set_running_len(self->rlw, RLW_LARGEST_RUNNING_COUNT);
84                 number -= RLW_LARGEST_RUNNING_COUNT;
85         }
86
87         if (number > 0) {
88                 buffer_push_rlw(self, 0);
89                 added++;
90
91                 if (v) rlw_set_run_bit(self->rlw, v);
92                 rlw_set_running_len(self->rlw, number);
93         }
94
95         return added;
96 }
97
98 size_t ewah_add_empty_words(struct ewah_bitmap *self, int v, size_t number)
99 {
100         if (number == 0)
101                 return 0;
102
103         self->bit_size += number * BITS_IN_EWORD;
104         return add_empty_words(self, v, number);
105 }
106
107 static size_t add_literal(struct ewah_bitmap *self, eword_t new_data)
108 {
109         eword_t current_num = rlw_get_literal_words(self->rlw);
110
111         if (current_num >= RLW_LARGEST_LITERAL_COUNT) {
112                 buffer_push_rlw(self, 0);
113
114                 rlw_set_literal_words(self->rlw, 1);
115                 buffer_push(self, new_data);
116                 return 2;
117         }
118
119         rlw_set_literal_words(self->rlw, current_num + 1);
120
121         /* sanity check */
122         assert(rlw_get_literal_words(self->rlw) == current_num + 1);
123
124         buffer_push(self, new_data);
125         return 1;
126 }
127
128 void ewah_add_dirty_words(
129         struct ewah_bitmap *self, const eword_t *buffer,
130         size_t number, int negate)
131 {
132         size_t literals, can_add;
133
134         while (1) {
135                 literals = rlw_get_literal_words(self->rlw);
136                 can_add = min_size(number, RLW_LARGEST_LITERAL_COUNT - literals);
137
138                 rlw_set_literal_words(self->rlw, literals + can_add);
139
140                 if (self->buffer_size + can_add >= self->alloc_size)
141                         buffer_grow(self, (self->buffer_size + can_add) * 3 / 2);
142
143                 if (negate) {
144                         size_t i;
145                         for (i = 0; i < can_add; ++i)
146                                 self->buffer[self->buffer_size++] = ~buffer[i];
147                 } else {
148                         memcpy(self->buffer + self->buffer_size,
149                                 buffer, can_add * sizeof(eword_t));
150                         self->buffer_size += can_add;
151                 }
152
153                 self->bit_size += can_add * BITS_IN_EWORD;
154
155                 if (number - can_add == 0)
156                         break;
157
158                 buffer_push_rlw(self, 0);
159                 buffer += can_add;
160                 number -= can_add;
161         }
162 }
163
164 static size_t add_empty_word(struct ewah_bitmap *self, int v)
165 {
166         int no_literal = (rlw_get_literal_words(self->rlw) == 0);
167         eword_t run_len = rlw_get_running_len(self->rlw);
168
169         if (no_literal && run_len == 0) {
170                 rlw_set_run_bit(self->rlw, v);
171                 assert(rlw_get_run_bit(self->rlw) == v);
172         }
173
174         if (no_literal && rlw_get_run_bit(self->rlw) == v &&
175                 run_len < RLW_LARGEST_RUNNING_COUNT) {
176                 rlw_set_running_len(self->rlw, run_len + 1);
177                 assert(rlw_get_running_len(self->rlw) == run_len + 1);
178                 return 0;
179         } else {
180                 buffer_push_rlw(self, 0);
181
182                 assert(rlw_get_running_len(self->rlw) == 0);
183                 assert(rlw_get_run_bit(self->rlw) == 0);
184                 assert(rlw_get_literal_words(self->rlw) == 0);
185
186                 rlw_set_run_bit(self->rlw, v);
187                 assert(rlw_get_run_bit(self->rlw) == v);
188
189                 rlw_set_running_len(self->rlw, 1);
190                 assert(rlw_get_running_len(self->rlw) == 1);
191                 assert(rlw_get_literal_words(self->rlw) == 0);
192                 return 1;
193         }
194 }
195
196 size_t ewah_add(struct ewah_bitmap *self, eword_t word)
197 {
198         self->bit_size += BITS_IN_EWORD;
199
200         if (word == 0)
201                 return add_empty_word(self, 0);
202
203         if (word == (eword_t)(~0))
204                 return add_empty_word(self, 1);
205
206         return add_literal(self, word);
207 }
208
209 void ewah_set(struct ewah_bitmap *self, size_t i)
210 {
211         const size_t dist =
212                 DIV_ROUND_UP(i + 1, BITS_IN_EWORD) -
213                 DIV_ROUND_UP(self->bit_size, BITS_IN_EWORD);
214
215         assert(i >= self->bit_size);
216
217         self->bit_size = i + 1;
218
219         if (dist > 0) {
220                 if (dist > 1)
221                         add_empty_words(self, 0, dist - 1);
222
223                 add_literal(self, (eword_t)1 << (i % BITS_IN_EWORD));
224                 return;
225         }
226
227         if (rlw_get_literal_words(self->rlw) == 0) {
228                 rlw_set_running_len(self->rlw,
229                         rlw_get_running_len(self->rlw) - 1);
230                 add_literal(self, (eword_t)1 << (i % BITS_IN_EWORD));
231                 return;
232         }
233
234         self->buffer[self->buffer_size - 1] |=
235                 ((eword_t)1 << (i % BITS_IN_EWORD));
236
237         /* check if we just completed a stream of 1s */
238         if (self->buffer[self->buffer_size - 1] == (eword_t)(~0)) {
239                 self->buffer[--self->buffer_size] = 0;
240                 rlw_set_literal_words(self->rlw,
241                         rlw_get_literal_words(self->rlw) - 1);
242                 add_empty_word(self, 1);
243         }
244 }
245
246 void ewah_each_bit(struct ewah_bitmap *self, void (*callback)(size_t, void*), void *payload)
247 {
248         size_t pos = 0;
249         size_t pointer = 0;
250         size_t k;
251
252         while (pointer < self->buffer_size) {
253                 eword_t *word = &self->buffer[pointer];
254
255                 if (rlw_get_run_bit(word)) {
256                         size_t len = rlw_get_running_len(word) * BITS_IN_EWORD;
257                         for (k = 0; k < len; ++k, ++pos)
258                                 callback(pos, payload);
259                 } else {
260                         pos += rlw_get_running_len(word) * BITS_IN_EWORD;
261                 }
262
263                 ++pointer;
264
265                 for (k = 0; k < rlw_get_literal_words(word); ++k) {
266                         int c;
267
268                         /* todo: zero count optimization */
269                         for (c = 0; c < BITS_IN_EWORD; ++c, ++pos) {
270                                 if ((self->buffer[pointer] & ((eword_t)1 << c)) != 0)
271                                         callback(pos, payload);
272                         }
273
274                         ++pointer;
275                 }
276         }
277 }
278
279 struct ewah_bitmap *ewah_new(void)
280 {
281         struct ewah_bitmap *self;
282
283         self = xmalloc(sizeof(struct ewah_bitmap));
284         self->alloc_size = 32;
285         ALLOC_ARRAY(self->buffer, self->alloc_size);
286
287         ewah_clear(self);
288         return self;
289 }
290
291 void ewah_clear(struct ewah_bitmap *self)
292 {
293         self->buffer_size = 1;
294         self->buffer[0] = 0;
295         self->bit_size = 0;
296         self->rlw = self->buffer;
297 }
298
299 void ewah_free(struct ewah_bitmap *self)
300 {
301         if (!self)
302                 return;
303
304         if (self->alloc_size)
305                 free(self->buffer);
306
307         free(self);
308 }
309
310 static void read_new_rlw(struct ewah_iterator *it)
311 {
312         const eword_t *word = NULL;
313
314         it->literals = 0;
315         it->compressed = 0;
316
317         while (1) {
318                 word = &it->buffer[it->pointer];
319
320                 it->rl = rlw_get_running_len(word);
321                 it->lw = rlw_get_literal_words(word);
322                 it->b = rlw_get_run_bit(word);
323
324                 if (it->rl || it->lw)
325                         return;
326
327                 if (it->pointer < it->buffer_size - 1) {
328                         it->pointer++;
329                 } else {
330                         it->pointer = it->buffer_size;
331                         return;
332                 }
333         }
334 }
335
336 int ewah_iterator_next(eword_t *next, struct ewah_iterator *it)
337 {
338         if (it->pointer >= it->buffer_size)
339                 return 0;
340
341         if (it->compressed < it->rl) {
342                 it->compressed++;
343                 *next = it->b ? (eword_t)(~0) : 0;
344         } else {
345                 assert(it->literals < it->lw);
346
347                 it->literals++;
348                 it->pointer++;
349
350                 assert(it->pointer < it->buffer_size);
351
352                 *next = it->buffer[it->pointer];
353         }
354
355         if (it->compressed == it->rl && it->literals == it->lw) {
356                 if (++it->pointer < it->buffer_size)
357                         read_new_rlw(it);
358         }
359
360         return 1;
361 }
362
363 void ewah_iterator_init(struct ewah_iterator *it, struct ewah_bitmap *parent)
364 {
365         it->buffer = parent->buffer;
366         it->buffer_size = parent->buffer_size;
367         it->pointer = 0;
368
369         it->lw = 0;
370         it->rl = 0;
371         it->compressed = 0;
372         it->literals = 0;
373         it->b = 0;
374
375         if (it->pointer < it->buffer_size)
376                 read_new_rlw(it);
377 }
378
379 void ewah_not(struct ewah_bitmap *self)
380 {
381         size_t pointer = 0;
382
383         while (pointer < self->buffer_size) {
384                 eword_t *word = &self->buffer[pointer];
385                 size_t literals, k;
386
387                 rlw_xor_run_bit(word);
388                 ++pointer;
389
390                 literals = rlw_get_literal_words(word);
391                 for (k = 0; k < literals; ++k) {
392                         self->buffer[pointer] = ~self->buffer[pointer];
393                         ++pointer;
394                 }
395         }
396 }
397
398 void ewah_xor(
399         struct ewah_bitmap *ewah_i,
400         struct ewah_bitmap *ewah_j,
401         struct ewah_bitmap *out)
402 {
403         struct rlw_iterator rlw_i;
404         struct rlw_iterator rlw_j;
405         size_t literals;
406
407         rlwit_init(&rlw_i, ewah_i);
408         rlwit_init(&rlw_j, ewah_j);
409
410         while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
411                 while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
412                         struct rlw_iterator *prey, *predator;
413                         size_t index;
414                         int negate_words;
415
416                         if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
417                                 prey = &rlw_i;
418                                 predator = &rlw_j;
419                         } else {
420                                 prey = &rlw_j;
421                                 predator = &rlw_i;
422                         }
423
424                         negate_words = !!predator->rlw.running_bit;
425                         index = rlwit_discharge(prey, out,
426                                 predator->rlw.running_len, negate_words);
427
428                         ewah_add_empty_words(out, negate_words,
429                                 predator->rlw.running_len - index);
430
431                         rlwit_discard_first_words(predator,
432                                 predator->rlw.running_len);
433                 }
434
435                 literals = min_size(
436                         rlw_i.rlw.literal_words,
437                         rlw_j.rlw.literal_words);
438
439                 if (literals) {
440                         size_t k;
441
442                         for (k = 0; k < literals; ++k) {
443                                 ewah_add(out,
444                                         rlw_i.buffer[rlw_i.literal_word_start + k] ^
445                                         rlw_j.buffer[rlw_j.literal_word_start + k]
446                                 );
447                         }
448
449                         rlwit_discard_first_words(&rlw_i, literals);
450                         rlwit_discard_first_words(&rlw_j, literals);
451                 }
452         }
453
454         if (rlwit_word_size(&rlw_i) > 0)
455                 rlwit_discharge(&rlw_i, out, ~0, 0);
456         else
457                 rlwit_discharge(&rlw_j, out, ~0, 0);
458
459         out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
460 }
461
462 void ewah_and(
463         struct ewah_bitmap *ewah_i,
464         struct ewah_bitmap *ewah_j,
465         struct ewah_bitmap *out)
466 {
467         struct rlw_iterator rlw_i;
468         struct rlw_iterator rlw_j;
469         size_t literals;
470
471         rlwit_init(&rlw_i, ewah_i);
472         rlwit_init(&rlw_j, ewah_j);
473
474         while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
475                 while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
476                         struct rlw_iterator *prey, *predator;
477
478                         if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
479                                 prey = &rlw_i;
480                                 predator = &rlw_j;
481                         } else {
482                                 prey = &rlw_j;
483                                 predator = &rlw_i;
484                         }
485
486                         if (predator->rlw.running_bit == 0) {
487                                 ewah_add_empty_words(out, 0,
488                                         predator->rlw.running_len);
489                                 rlwit_discard_first_words(prey,
490                                         predator->rlw.running_len);
491                                 rlwit_discard_first_words(predator,
492                                         predator->rlw.running_len);
493                         } else {
494                                 size_t index = rlwit_discharge(prey, out,
495                                         predator->rlw.running_len, 0);
496                                 ewah_add_empty_words(out, 0,
497                                         predator->rlw.running_len - index);
498                                 rlwit_discard_first_words(predator,
499                                         predator->rlw.running_len);
500                         }
501                 }
502
503                 literals = min_size(
504                         rlw_i.rlw.literal_words,
505                         rlw_j.rlw.literal_words);
506
507                 if (literals) {
508                         size_t k;
509
510                         for (k = 0; k < literals; ++k) {
511                                 ewah_add(out,
512                                         rlw_i.buffer[rlw_i.literal_word_start + k] &
513                                         rlw_j.buffer[rlw_j.literal_word_start + k]
514                                 );
515                         }
516
517                         rlwit_discard_first_words(&rlw_i, literals);
518                         rlwit_discard_first_words(&rlw_j, literals);
519                 }
520         }
521
522         if (rlwit_word_size(&rlw_i) > 0)
523                 rlwit_discharge_empty(&rlw_i, out);
524         else
525                 rlwit_discharge_empty(&rlw_j, out);
526
527         out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
528 }
529
530 void ewah_and_not(
531         struct ewah_bitmap *ewah_i,
532         struct ewah_bitmap *ewah_j,
533         struct ewah_bitmap *out)
534 {
535         struct rlw_iterator rlw_i;
536         struct rlw_iterator rlw_j;
537         size_t literals;
538
539         rlwit_init(&rlw_i, ewah_i);
540         rlwit_init(&rlw_j, ewah_j);
541
542         while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
543                 while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
544                         struct rlw_iterator *prey, *predator;
545
546                         if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
547                                 prey = &rlw_i;
548                                 predator = &rlw_j;
549                         } else {
550                                 prey = &rlw_j;
551                                 predator = &rlw_i;
552                         }
553
554                         if ((predator->rlw.running_bit && prey == &rlw_i) ||
555                                 (!predator->rlw.running_bit && prey != &rlw_i)) {
556                                 ewah_add_empty_words(out, 0,
557                                         predator->rlw.running_len);
558                                 rlwit_discard_first_words(prey,
559                                         predator->rlw.running_len);
560                                 rlwit_discard_first_words(predator,
561                                         predator->rlw.running_len);
562                         } else {
563                                 size_t index;
564                                 int negate_words;
565
566                                 negate_words = (&rlw_i != prey);
567                                 index = rlwit_discharge(prey, out,
568                                         predator->rlw.running_len, negate_words);
569                                 ewah_add_empty_words(out, negate_words,
570                                         predator->rlw.running_len - index);
571                                 rlwit_discard_first_words(predator,
572                                         predator->rlw.running_len);
573                         }
574                 }
575
576                 literals = min_size(
577                         rlw_i.rlw.literal_words,
578                         rlw_j.rlw.literal_words);
579
580                 if (literals) {
581                         size_t k;
582
583                         for (k = 0; k < literals; ++k) {
584                                 ewah_add(out,
585                                         rlw_i.buffer[rlw_i.literal_word_start + k] &
586                                         ~(rlw_j.buffer[rlw_j.literal_word_start + k])
587                                 );
588                         }
589
590                         rlwit_discard_first_words(&rlw_i, literals);
591                         rlwit_discard_first_words(&rlw_j, literals);
592                 }
593         }
594
595         if (rlwit_word_size(&rlw_i) > 0)
596                 rlwit_discharge(&rlw_i, out, ~0, 0);
597         else
598                 rlwit_discharge_empty(&rlw_j, out);
599
600         out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
601 }
602
603 void ewah_or(
604         struct ewah_bitmap *ewah_i,
605         struct ewah_bitmap *ewah_j,
606         struct ewah_bitmap *out)
607 {
608         struct rlw_iterator rlw_i;
609         struct rlw_iterator rlw_j;
610         size_t literals;
611
612         rlwit_init(&rlw_i, ewah_i);
613         rlwit_init(&rlw_j, ewah_j);
614
615         while (rlwit_word_size(&rlw_i) > 0 && rlwit_word_size(&rlw_j) > 0) {
616                 while (rlw_i.rlw.running_len > 0 || rlw_j.rlw.running_len > 0) {
617                         struct rlw_iterator *prey, *predator;
618
619                         if (rlw_i.rlw.running_len < rlw_j.rlw.running_len) {
620                                 prey = &rlw_i;
621                                 predator = &rlw_j;
622                         } else {
623                                 prey = &rlw_j;
624                                 predator = &rlw_i;
625                         }
626
627                         if (predator->rlw.running_bit) {
628                                 ewah_add_empty_words(out, 0,
629                                         predator->rlw.running_len);
630                                 rlwit_discard_first_words(prey,
631                                         predator->rlw.running_len);
632                                 rlwit_discard_first_words(predator,
633                                         predator->rlw.running_len);
634                         } else {
635                                 size_t index = rlwit_discharge(prey, out,
636                                         predator->rlw.running_len, 0);
637                                 ewah_add_empty_words(out, 0,
638                                         predator->rlw.running_len - index);
639                                 rlwit_discard_first_words(predator,
640                                         predator->rlw.running_len);
641                         }
642                 }
643
644                 literals = min_size(
645                         rlw_i.rlw.literal_words,
646                         rlw_j.rlw.literal_words);
647
648                 if (literals) {
649                         size_t k;
650
651                         for (k = 0; k < literals; ++k) {
652                                 ewah_add(out,
653                                         rlw_i.buffer[rlw_i.literal_word_start + k] |
654                                         rlw_j.buffer[rlw_j.literal_word_start + k]
655                                 );
656                         }
657
658                         rlwit_discard_first_words(&rlw_i, literals);
659                         rlwit_discard_first_words(&rlw_j, literals);
660                 }
661         }
662
663         if (rlwit_word_size(&rlw_i) > 0)
664                 rlwit_discharge(&rlw_i, out, ~0, 0);
665         else
666                 rlwit_discharge(&rlw_j, out, ~0, 0);
667
668         out->bit_size = max_size(ewah_i->bit_size, ewah_j->bit_size);
669 }
670
671
672 #define BITMAP_POOL_MAX 16
673 static struct ewah_bitmap *bitmap_pool[BITMAP_POOL_MAX];
674 static size_t bitmap_pool_size;
675
676 struct ewah_bitmap *ewah_pool_new(void)
677 {
678         if (bitmap_pool_size)
679                 return bitmap_pool[--bitmap_pool_size];
680
681         return ewah_new();
682 }
683
684 void ewah_pool_free(struct ewah_bitmap *self)
685 {
686         if (self == NULL)
687                 return;
688
689         if (bitmap_pool_size == BITMAP_POOL_MAX ||
690                 self->alloc_size == 0) {
691                 ewah_free(self);
692                 return;
693         }
694
695         ewah_clear(self);
696         bitmap_pool[bitmap_pool_size++] = self;
697 }
698
699 uint32_t ewah_checksum(struct ewah_bitmap *self)
700 {
701         const uint8_t *p = (uint8_t *)self->buffer;
702         uint32_t crc = (uint32_t)self->bit_size;
703         size_t size = self->buffer_size * sizeof(eword_t);
704
705         while (size--)
706                 crc = (crc << 5) - crc + (uint32_t)*p++;
707
708         return crc;
709 }