Merge branch 'nd/attr-pathspec-fix' into maint
[git] / list-objects-filter.c
1 #include "cache.h"
2 #include "dir.h"
3 #include "tag.h"
4 #include "commit.h"
5 #include "tree.h"
6 #include "blob.h"
7 #include "diff.h"
8 #include "tree-walk.h"
9 #include "revision.h"
10 #include "list-objects.h"
11 #include "list-objects-filter.h"
12 #include "list-objects-filter-options.h"
13 #include "oidset.h"
14 #include "object-store.h"
15
16 /* Remember to update object flag allocation in object.h */
17 /*
18  * FILTER_SHOWN_BUT_REVISIT -- we set this bit on tree objects
19  * that have been shown, but should be revisited if they appear
20  * in the traversal (until we mark it SEEN).  This is a way to
21  * let us silently de-dup calls to show() in the caller.  This
22  * is subtly different from the "revision.h:SHOWN" and the
23  * "sha1-name.c:ONELINE_SEEN" bits.  And also different from
24  * the non-de-dup usage in pack-bitmap.c
25  */
26 #define FILTER_SHOWN_BUT_REVISIT (1<<21)
27
28 /*
29  * A filter for list-objects to omit ALL blobs from the traversal.
30  * And to OPTIONALLY collect a list of the omitted OIDs.
31  */
32 struct filter_blobs_none_data {
33         struct oidset *omits;
34 };
35
36 static enum list_objects_filter_result filter_blobs_none(
37         enum list_objects_filter_situation filter_situation,
38         struct object *obj,
39         const char *pathname,
40         const char *filename,
41         void *filter_data_)
42 {
43         struct filter_blobs_none_data *filter_data = filter_data_;
44
45         switch (filter_situation) {
46         default:
47                 die("unknown filter_situation");
48                 return LOFR_ZERO;
49
50         case LOFS_BEGIN_TREE:
51                 assert(obj->type == OBJ_TREE);
52                 /* always include all tree objects */
53                 return LOFR_MARK_SEEN | LOFR_DO_SHOW;
54
55         case LOFS_END_TREE:
56                 assert(obj->type == OBJ_TREE);
57                 return LOFR_ZERO;
58
59         case LOFS_BLOB:
60                 assert(obj->type == OBJ_BLOB);
61                 assert((obj->flags & SEEN) == 0);
62
63                 if (filter_data->omits)
64                         oidset_insert(filter_data->omits, &obj->oid);
65                 return LOFR_MARK_SEEN; /* but not LOFR_DO_SHOW (hard omit) */
66         }
67 }
68
69 static void *filter_blobs_none__init(
70         struct oidset *omitted,
71         struct list_objects_filter_options *filter_options,
72         filter_object_fn *filter_fn,
73         filter_free_fn *filter_free_fn)
74 {
75         struct filter_blobs_none_data *d = xcalloc(1, sizeof(*d));
76         d->omits = omitted;
77
78         *filter_fn = filter_blobs_none;
79         *filter_free_fn = free;
80         return d;
81 }
82
83 /*
84  * A filter for list-objects to omit large blobs.
85  * And to OPTIONALLY collect a list of the omitted OIDs.
86  */
87 struct filter_blobs_limit_data {
88         struct oidset *omits;
89         unsigned long max_bytes;
90 };
91
92 static enum list_objects_filter_result filter_blobs_limit(
93         enum list_objects_filter_situation filter_situation,
94         struct object *obj,
95         const char *pathname,
96         const char *filename,
97         void *filter_data_)
98 {
99         struct filter_blobs_limit_data *filter_data = filter_data_;
100         unsigned long object_length;
101         enum object_type t;
102
103         switch (filter_situation) {
104         default:
105                 die("unknown filter_situation");
106                 return LOFR_ZERO;
107
108         case LOFS_BEGIN_TREE:
109                 assert(obj->type == OBJ_TREE);
110                 /* always include all tree objects */
111                 return LOFR_MARK_SEEN | LOFR_DO_SHOW;
112
113         case LOFS_END_TREE:
114                 assert(obj->type == OBJ_TREE);
115                 return LOFR_ZERO;
116
117         case LOFS_BLOB:
118                 assert(obj->type == OBJ_BLOB);
119                 assert((obj->flags & SEEN) == 0);
120
121                 t = oid_object_info(the_repository, &obj->oid, &object_length);
122                 if (t != OBJ_BLOB) { /* probably OBJ_NONE */
123                         /*
124                          * We DO NOT have the blob locally, so we cannot
125                          * apply the size filter criteria.  Be conservative
126                          * and force show it (and let the caller deal with
127                          * the ambiguity).
128                          */
129                         goto include_it;
130                 }
131
132                 if (object_length < filter_data->max_bytes)
133                         goto include_it;
134
135                 if (filter_data->omits)
136                         oidset_insert(filter_data->omits, &obj->oid);
137                 return LOFR_MARK_SEEN; /* but not LOFR_DO_SHOW (hard omit) */
138         }
139
140 include_it:
141         if (filter_data->omits)
142                 oidset_remove(filter_data->omits, &obj->oid);
143         return LOFR_MARK_SEEN | LOFR_DO_SHOW;
144 }
145
146 static void *filter_blobs_limit__init(
147         struct oidset *omitted,
148         struct list_objects_filter_options *filter_options,
149         filter_object_fn *filter_fn,
150         filter_free_fn *filter_free_fn)
151 {
152         struct filter_blobs_limit_data *d = xcalloc(1, sizeof(*d));
153         d->omits = omitted;
154         d->max_bytes = filter_options->blob_limit_value;
155
156         *filter_fn = filter_blobs_limit;
157         *filter_free_fn = free;
158         return d;
159 }
160
161 /*
162  * A filter driven by a sparse-checkout specification to only
163  * include blobs that a sparse checkout would populate.
164  *
165  * The sparse-checkout spec can be loaded from a blob with the
166  * given OID or from a local pathname.  We allow an OID because
167  * the repo may be bare or we may be doing the filtering on the
168  * server.
169  */
170 struct frame {
171         /*
172          * defval is the usual default include/exclude value that
173          * should be inherited as we recurse into directories based
174          * upon pattern matching of the directory itself or of a
175          * containing directory.
176          */
177         int defval;
178
179         /*
180          * 1 if the directory (recursively) contains any provisionally
181          * omitted objects.
182          *
183          * 0 if everything (recursively) contained in this directory
184          * has been explicitly included (SHOWN) in the result and
185          * the directory may be short-cut later in the traversal.
186          */
187         unsigned child_prov_omit : 1;
188 };
189
190 struct filter_sparse_data {
191         struct oidset *omits;
192         struct exclude_list el;
193
194         size_t nr, alloc;
195         struct frame *array_frame;
196 };
197
198 static enum list_objects_filter_result filter_sparse(
199         enum list_objects_filter_situation filter_situation,
200         struct object *obj,
201         const char *pathname,
202         const char *filename,
203         void *filter_data_)
204 {
205         struct filter_sparse_data *filter_data = filter_data_;
206         int val, dtype;
207         struct frame *frame;
208
209         switch (filter_situation) {
210         default:
211                 die("unknown filter_situation");
212                 return LOFR_ZERO;
213
214         case LOFS_BEGIN_TREE:
215                 assert(obj->type == OBJ_TREE);
216                 dtype = DT_DIR;
217                 val = is_excluded_from_list(pathname, strlen(pathname),
218                                             filename, &dtype, &filter_data->el,
219                                             &the_index);
220                 if (val < 0)
221                         val = filter_data->array_frame[filter_data->nr].defval;
222
223                 ALLOC_GROW(filter_data->array_frame, filter_data->nr + 1,
224                            filter_data->alloc);
225                 filter_data->nr++;
226                 filter_data->array_frame[filter_data->nr].defval = val;
227                 filter_data->array_frame[filter_data->nr].child_prov_omit = 0;
228
229                 /*
230                  * A directory with this tree OID may appear in multiple
231                  * places in the tree. (Think of a directory move or copy,
232                  * with no other changes, so the OID is the same, but the
233                  * full pathnames of objects within this directory are new
234                  * and may match is_excluded() patterns differently.)
235                  * So we cannot mark this directory as SEEN (yet), since
236                  * that will prevent process_tree() from revisiting this
237                  * tree object with other pathname prefixes.
238                  *
239                  * Only _DO_SHOW the tree object the first time we visit
240                  * this tree object.
241                  *
242                  * We always show all tree objects.  A future optimization
243                  * may want to attempt to narrow this.
244                  */
245                 if (obj->flags & FILTER_SHOWN_BUT_REVISIT)
246                         return LOFR_ZERO;
247                 obj->flags |= FILTER_SHOWN_BUT_REVISIT;
248                 return LOFR_DO_SHOW;
249
250         case LOFS_END_TREE:
251                 assert(obj->type == OBJ_TREE);
252                 assert(filter_data->nr > 0);
253
254                 frame = &filter_data->array_frame[filter_data->nr];
255                 filter_data->nr--;
256
257                 /*
258                  * Tell our parent directory if any of our children were
259                  * provisionally omitted.
260                  */
261                 filter_data->array_frame[filter_data->nr].child_prov_omit |=
262                         frame->child_prov_omit;
263
264                 /*
265                  * If there are NO provisionally omitted child objects (ALL child
266                  * objects in this folder were INCLUDED), then we can mark the
267                  * folder as SEEN (so we will not have to revisit it again).
268                  */
269                 if (!frame->child_prov_omit)
270                         return LOFR_MARK_SEEN;
271                 return LOFR_ZERO;
272
273         case LOFS_BLOB:
274                 assert(obj->type == OBJ_BLOB);
275                 assert((obj->flags & SEEN) == 0);
276
277                 frame = &filter_data->array_frame[filter_data->nr];
278
279                 dtype = DT_REG;
280                 val = is_excluded_from_list(pathname, strlen(pathname),
281                                             filename, &dtype, &filter_data->el,
282                                             &the_index);
283                 if (val < 0)
284                         val = frame->defval;
285                 if (val > 0) {
286                         if (filter_data->omits)
287                                 oidset_remove(filter_data->omits, &obj->oid);
288                         return LOFR_MARK_SEEN | LOFR_DO_SHOW;
289                 }
290
291                 /*
292                  * Provisionally omit it.  We've already established that
293                  * this pathname is not in the sparse-checkout specification
294                  * with the CURRENT pathname, so we *WANT* to omit this blob.
295                  *
296                  * However, a pathname elsewhere in the tree may also
297                  * reference this same blob, so we cannot reject it yet.
298                  * Leave the LOFR_ bits unset so that if the blob appears
299                  * again in the traversal, we will be asked again.
300                  */
301                 if (filter_data->omits)
302                         oidset_insert(filter_data->omits, &obj->oid);
303
304                 /*
305                  * Remember that at least 1 blob in this tree was
306                  * provisionally omitted.  This prevents us from short
307                  * cutting the tree in future iterations.
308                  */
309                 frame->child_prov_omit = 1;
310                 return LOFR_ZERO;
311         }
312 }
313
314
315 static void filter_sparse_free(void *filter_data)
316 {
317         struct filter_sparse_data *d = filter_data;
318         /* TODO free contents of 'd' */
319         free(d);
320 }
321
322 static void *filter_sparse_oid__init(
323         struct oidset *omitted,
324         struct list_objects_filter_options *filter_options,
325         filter_object_fn *filter_fn,
326         filter_free_fn *filter_free_fn)
327 {
328         struct filter_sparse_data *d = xcalloc(1, sizeof(*d));
329         d->omits = omitted;
330         if (add_excludes_from_blob_to_list(filter_options->sparse_oid_value,
331                                            NULL, 0, &d->el) < 0)
332                 die("could not load filter specification");
333
334         ALLOC_GROW(d->array_frame, d->nr + 1, d->alloc);
335         d->array_frame[d->nr].defval = 0; /* default to include */
336         d->array_frame[d->nr].child_prov_omit = 0;
337
338         *filter_fn = filter_sparse;
339         *filter_free_fn = filter_sparse_free;
340         return d;
341 }
342
343 static void *filter_sparse_path__init(
344         struct oidset *omitted,
345         struct list_objects_filter_options *filter_options,
346         filter_object_fn *filter_fn,
347         filter_free_fn *filter_free_fn)
348 {
349         struct filter_sparse_data *d = xcalloc(1, sizeof(*d));
350         d->omits = omitted;
351         if (add_excludes_from_file_to_list(filter_options->sparse_path_value,
352                                            NULL, 0, &d->el, NULL) < 0)
353                 die("could not load filter specification");
354
355         ALLOC_GROW(d->array_frame, d->nr + 1, d->alloc);
356         d->array_frame[d->nr].defval = 0; /* default to include */
357         d->array_frame[d->nr].child_prov_omit = 0;
358
359         *filter_fn = filter_sparse;
360         *filter_free_fn = filter_sparse_free;
361         return d;
362 }
363
364 typedef void *(*filter_init_fn)(
365         struct oidset *omitted,
366         struct list_objects_filter_options *filter_options,
367         filter_object_fn *filter_fn,
368         filter_free_fn *filter_free_fn);
369
370 /*
371  * Must match "enum list_objects_filter_choice".
372  */
373 static filter_init_fn s_filters[] = {
374         NULL,
375         filter_blobs_none__init,
376         filter_blobs_limit__init,
377         filter_sparse_oid__init,
378         filter_sparse_path__init,
379 };
380
381 void *list_objects_filter__init(
382         struct oidset *omitted,
383         struct list_objects_filter_options *filter_options,
384         filter_object_fn *filter_fn,
385         filter_free_fn *filter_free_fn)
386 {
387         filter_init_fn init_fn;
388
389         assert((sizeof(s_filters) / sizeof(s_filters[0])) == LOFC__COUNT);
390
391         if (filter_options->choice >= LOFC__COUNT)
392                 die("invalid list-objects filter choice: %d",
393                     filter_options->choice);
394
395         init_fn = s_filters[filter_options->choice];
396         if (init_fn)
397                 return init_fn(omitted, filter_options,
398                                filter_fn, filter_free_fn);
399         *filter_fn = NULL;
400         *filter_free_fn = NULL;
401         return NULL;
402 }