3 test_description='git rev-list using object filtering'
5 GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
6 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
10 # Test the blob:none filter.
12 test_expect_success 'setup r1' '
13 echo "{print \$1}" >print_1.awk &&
14 echo "{print \$2}" >print_2.awk &&
19 echo "This is file: $n" > r1/file.$n
21 git -C r1 commit -m "$n"
25 test_expect_success 'verify blob:none omits all 5 blobs' '
26 git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
28 awk -f print_2.awk ls_files_result |
31 git -C r1 rev-list --quiet --objects --filter-print-omitted \
32 --filter=blob:none HEAD >revs &&
33 awk -f print_1.awk revs |
37 test_cmp expected observed
40 test_expect_success 'specify blob explicitly prevents filtering' '
41 file_3=$(git -C r1 ls-files -s file.3 |
42 awk -f print_2.awk) &&
44 file_4=$(git -C r1 ls-files -s file.4 |
45 awk -f print_2.awk) &&
47 git -C r1 rev-list --objects --filter=blob:none HEAD $file_3 >observed &&
48 grep "$file_3" observed &&
49 ! grep "$file_4" observed
52 test_expect_success 'verify emitted+omitted == all' '
53 git -C r1 rev-list --objects HEAD >revs &&
54 awk -f print_1.awk revs |
57 git -C r1 rev-list --objects --filter-print-omitted --filter=blob:none \
59 awk -f print_1.awk revs |
63 test_cmp expected observed
67 # Test blob:limit=<n>[kmg] filter.
68 # We boundary test around the size parameter. The filter is strictly less than
69 # the value, so size 500 and 1000 should have the same results, but 1001 should
72 test_expect_success 'setup r2' '
76 printf "%"$n"s" X > r2/large.$n
77 git -C r2 add large.$n
78 git -C r2 commit -m "$n"
82 test_expect_success 'verify blob:limit=500 omits all blobs' '
83 git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
84 awk -f print_2.awk ls_files_result |
87 git -C r2 rev-list --quiet --objects --filter-print-omitted \
88 --filter=blob:limit=500 HEAD >revs &&
89 awk -f print_1.awk revs |
93 test_cmp expected observed
96 test_expect_success 'verify emitted+omitted == all' '
97 git -C r2 rev-list --objects HEAD >revs &&
98 awk -f print_1.awk revs |
101 git -C r2 rev-list --objects --filter-print-omitted \
102 --filter=blob:limit=500 HEAD >revs &&
103 awk -f print_1.awk revs |
107 test_cmp expected observed
110 test_expect_success 'verify blob:limit=1000' '
111 git -C r2 ls-files -s large.1000 large.10000 >ls_files_result &&
112 awk -f print_2.awk ls_files_result |
115 git -C r2 rev-list --quiet --objects --filter-print-omitted \
116 --filter=blob:limit=1000 HEAD >revs &&
117 awk -f print_1.awk revs |
121 test_cmp expected observed
124 test_expect_success 'verify blob:limit=1001' '
125 git -C r2 ls-files -s large.10000 >ls_files_result &&
126 awk -f print_2.awk ls_files_result |
129 git -C r2 rev-list --quiet --objects --filter-print-omitted \
130 --filter=blob:limit=1001 HEAD >revs &&
131 awk -f print_1.awk revs |
135 test_cmp expected observed
138 test_expect_success 'verify blob:limit=1k' '
139 git -C r2 ls-files -s large.10000 >ls_files_result &&
140 awk -f print_2.awk ls_files_result |
143 git -C r2 rev-list --quiet --objects --filter-print-omitted \
144 --filter=blob:limit=1k HEAD >revs &&
145 awk -f print_1.awk revs |
149 test_cmp expected observed
152 test_expect_success 'verify blob:limit=1m' '
153 git -C r2 rev-list --quiet --objects --filter-print-omitted \
154 --filter=blob:limit=1m HEAD >revs &&
155 awk -f print_1.awk revs |
159 test_must_be_empty observed
162 # Test sparse:path=<path> filter.
164 # NOTE: sparse:path filter support has been dropped for security reasons,
165 # so the tests have been changed to make sure that using it fails.
167 # Use a local file containing a sparse-checkout specification to filter
168 # out blobs not required for the corresponding sparse-checkout. We do not
169 # require sparse-checkout to actually be enabled.
171 test_expect_success 'setup r3' '
174 for n in sparse1 sparse2
176 echo "This is file: $n" > r3/$n
178 echo "This is file: dir1/$n" > r3/dir1/$n
179 git -C r3 add dir1/$n
181 git -C r3 commit -m "sparse" &&
182 echo dir1/ >pattern1 &&
183 echo sparse1 >pattern2
186 test_expect_success 'verify sparse:path=pattern1 fails' '
187 test_must_fail git -C r3 rev-list --quiet --objects \
188 --filter-print-omitted --filter=sparse:path=../pattern1 HEAD
191 test_expect_success 'verify sparse:path=pattern2 fails' '
192 test_must_fail git -C r3 rev-list --quiet --objects \
193 --filter-print-omitted --filter=sparse:path=../pattern2 HEAD
196 # Test sparse:oid=<oid-ish> filter.
197 # Use a blob containing a sparse-checkout specification to filter
198 # out blobs not required for the corresponding sparse-checkout. We do not
199 # require sparse-checkout to actually be enabled.
201 test_expect_success 'setup r3 part 2' '
202 echo dir1/ >r3/pattern &&
203 git -C r3 add pattern &&
204 git -C r3 commit -m "pattern"
207 test_expect_success 'verify sparse:oid=OID omits top-level files' '
208 git -C r3 ls-files -s pattern sparse1 sparse2 >ls_files_result &&
209 awk -f print_2.awk ls_files_result |
212 oid=$(git -C r3 ls-files -s pattern | awk -f print_2.awk) &&
214 git -C r3 rev-list --quiet --objects --filter-print-omitted \
215 --filter=sparse:oid=$oid HEAD >revs &&
216 awk -f print_1.awk revs |
220 test_cmp expected observed
223 test_expect_success 'verify sparse:oid=oid-ish omits top-level files' '
224 git -C r3 ls-files -s pattern sparse1 sparse2 >ls_files_result &&
225 awk -f print_2.awk ls_files_result |
228 git -C r3 rev-list --quiet --objects --filter-print-omitted \
229 --filter=sparse:oid=main:pattern HEAD >revs &&
230 awk -f print_1.awk revs |
234 test_cmp expected observed
237 test_expect_success 'rev-list W/ --missing=print and --missing=allow-any for trees' '
238 TREE=$(git -C r3 rev-parse HEAD:dir1) &&
240 # Create a spare repo because we will be deleting objects from this one.
243 rm r3.b/.git/objects/$(echo $TREE | sed "s|^..|&/|") &&
245 git -C r3.b rev-list --quiet --missing=print --objects HEAD \
246 >missing_objs 2>rev_list_err &&
247 echo "?$TREE" >expected &&
248 test_cmp expected missing_objs &&
250 # do not complain when a missing tree cannot be parsed
251 test_must_be_empty rev_list_err &&
253 git -C r3.b rev-list --missing=allow-any --objects HEAD \
254 >objs 2>rev_list_err &&
256 test_must_be_empty rev_list_err
259 # Test tree:0 filter.
261 test_expect_success 'verify tree:0 includes trees in "filtered" output' '
262 git -C r3 rev-list --quiet --objects --filter-print-omitted \
263 --filter=tree:0 HEAD >revs &&
265 awk -f print_1.awk revs |
267 xargs -n1 git -C r3 cat-file -t >unsorted_filtered_types &&
269 sort -u unsorted_filtered_types >filtered_types &&
270 test_write_lines blob tree >expected &&
271 test_cmp expected filtered_types
274 # Make sure tree:0 does not iterate through any trees.
276 test_expect_success 'verify skipping tree iteration when not collecting omits' '
277 GIT_TRACE=1 git -C r3 rev-list \
278 --objects --filter=tree:0 HEAD 2>filter_trace &&
279 grep "Skipping contents of tree [.][.][.]" filter_trace >actual &&
280 # One line for each commit traversed.
281 test_line_count = 2 actual &&
283 # Make sure no other trees were considered besides the root.
284 ! grep "Skipping contents of tree [^.]" filter_trace &&
286 # Try this again with "combine:". If both sub-filters are skipping
287 # trees, the composite filter should also skip trees. This is not
288 # important unless the user does combine:tree:X+tree:Y or another filter
289 # besides "tree:" is implemented in the future which can skip trees.
290 GIT_TRACE=1 git -C r3 rev-list \
291 --objects --filter=combine:tree:1+tree:3 HEAD 2>filter_trace &&
293 # Only skip the dir1/ tree, which is shared between the two commits.
294 grep "Skipping contents of tree " filter_trace >actual &&
295 test_write_lines "Skipping contents of tree dir1/..." >expected &&
296 test_cmp expected actual
299 # Test tree:# filters.
305 hash=$(git -C r3 rev-parse $commit:$name) &&
306 grep "^$hash $name$" actual
309 test_expect_success 'verify tree:1 includes root trees' '
310 git -C r3 rev-list --objects --filter=tree:1 HEAD >actual &&
312 # We should get two root directories and two commits.
313 expect_has HEAD "" &&
314 expect_has HEAD~1 "" &&
315 test_line_count = 4 actual
318 test_expect_success 'verify tree:2 includes root trees and immediate children' '
319 git -C r3 rev-list --objects --filter=tree:2 HEAD >actual &&
321 expect_has HEAD "" &&
322 expect_has HEAD~1 "" &&
323 expect_has HEAD dir1 &&
324 expect_has HEAD pattern &&
325 expect_has HEAD sparse1 &&
326 expect_has HEAD sparse2 &&
328 # There are also 2 commit objects
329 test_line_count = 8 actual
332 test_expect_success 'verify tree:3 includes everything expected' '
333 git -C r3 rev-list --objects --filter=tree:3 HEAD >actual &&
335 expect_has HEAD "" &&
336 expect_has HEAD~1 "" &&
337 expect_has HEAD dir1 &&
338 expect_has HEAD dir1/sparse1 &&
339 expect_has HEAD dir1/sparse2 &&
340 expect_has HEAD pattern &&
341 expect_has HEAD sparse1 &&
342 expect_has HEAD sparse2 &&
344 # There are also 2 commit objects
345 test_line_count = 10 actual
348 test_expect_success 'combine:... for a simple combination' '
349 git -C r3 rev-list --objects --filter=combine:tree:2+blob:none HEAD \
352 expect_has HEAD "" &&
353 expect_has HEAD~1 "" &&
354 expect_has HEAD dir1 &&
356 # There are also 2 commit objects
357 test_line_count = 5 actual &&
359 cp actual expected &&
361 # Try again using repeated --filter - this is equivalent to a manual
362 # combine with "combine:...+..."
363 git -C r3 rev-list --objects --filter=combine:tree:2 \
364 --filter=blob:none HEAD >actual &&
366 test_cmp expected actual
369 test_expect_success 'combine:... with URL encoding' '
370 git -C r3 rev-list --objects \
371 --filter=combine:tree%3a2+blob:%6Eon%65 HEAD >actual &&
373 expect_has HEAD "" &&
374 expect_has HEAD~1 "" &&
375 expect_has HEAD dir1 &&
377 # There are also 2 commit objects
378 test_line_count = 5 actual
381 expect_invalid_filter_spec () {
385 test_must_fail git -C r3 rev-list --objects --filter="$spec" HEAD \
386 >actual 2>actual_stderr &&
387 test_must_be_empty actual &&
388 test_i18ngrep "$err" actual_stderr
391 test_expect_success 'combine:... while URL-encoding things that should not be' '
392 expect_invalid_filter_spec combine%3Atree:2+blob:none \
393 "invalid filter-spec"
396 test_expect_success 'combine: with nothing after the :' '
397 expect_invalid_filter_spec combine: "expected something after combine:"
400 test_expect_success 'parse error in first sub-filter in combine:' '
401 expect_invalid_filter_spec combine:tree:asdf+blob:none \
402 "expected .tree:<depth>."
405 test_expect_success 'combine:... with non-encoded reserved chars' '
406 expect_invalid_filter_spec combine:tree:2+sparse:@xyz \
407 "must escape char in sub-filter-spec: .@." &&
408 expect_invalid_filter_spec combine:tree:2+sparse:\` \
409 "must escape char in sub-filter-spec: .\`." &&
410 expect_invalid_filter_spec combine:tree:2+sparse:~abc \
411 "must escape char in sub-filter-spec: .\~."
414 test_expect_success 'validate err msg for "combine:<valid-filter>+"' '
415 expect_invalid_filter_spec combine:tree:2+ "expected .tree:<depth>."
418 test_expect_success 'combine:... with edge-case hex digits: Ff Aa 0 9' '
419 git -C r3 rev-list --objects --filter="combine:tree:2+bl%6Fb:n%6fne" \
421 test_line_count = 5 actual &&
422 git -C r3 rev-list --objects --filter="combine:tree%3A2+blob%3anone" \
424 test_line_count = 5 actual &&
425 git -C r3 rev-list --objects --filter="combine:tree:%30" HEAD >actual &&
426 test_line_count = 2 actual &&
427 git -C r3 rev-list --objects --filter="combine:tree:%39+blob:none" \
429 test_line_count = 5 actual
432 test_expect_success 'add sparse pattern blobs whose paths have reserved chars' '
433 cp r3/pattern r3/pattern1+renamed% &&
434 cp r3/pattern "r3/p;at%ter+n" &&
435 cp r3/pattern r3/^~pattern &&
436 git -C r3 add pattern1+renamed% "p;at%ter+n" ^~pattern &&
437 git -C r3 commit -m "add sparse pattern files with reserved chars"
440 test_expect_success 'combine:... with more than two sub-filters' '
441 git -C r3 rev-list --objects \
442 --filter=combine:tree:3+blob:limit=40+sparse:oid=main:pattern \
445 expect_has HEAD "" &&
446 expect_has HEAD~1 "" &&
447 expect_has HEAD~2 "" &&
448 expect_has HEAD dir1 &&
449 expect_has HEAD dir1/sparse1 &&
450 expect_has HEAD dir1/sparse2 &&
452 # Should also have 3 commits
453 test_line_count = 9 actual &&
455 # Try again, this time making sure the last sub-filter is only
459 git -C r3 rev-list --objects \
460 --filter=combine:tree:3+blob:limit=40+sparse:oid=main:pattern1%2brenamed%25 \
462 test_cmp expect actual &&
464 # Use the same composite filter again, but with a pattern file name that
465 # requires encoding multiple characters, and use implicit filter
467 test_when_finished "rm -f trace1" &&
468 GIT_TRACE=$(pwd)/trace1 git -C r3 rev-list --objects \
469 --filter=tree:3 --filter=blob:limit=40 \
470 --filter=sparse:oid="main:p;at%ter+n" \
473 test_cmp expect actual &&
474 grep "Add to combine filter-spec: sparse:oid=main:p%3bat%25ter%2bn" \
477 # Repeat the above test, but this time, the characters to encode are in
478 # the LHS of the combined filter.
479 test_when_finished "rm -f trace2" &&
480 GIT_TRACE=$(pwd)/trace2 git -C r3 rev-list --objects \
481 --filter=sparse:oid=main:^~pattern \
482 --filter=tree:3 --filter=blob:limit=40 \
485 test_cmp expect actual &&
486 grep "Add to combine filter-spec: sparse:oid=main:%5e%7epattern" \
490 # Test provisional omit collection logic with a repo that has objects appearing
491 # at multiple depths - first deeper than the filter's threshold, then shallow.
493 test_expect_success 'setup r4' '
498 echo bar > r4/subdir/bar &&
501 cp -r r4/foo r4/subdir r4/filt &&
503 git -C r4 add foo subdir filt &&
504 git -C r4 commit -m "commit msg"
507 expect_has_with_different_name () {
511 hash=$(git -C $repo rev-parse HEAD:$name) &&
512 ! grep "^$hash $name$" actual &&
513 grep "^$hash " actual &&
514 ! grep "~$hash" actual
517 test_expect_success 'test tree:# filter provisional omit for blob and tree' '
518 git -C r4 rev-list --objects --filter-print-omitted --filter=tree:2 \
520 expect_has_with_different_name r4 filt/foo &&
521 expect_has_with_different_name r4 filt/subdir
524 test_expect_success 'verify skipping tree iteration when collecting omits' '
525 GIT_TRACE=1 git -C r4 rev-list --filter-print-omitted \
526 --objects --filter=tree:0 HEAD 2>filter_trace &&
527 grep "^Skipping contents of tree " filter_trace >actual &&
529 echo "Skipping contents of tree subdir/..." >expect &&
530 test_cmp expect actual
533 test_expect_success 'setup r5' '
535 mkdir -p r5/subdir &&
537 echo 1 >r5/short-root &&
538 echo 12345 >r5/long-root &&
539 echo a >r5/subdir/short-subdir &&
540 echo abcde >r5/subdir/long-subdir &&
542 git -C r5 add short-root long-root subdir &&
543 git -C r5 commit -m "commit msg"
546 test_expect_success 'verify collecting omits in combined: filter' '
547 # Note that this test guards against the naive implementation of simply
548 # giving both filters the same "omits" set and expecting it to
549 # automatically merge them.
550 git -C r5 rev-list --objects --quiet --filter-print-omitted \
551 --filter=combine:tree:2+blob:limit=3 HEAD >actual &&
553 # Expect 0 trees/commits, 3 blobs omitted (all blobs except short-root)
554 omitted_1=$(echo 12345 | git hash-object --stdin) &&
555 omitted_2=$(echo a | git hash-object --stdin) &&
556 omitted_3=$(echo abcde | git hash-object --stdin) &&
558 grep ~$omitted_1 actual &&
559 grep ~$omitted_2 actual &&
560 grep ~$omitted_3 actual &&
561 test_line_count = 3 actual
564 # Test tree:<depth> where a tree is iterated to twice - once where a subentry is
565 # too deep to be included, and again where the blob inside it is shallow enough
566 # to be included. This makes sure we don't use LOFR_MARK_SEEN incorrectly (we
567 # can't use it because a tree can be iterated over again at a lower depth).
569 test_expect_success 'tree:<depth> where we iterate over tree at two levels' '
572 mkdir -p r5/a/subdir/b &&
573 echo foo > r5/a/subdir/b/foo &&
575 mkdir -p r5/subdir/b &&
576 echo foo > r5/subdir/b/foo &&
578 git -C r5 add a subdir &&
579 git -C r5 commit -m "commit msg" &&
581 git -C r5 rev-list --objects --filter=tree:4 HEAD >actual &&
582 expect_has_with_different_name r5 a/subdir/b/foo
585 test_expect_success 'tree:<depth> which filters out blob but given as arg' '
586 blob_hash=$(git -C r4 rev-parse HEAD:subdir/bar) &&
588 git -C r4 rev-list --objects --filter=tree:1 HEAD $blob_hash >actual &&
589 grep ^$blob_hash actual
592 # Delete some loose objects and use rev-list, but WITHOUT any filtering.
593 # This models previously omitted objects that we did not receive.
595 test_expect_success 'rev-list W/ --missing=print' '
596 git -C r1 ls-files -s file.1 file.2 file.3 file.4 file.5 \
598 awk -f print_2.awk ls_files_result |
601 for id in `cat expected | sed "s|..|&/|"`
603 rm r1/.git/objects/$id
606 git -C r1 rev-list --quiet --missing=print --objects HEAD >revs &&
607 awk -f print_1.awk revs |
611 test_cmp expected observed
614 test_expect_success 'rev-list W/O --missing fails' '
615 test_must_fail git -C r1 rev-list --quiet --objects HEAD
618 test_expect_success 'rev-list W/ missing=allow-any' '
619 git -C r1 rev-list --quiet --missing=allow-any --objects HEAD
622 # Test expansion of filter specs.
624 test_expect_success 'expand blob limit in protocol' '
625 git -C r2 config --local uploadpack.allowfilter 1 &&
626 GIT_TRACE_PACKET="$(pwd)/trace" git -c protocol.version=2 clone \
627 --filter=blob:limit=1k "file://$(pwd)/r2" limit &&
628 ! grep "blob:limit=1k" trace &&
629 grep "blob:limit=1024" trace