Merge branch 'jt/packfile-as-uri-doc'
[git] / t / t4216-log-bloom.sh
1 #!/bin/sh
2
3 test_description='git log for a path with Bloom filters'
4 GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME=main
5 export GIT_TEST_DEFAULT_INITIAL_BRANCH_NAME
6
7 . ./test-lib.sh
8
9 GIT_TEST_COMMIT_GRAPH=0
10 GIT_TEST_COMMIT_GRAPH_CHANGED_PATHS=0
11
12 test_expect_success 'setup test - repo, commits, commit graph, log outputs' '
13         git init &&
14         mkdir A A/B A/B/C &&
15         test_commit c1 A/file1 &&
16         test_commit c2 A/B/file2 &&
17         test_commit c3 A/B/C/file3 &&
18         test_commit c4 A/file1 &&
19         test_commit c5 A/B/file2 &&
20         test_commit c6 A/B/C/file3 &&
21         test_commit c7 A/file1 &&
22         test_commit c8 A/B/file2 &&
23         test_commit c9 A/B/C/file3 &&
24         test_commit c10 file_to_be_deleted &&
25         git checkout -b side HEAD~4 &&
26         test_commit side-1 file4 &&
27         git checkout main &&
28         git merge side &&
29         test_commit c11 file5 &&
30         mv file5 file5_renamed &&
31         git add file5_renamed &&
32         git commit -m "rename" &&
33         rm file_to_be_deleted &&
34         git add . &&
35         git commit -m "file removed" &&
36         git commit --allow-empty -m "empty" &&
37         git commit-graph write --reachable --changed-paths &&
38
39         test_oid_cache <<-EOF
40         oid_version sha1:1
41         oid_version sha256:2
42         EOF
43 '
44
45 graph_read_expect () {
46         NUM_CHUNKS=5
47         cat >expect <<- EOF
48         header: 43475048 1 $(test_oid oid_version) $NUM_CHUNKS 0
49         num_commits: $1
50         chunks: oid_fanout oid_lookup commit_metadata bloom_indexes bloom_data
51         EOF
52         test-tool read-graph >actual &&
53         test_cmp expect actual
54 }
55
56 test_expect_success 'commit-graph write wrote out the bloom chunks' '
57         graph_read_expect 16
58 '
59
60 # Turn off any inherited trace2 settings for this test.
61 sane_unset GIT_TRACE2 GIT_TRACE2_PERF GIT_TRACE2_EVENT
62 sane_unset GIT_TRACE2_PERF_BRIEF
63 sane_unset GIT_TRACE2_CONFIG_PARAMS
64
65 setup () {
66         rm -f "$TRASH_DIRECTORY/trace.perf" &&
67         git -c core.commitGraph=false log --pretty="format:%s" $1 >log_wo_bloom &&
68         GIT_TRACE2_PERF="$TRASH_DIRECTORY/trace.perf" git -c core.commitGraph=true log --pretty="format:%s" $1 >log_w_bloom
69 }
70
71 test_bloom_filters_used () {
72         log_args=$1
73         bloom_trace_prefix="statistics:{\"filter_not_present\":${2:-0},\"maybe\""
74         setup "$log_args" &&
75         grep -q "$bloom_trace_prefix" "$TRASH_DIRECTORY/trace.perf" &&
76         test_cmp log_wo_bloom log_w_bloom &&
77     test_path_is_file "$TRASH_DIRECTORY/trace.perf"
78 }
79
80 test_bloom_filters_not_used () {
81         log_args=$1
82         setup "$log_args" &&
83         ! grep -q "statistics:{\"filter_not_present\":" "$TRASH_DIRECTORY/trace.perf" &&
84         test_cmp log_wo_bloom log_w_bloom
85 }
86
87 for path in A A/B A/B/C A/file1 A/B/file2 A/B/C/file3 file4 file5 file5_renamed file_to_be_deleted
88 do
89         for option in "" \
90               "--all" \
91                       "--full-history" \
92                       "--full-history --simplify-merges" \
93                       "--simplify-merges" \
94                       "--simplify-by-decoration" \
95                       "--follow" \
96                       "--first-parent" \
97                       "--topo-order" \
98                       "--date-order" \
99                       "--author-date-order" \
100                       "--ancestry-path side..main"
101         do
102                 test_expect_success "git log option: $option for path: $path" '
103                         test_bloom_filters_used "$option -- $path" &&
104                         test_config commitgraph.readChangedPaths false &&
105                         test_bloom_filters_not_used "$option -- $path"
106                 '
107         done
108 done
109
110 test_expect_success 'git log -- folder works with and without the trailing slash' '
111         test_bloom_filters_used "-- A" &&
112         test_bloom_filters_used "-- A/"
113 '
114
115 test_expect_success 'git log for path that does not exist. ' '
116         test_bloom_filters_used "-- path_does_not_exist"
117 '
118
119 test_expect_success 'git log with --walk-reflogs does not use Bloom filters' '
120         test_bloom_filters_not_used "--walk-reflogs -- A"
121 '
122
123 test_expect_success 'git log -- multiple path specs does not use Bloom filters' '
124         test_bloom_filters_not_used "-- file4 A/file1"
125 '
126
127 test_expect_success 'git log -- "." pathspec at root does not use Bloom filters' '
128         test_bloom_filters_not_used "-- ."
129 '
130
131 test_expect_success 'git log with wildcard that resolves to a single path uses Bloom filters' '
132         test_bloom_filters_used "-- *4" &&
133         test_bloom_filters_used "-- *renamed"
134 '
135
136 test_expect_success 'git log with wildcard that resolves to a multiple paths does not uses Bloom filters' '
137         test_bloom_filters_not_used "-- *" &&
138         test_bloom_filters_not_used "-- file*"
139 '
140
141 test_expect_success 'setup - add commit-graph to the chain without Bloom filters' '
142         test_commit c14 A/anotherFile2 &&
143         test_commit c15 A/B/anotherFile2 &&
144         test_commit c16 A/B/C/anotherFile2 &&
145         git commit-graph write --reachable --split --no-changed-paths &&
146         test_line_count = 2 .git/objects/info/commit-graphs/commit-graph-chain
147 '
148
149 test_expect_success 'use Bloom filters even if the latest graph does not have Bloom filters' '
150         # Ensure that the number of empty filters is equal to the number of
151         # filters in the latest graph layer to prove that they are loaded (and
152         # ignored).
153         test_bloom_filters_used "-- A/B" 3
154 '
155
156 test_expect_success 'setup - add commit-graph to the chain with Bloom filters' '
157         test_commit c17 A/anotherFile3 &&
158         git commit-graph write --reachable --changed-paths --split &&
159         test_line_count = 3 .git/objects/info/commit-graphs/commit-graph-chain
160 '
161
162 test_bloom_filters_used_when_some_filters_are_missing () {
163         log_args=$1
164         bloom_trace_prefix="statistics:{\"filter_not_present\":3,\"maybe\":6,\"definitely_not\":9"
165         setup "$log_args" &&
166         grep -q "$bloom_trace_prefix" "$TRASH_DIRECTORY/trace.perf" &&
167         test_cmp log_wo_bloom log_w_bloom
168 }
169
170 test_expect_success 'Use Bloom filters if they exist in the latest but not all commit graphs in the chain.' '
171         test_bloom_filters_used_when_some_filters_are_missing "-- A/B"
172 '
173
174 test_expect_success 'persist filter settings' '
175         test_when_finished rm -rf .git/objects/info/commit-graph* &&
176         rm -rf .git/objects/info/commit-graph* &&
177         GIT_TRACE2_EVENT="$(pwd)/trace2.txt" \
178                 GIT_TRACE2_EVENT_NESTING=5 \
179                 GIT_TEST_BLOOM_SETTINGS_NUM_HASHES=9 \
180                 GIT_TEST_BLOOM_SETTINGS_BITS_PER_ENTRY=15 \
181                 git commit-graph write --reachable --changed-paths &&
182         grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15,\"max_changed_paths\":512" trace2.txt &&
183         GIT_TRACE2_EVENT="$(pwd)/trace2-auto.txt" \
184                 GIT_TRACE2_EVENT_NESTING=5 \
185                 git commit-graph write --reachable --changed-paths &&
186         grep "{\"hash_version\":1,\"num_hashes\":9,\"bits_per_entry\":15,\"max_changed_paths\":512" trace2-auto.txt
187 '
188
189 test_max_changed_paths () {
190         grep "\"max_changed_paths\":$1" $2
191 }
192
193 test_filter_not_computed () {
194         grep "\"key\":\"filter-not-computed\",\"value\":\"$1\"" $2
195 }
196
197 test_filter_computed () {
198         grep "\"key\":\"filter-computed\",\"value\":\"$1\"" $2
199 }
200
201 test_filter_trunc_empty () {
202         grep "\"key\":\"filter-trunc-empty\",\"value\":\"$1\"" $2
203 }
204
205 test_filter_trunc_large () {
206         grep "\"key\":\"filter-trunc-large\",\"value\":\"$1\"" $2
207 }
208
209 test_expect_success 'correctly report changes over limit' '
210         git init limits &&
211         (
212                 cd limits &&
213                 mkdir d &&
214                 mkdir d/e &&
215
216                 for i in $(test_seq 1 2)
217                 do
218                         printf $i >d/file$i.txt &&
219                         printf $i >d/e/file$i.txt || return 1
220                 done &&
221
222                 mkdir mode &&
223                 printf bash >mode/script.sh &&
224
225                 mkdir foo &&
226                 touch foo/bar &&
227                 touch foo.txt &&
228
229                 git add d foo foo.txt mode &&
230                 git commit -m "files" &&
231
232                 # Commit has 7 file and 4 directory adds
233                 GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=10 \
234                         GIT_TRACE2_EVENT="$(pwd)/trace" \
235                         git commit-graph write --reachable --changed-paths &&
236                 test_max_changed_paths 10 trace &&
237                 test_filter_computed 1 trace &&
238                 test_filter_trunc_large 1 trace &&
239
240                 for path in $(git ls-tree -r --name-only HEAD)
241                 do
242                         git -c commitGraph.readChangedPaths=false log \
243                                 -- $path >expect &&
244                         git log -- $path >actual &&
245                         test_cmp expect actual || return 1
246                 done &&
247
248                 # Make a variety of path changes
249                 printf new1 >d/e/file1.txt &&
250                 printf new2 >d/file2.txt &&
251                 rm d/e/file2.txt &&
252                 rm -r foo &&
253                 printf text >foo &&
254                 mkdir f &&
255                 printf new1 >f/file1.txt &&
256
257                 # including a mode-only change (counts as modified)
258                 git update-index --chmod=+x mode/script.sh &&
259
260                 git add foo d f &&
261                 git commit -m "complicated" &&
262
263                 # start from scratch and rebuild
264                 rm -f .git/objects/info/commit-graph &&
265                 GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=10 \
266                         GIT_TRACE2_EVENT="$(pwd)/trace-edit" \
267                         git commit-graph write --reachable --changed-paths &&
268                 test_max_changed_paths 10 trace-edit &&
269                 test_filter_computed 2 trace-edit &&
270                 test_filter_trunc_large 2 trace-edit &&
271
272                 for path in $(git ls-tree -r --name-only HEAD)
273                 do
274                         git -c commitGraph.readChangedPaths=false log \
275                                 -- $path >expect &&
276                         git log -- $path >actual &&
277                         test_cmp expect actual || return 1
278                 done &&
279
280                 # start from scratch and rebuild
281                 rm -f .git/objects/info/commit-graph &&
282                 GIT_TEST_BLOOM_SETTINGS_MAX_CHANGED_PATHS=11 \
283                         GIT_TRACE2_EVENT="$(pwd)/trace-update" \
284                         git commit-graph write --reachable --changed-paths &&
285                 test_max_changed_paths 11 trace-update &&
286                 test_filter_computed 2 trace-update &&
287                 test_filter_trunc_large 0 trace-update &&
288
289                 for path in $(git ls-tree -r --name-only HEAD)
290                 do
291                         git -c commitGraph.readChangedPaths=false log \
292                                 -- $path >expect &&
293                         git log -- $path >actual &&
294                         test_cmp expect actual || return 1
295                 done
296         )
297 '
298
299 test_expect_success 'correctly report commits with no changed paths' '
300         git init empty &&
301         test_when_finished "rm -fr empty" &&
302         (
303                 cd empty &&
304
305                 git commit --allow-empty -m "initial commit" &&
306
307                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
308                         git commit-graph write --reachable --changed-paths &&
309                 test_filter_computed 1 trace.event &&
310                 test_filter_not_computed 0 trace.event &&
311                 test_filter_trunc_empty 1 trace.event &&
312                 test_filter_trunc_large 0 trace.event
313         )
314 '
315
316 test_expect_success 'Bloom generation is limited by --max-new-filters' '
317         (
318                 cd limits &&
319                 test_commit c2 filter &&
320                 test_commit c3 filter &&
321                 test_commit c4 no-filter &&
322
323                 rm -f trace.event &&
324                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
325                         git commit-graph write --reachable --split=replace \
326                                 --changed-paths --max-new-filters=2 &&
327
328                 test_filter_computed 2 trace.event &&
329                 test_filter_not_computed 3 trace.event &&
330                 test_filter_trunc_empty 0 trace.event &&
331                 test_filter_trunc_large 0 trace.event
332         )
333 '
334
335 test_expect_success 'Bloom generation backfills previously-skipped filters' '
336         # Check specifying commitGraph.maxNewFilters over "git config" works.
337         test_config -C limits commitGraph.maxNewFilters 1 &&
338         (
339                 cd limits &&
340
341                 rm -f trace.event &&
342                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
343                         git commit-graph write --reachable --changed-paths \
344                                 --split=replace &&
345                 test_filter_computed 1 trace.event &&
346                 test_filter_not_computed 4 trace.event &&
347                 test_filter_trunc_empty 0 trace.event &&
348                 test_filter_trunc_large 0 trace.event
349         )
350 '
351
352 test_expect_success '--max-new-filters overrides configuration' '
353         git init override &&
354         test_when_finished "rm -fr override" &&
355         test_config -C override commitGraph.maxNewFilters 2 &&
356         (
357                 cd override &&
358                 test_commit one &&
359                 test_commit two &&
360
361                 rm -f trace.event &&
362                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
363                         git commit-graph write --reachable --changed-paths \
364                                 --max-new-filters=1 &&
365                 test_filter_computed 1 trace.event &&
366                 test_filter_not_computed 1 trace.event &&
367                 test_filter_trunc_empty 0 trace.event &&
368                 test_filter_trunc_large 0 trace.event
369         )
370 '
371
372 test_expect_success 'Bloom generation backfills empty commits' '
373         git init empty &&
374         test_when_finished "rm -fr empty" &&
375         (
376                 cd empty &&
377                 for i in $(test_seq 1 6)
378                 do
379                         git commit --allow-empty -m "$i"
380                 done &&
381
382                 # Generate Bloom filters for empty commits 1-6, two at a time.
383                 for i in $(test_seq 1 3)
384                 do
385                         rm -f trace.event &&
386                         GIT_TRACE2_EVENT="$(pwd)/trace.event" \
387                                 git commit-graph write --reachable \
388                                         --changed-paths --max-new-filters=2 &&
389                         test_filter_computed 2 trace.event &&
390                         test_filter_not_computed 4 trace.event &&
391                         test_filter_trunc_empty 2 trace.event &&
392                         test_filter_trunc_large 0 trace.event
393                 done &&
394
395                 # Finally, make sure that once all commits have filters, that
396                 # none are subsequently recomputed.
397                 rm -f trace.event &&
398                 GIT_TRACE2_EVENT="$(pwd)/trace.event" \
399                         git commit-graph write --reachable \
400                                 --changed-paths --max-new-filters=2 &&
401                 test_filter_computed 0 trace.event &&
402                 test_filter_not_computed 6 trace.event &&
403                 test_filter_trunc_empty 0 trace.event &&
404                 test_filter_trunc_large 0 trace.event
405         )
406 '
407
408 test_done