protocol-v2 doc: put HTTP headers after request
[git] / upload-pack.c
1 #include "cache.h"
2 #include "config.h"
3 #include "refs.h"
4 #include "pkt-line.h"
5 #include "sideband.h"
6 #include "tag.h"
7 #include "object.h"
8 #include "commit.h"
9 #include "diff.h"
10 #include "revision.h"
11 #include "list-objects.h"
12 #include "run-command.h"
13 #include "connect.h"
14 #include "sigchain.h"
15 #include "version.h"
16 #include "string-list.h"
17 #include "argv-array.h"
18 #include "prio-queue.h"
19 #include "protocol.h"
20 #include "upload-pack.h"
21 #include "serve.h"
22
23 /* Remember to update object flag allocation in object.h */
24 #define THEY_HAVE       (1u << 11)
25 #define OUR_REF         (1u << 12)
26 #define WANTED          (1u << 13)
27 #define COMMON_KNOWN    (1u << 14)
28 #define REACHABLE       (1u << 15)
29
30 #define SHALLOW         (1u << 16)
31 #define NOT_SHALLOW     (1u << 17)
32 #define CLIENT_SHALLOW  (1u << 18)
33 #define HIDDEN_REF      (1u << 19)
34
35 static timestamp_t oldest_have;
36
37 static int deepen_relative;
38 static int multi_ack;
39 static int no_done;
40 static int use_thin_pack, use_ofs_delta, use_include_tag;
41 static int no_progress, daemon_mode;
42 /* Allow specifying sha1 if it is a ref tip. */
43 #define ALLOW_TIP_SHA1  01
44 /* Allow request of a sha1 if it is reachable from a ref (possibly hidden ref). */
45 #define ALLOW_REACHABLE_SHA1    02
46 /* Allow request of any sha1. Implies ALLOW_TIP_SHA1 and ALLOW_REACHABLE_SHA1. */
47 #define ALLOW_ANY_SHA1  07
48 static unsigned int allow_unadvertised_object_request;
49 static int shallow_nr;
50 static struct object_array have_obj;
51 static struct object_array want_obj;
52 static struct object_array extra_edge_obj;
53 static unsigned int timeout;
54 static int keepalive = 5;
55 /* 0 for no sideband,
56  * otherwise maximum packet size (up to 65520 bytes).
57  */
58 static int use_sideband;
59 static int stateless_rpc;
60 static const char *pack_objects_hook;
61
62 static void reset_timeout(void)
63 {
64         alarm(timeout);
65 }
66
67 static void send_client_data(int fd, const char *data, ssize_t sz)
68 {
69         if (use_sideband) {
70                 send_sideband(1, fd, data, sz, use_sideband);
71                 return;
72         }
73         if (fd == 3)
74                 /* emergency quit */
75                 fd = 2;
76         if (fd == 2) {
77                 /* XXX: are we happy to lose stuff here? */
78                 xwrite(fd, data, sz);
79                 return;
80         }
81         write_or_die(fd, data, sz);
82 }
83
84 static int write_one_shallow(const struct commit_graft *graft, void *cb_data)
85 {
86         FILE *fp = cb_data;
87         if (graft->nr_parent == -1)
88                 fprintf(fp, "--shallow %s\n", oid_to_hex(&graft->oid));
89         return 0;
90 }
91
92 static void create_pack_file(void)
93 {
94         struct child_process pack_objects = CHILD_PROCESS_INIT;
95         char data[8193], progress[128];
96         char abort_msg[] = "aborting due to possible repository "
97                 "corruption on the remote side.";
98         int buffered = -1;
99         ssize_t sz;
100         int i;
101         FILE *pipe_fd;
102
103         if (!pack_objects_hook)
104                 pack_objects.git_cmd = 1;
105         else {
106                 argv_array_push(&pack_objects.args, pack_objects_hook);
107                 argv_array_push(&pack_objects.args, "git");
108                 pack_objects.use_shell = 1;
109         }
110
111         if (shallow_nr) {
112                 argv_array_push(&pack_objects.args, "--shallow-file");
113                 argv_array_push(&pack_objects.args, "");
114         }
115         argv_array_push(&pack_objects.args, "pack-objects");
116         argv_array_push(&pack_objects.args, "--revs");
117         if (use_thin_pack)
118                 argv_array_push(&pack_objects.args, "--thin");
119
120         argv_array_push(&pack_objects.args, "--stdout");
121         if (shallow_nr)
122                 argv_array_push(&pack_objects.args, "--shallow");
123         if (!no_progress)
124                 argv_array_push(&pack_objects.args, "--progress");
125         if (use_ofs_delta)
126                 argv_array_push(&pack_objects.args, "--delta-base-offset");
127         if (use_include_tag)
128                 argv_array_push(&pack_objects.args, "--include-tag");
129
130         pack_objects.in = -1;
131         pack_objects.out = -1;
132         pack_objects.err = -1;
133
134         if (start_command(&pack_objects))
135                 die("git upload-pack: unable to fork git-pack-objects");
136
137         pipe_fd = xfdopen(pack_objects.in, "w");
138
139         if (shallow_nr)
140                 for_each_commit_graft(write_one_shallow, pipe_fd);
141
142         for (i = 0; i < want_obj.nr; i++)
143                 fprintf(pipe_fd, "%s\n",
144                         oid_to_hex(&want_obj.objects[i].item->oid));
145         fprintf(pipe_fd, "--not\n");
146         for (i = 0; i < have_obj.nr; i++)
147                 fprintf(pipe_fd, "%s\n",
148                         oid_to_hex(&have_obj.objects[i].item->oid));
149         for (i = 0; i < extra_edge_obj.nr; i++)
150                 fprintf(pipe_fd, "%s\n",
151                         oid_to_hex(&extra_edge_obj.objects[i].item->oid));
152         fprintf(pipe_fd, "\n");
153         fflush(pipe_fd);
154         fclose(pipe_fd);
155
156         /* We read from pack_objects.err to capture stderr output for
157          * progress bar, and pack_objects.out to capture the pack data.
158          */
159
160         while (1) {
161                 struct pollfd pfd[2];
162                 int pe, pu, pollsize;
163                 int ret;
164
165                 reset_timeout();
166
167                 pollsize = 0;
168                 pe = pu = -1;
169
170                 if (0 <= pack_objects.out) {
171                         pfd[pollsize].fd = pack_objects.out;
172                         pfd[pollsize].events = POLLIN;
173                         pu = pollsize;
174                         pollsize++;
175                 }
176                 if (0 <= pack_objects.err) {
177                         pfd[pollsize].fd = pack_objects.err;
178                         pfd[pollsize].events = POLLIN;
179                         pe = pollsize;
180                         pollsize++;
181                 }
182
183                 if (!pollsize)
184                         break;
185
186                 ret = poll(pfd, pollsize,
187                         keepalive < 0 ? -1 : 1000 * keepalive);
188
189                 if (ret < 0) {
190                         if (errno != EINTR) {
191                                 error_errno("poll failed, resuming");
192                                 sleep(1);
193                         }
194                         continue;
195                 }
196                 if (0 <= pe && (pfd[pe].revents & (POLLIN|POLLHUP))) {
197                         /* Status ready; we ship that in the side-band
198                          * or dump to the standard error.
199                          */
200                         sz = xread(pack_objects.err, progress,
201                                   sizeof(progress));
202                         if (0 < sz)
203                                 send_client_data(2, progress, sz);
204                         else if (sz == 0) {
205                                 close(pack_objects.err);
206                                 pack_objects.err = -1;
207                         }
208                         else
209                                 goto fail;
210                         /* give priority to status messages */
211                         continue;
212                 }
213                 if (0 <= pu && (pfd[pu].revents & (POLLIN|POLLHUP))) {
214                         /* Data ready; we keep the last byte to ourselves
215                          * in case we detect broken rev-list, so that we
216                          * can leave the stream corrupted.  This is
217                          * unfortunate -- unpack-objects would happily
218                          * accept a valid packdata with trailing garbage,
219                          * so appending garbage after we pass all the
220                          * pack data is not good enough to signal
221                          * breakage to downstream.
222                          */
223                         char *cp = data;
224                         ssize_t outsz = 0;
225                         if (0 <= buffered) {
226                                 *cp++ = buffered;
227                                 outsz++;
228                         }
229                         sz = xread(pack_objects.out, cp,
230                                   sizeof(data) - outsz);
231                         if (0 < sz)
232                                 ;
233                         else if (sz == 0) {
234                                 close(pack_objects.out);
235                                 pack_objects.out = -1;
236                         }
237                         else
238                                 goto fail;
239                         sz += outsz;
240                         if (1 < sz) {
241                                 buffered = data[sz-1] & 0xFF;
242                                 sz--;
243                         }
244                         else
245                                 buffered = -1;
246                         send_client_data(1, data, sz);
247                 }
248
249                 /*
250                  * We hit the keepalive timeout without saying anything; send
251                  * an empty message on the data sideband just to let the other
252                  * side know we're still working on it, but don't have any data
253                  * yet.
254                  *
255                  * If we don't have a sideband channel, there's no room in the
256                  * protocol to say anything, so those clients are just out of
257                  * luck.
258                  */
259                 if (!ret && use_sideband) {
260                         static const char buf[] = "0005\1";
261                         write_or_die(1, buf, 5);
262                 }
263         }
264
265         if (finish_command(&pack_objects)) {
266                 error("git upload-pack: git-pack-objects died with error.");
267                 goto fail;
268         }
269
270         /* flush the data */
271         if (0 <= buffered) {
272                 data[0] = buffered;
273                 send_client_data(1, data, 1);
274                 fprintf(stderr, "flushed.\n");
275         }
276         if (use_sideband)
277                 packet_flush(1);
278         return;
279
280  fail:
281         send_client_data(3, abort_msg, sizeof(abort_msg));
282         die("git upload-pack: %s", abort_msg);
283 }
284
285 static int got_oid(const char *hex, struct object_id *oid)
286 {
287         struct object *o;
288         int we_knew_they_have = 0;
289
290         if (get_oid_hex(hex, oid))
291                 die("git upload-pack: expected SHA1 object, got '%s'", hex);
292         if (!has_object_file(oid))
293                 return -1;
294
295         o = parse_object(oid);
296         if (!o)
297                 die("oops (%s)", oid_to_hex(oid));
298         if (o->type == OBJ_COMMIT) {
299                 struct commit_list *parents;
300                 struct commit *commit = (struct commit *)o;
301                 if (o->flags & THEY_HAVE)
302                         we_knew_they_have = 1;
303                 else
304                         o->flags |= THEY_HAVE;
305                 if (!oldest_have || (commit->date < oldest_have))
306                         oldest_have = commit->date;
307                 for (parents = commit->parents;
308                      parents;
309                      parents = parents->next)
310                         parents->item->object.flags |= THEY_HAVE;
311         }
312         if (!we_knew_they_have) {
313                 add_object_array(o, NULL, &have_obj);
314                 return 1;
315         }
316         return 0;
317 }
318
319 static int reachable(struct commit *want)
320 {
321         struct prio_queue work = { compare_commits_by_commit_date };
322
323         prio_queue_put(&work, want);
324         while (work.nr) {
325                 struct commit_list *list;
326                 struct commit *commit = prio_queue_get(&work);
327
328                 if (commit->object.flags & THEY_HAVE) {
329                         want->object.flags |= COMMON_KNOWN;
330                         break;
331                 }
332                 if (!commit->object.parsed)
333                         parse_object(&commit->object.oid);
334                 if (commit->object.flags & REACHABLE)
335                         continue;
336                 commit->object.flags |= REACHABLE;
337                 if (commit->date < oldest_have)
338                         continue;
339                 for (list = commit->parents; list; list = list->next) {
340                         struct commit *parent = list->item;
341                         if (!(parent->object.flags & REACHABLE))
342                                 prio_queue_put(&work, parent);
343                 }
344         }
345         want->object.flags |= REACHABLE;
346         clear_commit_marks(want, REACHABLE);
347         clear_prio_queue(&work);
348         return (want->object.flags & COMMON_KNOWN);
349 }
350
351 static int ok_to_give_up(void)
352 {
353         int i;
354
355         if (!have_obj.nr)
356                 return 0;
357
358         for (i = 0; i < want_obj.nr; i++) {
359                 struct object *want = want_obj.objects[i].item;
360
361                 if (want->flags & COMMON_KNOWN)
362                         continue;
363                 want = deref_tag(want, "a want line", 0);
364                 if (!want || want->type != OBJ_COMMIT) {
365                         /* no way to tell if this is reachable by
366                          * looking at the ancestry chain alone, so
367                          * leave a note to ourselves not to worry about
368                          * this object anymore.
369                          */
370                         want_obj.objects[i].item->flags |= COMMON_KNOWN;
371                         continue;
372                 }
373                 if (!reachable((struct commit *)want))
374                         return 0;
375         }
376         return 1;
377 }
378
379 static int get_common_commits(void)
380 {
381         struct object_id oid;
382         char last_hex[GIT_MAX_HEXSZ + 1];
383         int got_common = 0;
384         int got_other = 0;
385         int sent_ready = 0;
386
387         save_commit_buffer = 0;
388
389         for (;;) {
390                 char *line = packet_read_line(0, NULL);
391                 const char *arg;
392
393                 reset_timeout();
394
395                 if (!line) {
396                         if (multi_ack == 2 && got_common
397                             && !got_other && ok_to_give_up()) {
398                                 sent_ready = 1;
399                                 packet_write_fmt(1, "ACK %s ready\n", last_hex);
400                         }
401                         if (have_obj.nr == 0 || multi_ack)
402                                 packet_write_fmt(1, "NAK\n");
403
404                         if (no_done && sent_ready) {
405                                 packet_write_fmt(1, "ACK %s\n", last_hex);
406                                 return 0;
407                         }
408                         if (stateless_rpc)
409                                 exit(0);
410                         got_common = 0;
411                         got_other = 0;
412                         continue;
413                 }
414                 if (skip_prefix(line, "have ", &arg)) {
415                         switch (got_oid(arg, &oid)) {
416                         case -1: /* they have what we do not */
417                                 got_other = 1;
418                                 if (multi_ack && ok_to_give_up()) {
419                                         const char *hex = oid_to_hex(&oid);
420                                         if (multi_ack == 2) {
421                                                 sent_ready = 1;
422                                                 packet_write_fmt(1, "ACK %s ready\n", hex);
423                                         } else
424                                                 packet_write_fmt(1, "ACK %s continue\n", hex);
425                                 }
426                                 break;
427                         default:
428                                 got_common = 1;
429                                 memcpy(last_hex, oid_to_hex(&oid), 41);
430                                 if (multi_ack == 2)
431                                         packet_write_fmt(1, "ACK %s common\n", last_hex);
432                                 else if (multi_ack)
433                                         packet_write_fmt(1, "ACK %s continue\n", last_hex);
434                                 else if (have_obj.nr == 1)
435                                         packet_write_fmt(1, "ACK %s\n", last_hex);
436                                 break;
437                         }
438                         continue;
439                 }
440                 if (!strcmp(line, "done")) {
441                         if (have_obj.nr > 0) {
442                                 if (multi_ack)
443                                         packet_write_fmt(1, "ACK %s\n", last_hex);
444                                 return 0;
445                         }
446                         packet_write_fmt(1, "NAK\n");
447                         return -1;
448                 }
449                 die("git upload-pack: expected SHA1 list, got '%s'", line);
450         }
451 }
452
453 static int is_our_ref(struct object *o)
454 {
455         int allow_hidden_ref = (allow_unadvertised_object_request &
456                         (ALLOW_TIP_SHA1 | ALLOW_REACHABLE_SHA1));
457         return o->flags & ((allow_hidden_ref ? HIDDEN_REF : 0) | OUR_REF);
458 }
459
460 /*
461  * on successful case, it's up to the caller to close cmd->out
462  */
463 static int do_reachable_revlist(struct child_process *cmd,
464                                 struct object_array *src,
465                                 struct object_array *reachable)
466 {
467         static const char *argv[] = {
468                 "rev-list", "--stdin", NULL,
469         };
470         struct object *o;
471         char namebuf[42]; /* ^ + SHA-1 + LF */
472         int i;
473
474         cmd->argv = argv;
475         cmd->git_cmd = 1;
476         cmd->no_stderr = 1;
477         cmd->in = -1;
478         cmd->out = -1;
479
480         /*
481          * If the next rev-list --stdin encounters an unknown commit,
482          * it terminates, which will cause SIGPIPE in the write loop
483          * below.
484          */
485         sigchain_push(SIGPIPE, SIG_IGN);
486
487         if (start_command(cmd))
488                 goto error;
489
490         namebuf[0] = '^';
491         namebuf[GIT_SHA1_HEXSZ + 1] = '\n';
492         for (i = get_max_object_index(); 0 < i; ) {
493                 o = get_indexed_object(--i);
494                 if (!o)
495                         continue;
496                 if (reachable && o->type == OBJ_COMMIT)
497                         o->flags &= ~TMP_MARK;
498                 if (!is_our_ref(o))
499                         continue;
500                 memcpy(namebuf + 1, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ);
501                 if (write_in_full(cmd->in, namebuf, GIT_SHA1_HEXSZ + 2) < 0)
502                         goto error;
503         }
504         namebuf[GIT_SHA1_HEXSZ] = '\n';
505         for (i = 0; i < src->nr; i++) {
506                 o = src->objects[i].item;
507                 if (is_our_ref(o)) {
508                         if (reachable)
509                                 add_object_array(o, NULL, reachable);
510                         continue;
511                 }
512                 if (reachable && o->type == OBJ_COMMIT)
513                         o->flags |= TMP_MARK;
514                 memcpy(namebuf, oid_to_hex(&o->oid), GIT_SHA1_HEXSZ);
515                 if (write_in_full(cmd->in, namebuf, GIT_SHA1_HEXSZ + 1) < 0)
516                         goto error;
517         }
518         close(cmd->in);
519         cmd->in = -1;
520         sigchain_pop(SIGPIPE);
521
522         return 0;
523
524 error:
525         sigchain_pop(SIGPIPE);
526
527         if (cmd->in >= 0)
528                 close(cmd->in);
529         if (cmd->out >= 0)
530                 close(cmd->out);
531         return -1;
532 }
533
534 static int get_reachable_list(struct object_array *src,
535                               struct object_array *reachable)
536 {
537         struct child_process cmd = CHILD_PROCESS_INIT;
538         int i;
539         struct object *o;
540         char namebuf[42]; /* ^ + SHA-1 + LF */
541
542         if (do_reachable_revlist(&cmd, src, reachable) < 0)
543                 return -1;
544
545         while ((i = read_in_full(cmd.out, namebuf, 41)) == 41) {
546                 struct object_id sha1;
547
548                 if (namebuf[40] != '\n' || get_oid_hex(namebuf, &sha1))
549                         break;
550
551                 o = lookup_object(sha1.hash);
552                 if (o && o->type == OBJ_COMMIT) {
553                         o->flags &= ~TMP_MARK;
554                 }
555         }
556         for (i = get_max_object_index(); 0 < i; i--) {
557                 o = get_indexed_object(i - 1);
558                 if (o && o->type == OBJ_COMMIT &&
559                     (o->flags & TMP_MARK)) {
560                         add_object_array(o, NULL, reachable);
561                                 o->flags &= ~TMP_MARK;
562                 }
563         }
564         close(cmd.out);
565
566         if (finish_command(&cmd))
567                 return -1;
568
569         return 0;
570 }
571
572 static int has_unreachable(struct object_array *src)
573 {
574         struct child_process cmd = CHILD_PROCESS_INIT;
575         char buf[1];
576         int i;
577
578         if (do_reachable_revlist(&cmd, src, NULL) < 0)
579                 return 1;
580
581         /*
582          * The commits out of the rev-list are not ancestors of
583          * our ref.
584          */
585         i = read_in_full(cmd.out, buf, 1);
586         if (i)
587                 goto error;
588         close(cmd.out);
589         cmd.out = -1;
590
591         /*
592          * rev-list may have died by encountering a bad commit
593          * in the history, in which case we do want to bail out
594          * even when it showed no commit.
595          */
596         if (finish_command(&cmd))
597                 goto error;
598
599         /* All the non-tip ones are ancestors of what we advertised */
600         return 0;
601
602 error:
603         sigchain_pop(SIGPIPE);
604         if (cmd.out >= 0)
605                 close(cmd.out);
606         return 1;
607 }
608
609 static void check_non_tip(void)
610 {
611         int i;
612
613         /*
614          * In the normal in-process case without
615          * uploadpack.allowReachableSHA1InWant,
616          * non-tip requests can never happen.
617          */
618         if (!stateless_rpc && !(allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1))
619                 goto error;
620         if (!has_unreachable(&want_obj))
621                 /* All the non-tip ones are ancestors of what we advertised */
622                 return;
623
624 error:
625         /* Pick one of them (we know there at least is one) */
626         for (i = 0; i < want_obj.nr; i++) {
627                 struct object *o = want_obj.objects[i].item;
628                 if (!is_our_ref(o))
629                         die("git upload-pack: not our ref %s",
630                             oid_to_hex(&o->oid));
631         }
632 }
633
634 static void send_shallow(struct commit_list *result)
635 {
636         while (result) {
637                 struct object *object = &result->item->object;
638                 if (!(object->flags & (CLIENT_SHALLOW|NOT_SHALLOW))) {
639                         packet_write_fmt(1, "shallow %s",
640                                          oid_to_hex(&object->oid));
641                         register_shallow(&object->oid);
642                         shallow_nr++;
643                 }
644                 result = result->next;
645         }
646 }
647
648 static void send_unshallow(const struct object_array *shallows)
649 {
650         int i;
651
652         for (i = 0; i < shallows->nr; i++) {
653                 struct object *object = shallows->objects[i].item;
654                 if (object->flags & NOT_SHALLOW) {
655                         struct commit_list *parents;
656                         packet_write_fmt(1, "unshallow %s",
657                                          oid_to_hex(&object->oid));
658                         object->flags &= ~CLIENT_SHALLOW;
659                         /*
660                          * We want to _register_ "object" as shallow, but we
661                          * also need to traverse object's parents to deepen a
662                          * shallow clone. Unregister it for now so we can
663                          * parse and add the parents to the want list, then
664                          * re-register it.
665                          */
666                         unregister_shallow(&object->oid);
667                         object->parsed = 0;
668                         parse_commit_or_die((struct commit *)object);
669                         parents = ((struct commit *)object)->parents;
670                         while (parents) {
671                                 add_object_array(&parents->item->object,
672                                                  NULL, &want_obj);
673                                 parents = parents->next;
674                         }
675                         add_object_array(object, NULL, &extra_edge_obj);
676                 }
677                 /* make sure commit traversal conforms to client */
678                 register_shallow(&object->oid);
679         }
680 }
681
682 static void deepen(int depth, int deepen_relative,
683                    struct object_array *shallows)
684 {
685         if (depth == INFINITE_DEPTH && !is_repository_shallow()) {
686                 int i;
687
688                 for (i = 0; i < shallows->nr; i++) {
689                         struct object *object = shallows->objects[i].item;
690                         object->flags |= NOT_SHALLOW;
691                 }
692         } else if (deepen_relative) {
693                 struct object_array reachable_shallows = OBJECT_ARRAY_INIT;
694                 struct commit_list *result;
695
696                 get_reachable_list(shallows, &reachable_shallows);
697                 result = get_shallow_commits(&reachable_shallows,
698                                              depth + 1,
699                                              SHALLOW, NOT_SHALLOW);
700                 send_shallow(result);
701                 free_commit_list(result);
702                 object_array_clear(&reachable_shallows);
703         } else {
704                 struct commit_list *result;
705
706                 result = get_shallow_commits(&want_obj, depth,
707                                              SHALLOW, NOT_SHALLOW);
708                 send_shallow(result);
709                 free_commit_list(result);
710         }
711
712         send_unshallow(shallows);
713 }
714
715 static void deepen_by_rev_list(int ac, const char **av,
716                                struct object_array *shallows)
717 {
718         struct commit_list *result;
719
720         result = get_shallow_commits_by_rev_list(ac, av, SHALLOW, NOT_SHALLOW);
721         send_shallow(result);
722         free_commit_list(result);
723         send_unshallow(shallows);
724 }
725
726 /* Returns 1 if a shallow list is sent or 0 otherwise */
727 static int send_shallow_list(int depth, int deepen_rev_list,
728                              timestamp_t deepen_since,
729                              struct string_list *deepen_not,
730                              struct object_array *shallows)
731 {
732         int ret = 0;
733
734         if (depth > 0 && deepen_rev_list)
735                 die("git upload-pack: deepen and deepen-since (or deepen-not) cannot be used together");
736         if (depth > 0) {
737                 deepen(depth, deepen_relative, shallows);
738                 ret = 1;
739         } else if (deepen_rev_list) {
740                 struct argv_array av = ARGV_ARRAY_INIT;
741                 int i;
742
743                 argv_array_push(&av, "rev-list");
744                 if (deepen_since)
745                         argv_array_pushf(&av, "--max-age=%"PRItime, deepen_since);
746                 if (deepen_not->nr) {
747                         argv_array_push(&av, "--not");
748                         for (i = 0; i < deepen_not->nr; i++) {
749                                 struct string_list_item *s = deepen_not->items + i;
750                                 argv_array_push(&av, s->string);
751                         }
752                         argv_array_push(&av, "--not");
753                 }
754                 for (i = 0; i < want_obj.nr; i++) {
755                         struct object *o = want_obj.objects[i].item;
756                         argv_array_push(&av, oid_to_hex(&o->oid));
757                 }
758                 deepen_by_rev_list(av.argc, av.argv, shallows);
759                 argv_array_clear(&av);
760                 ret = 1;
761         } else {
762                 if (shallows->nr > 0) {
763                         int i;
764                         for (i = 0; i < shallows->nr; i++)
765                                 register_shallow(&shallows->objects[i].item->oid);
766                 }
767         }
768
769         shallow_nr += shallows->nr;
770         return ret;
771 }
772
773 static int process_shallow(const char *line, struct object_array *shallows)
774 {
775         const char *arg;
776         if (skip_prefix(line, "shallow ", &arg)) {
777                 struct object_id oid;
778                 struct object *object;
779                 if (get_oid_hex(arg, &oid))
780                         die("invalid shallow line: %s", line);
781                 object = parse_object(&oid);
782                 if (!object)
783                         return 1;
784                 if (object->type != OBJ_COMMIT)
785                         die("invalid shallow object %s", oid_to_hex(&oid));
786                 if (!(object->flags & CLIENT_SHALLOW)) {
787                         object->flags |= CLIENT_SHALLOW;
788                         add_object_array(object, NULL, shallows);
789                 }
790                 return 1;
791         }
792
793         return 0;
794 }
795
796 static int process_deepen(const char *line, int *depth)
797 {
798         const char *arg;
799         if (skip_prefix(line, "deepen ", &arg)) {
800                 char *end = NULL;
801                 *depth = (int)strtol(arg, &end, 0);
802                 if (!end || *end || *depth <= 0)
803                         die("Invalid deepen: %s", line);
804                 return 1;
805         }
806
807         return 0;
808 }
809
810 static int process_deepen_since(const char *line, timestamp_t *deepen_since, int *deepen_rev_list)
811 {
812         const char *arg;
813         if (skip_prefix(line, "deepen-since ", &arg)) {
814                 char *end = NULL;
815                 *deepen_since = parse_timestamp(arg, &end, 0);
816                 if (!end || *end || !deepen_since ||
817                     /* revisions.c's max_age -1 is special */
818                     *deepen_since == -1)
819                         die("Invalid deepen-since: %s", line);
820                 *deepen_rev_list = 1;
821                 return 1;
822         }
823         return 0;
824 }
825
826 static int process_deepen_not(const char *line, struct string_list *deepen_not, int *deepen_rev_list)
827 {
828         const char *arg;
829         if (skip_prefix(line, "deepen-not ", &arg)) {
830                 char *ref = NULL;
831                 struct object_id oid;
832                 if (expand_ref(arg, strlen(arg), &oid, &ref) != 1)
833                         die("git upload-pack: ambiguous deepen-not: %s", line);
834                 string_list_append(deepen_not, ref);
835                 free(ref);
836                 *deepen_rev_list = 1;
837                 return 1;
838         }
839         return 0;
840 }
841
842 static void receive_needs(void)
843 {
844         struct object_array shallows = OBJECT_ARRAY_INIT;
845         struct string_list deepen_not = STRING_LIST_INIT_DUP;
846         int depth = 0;
847         int has_non_tip = 0;
848         timestamp_t deepen_since = 0;
849         int deepen_rev_list = 0;
850
851         shallow_nr = 0;
852         for (;;) {
853                 struct object *o;
854                 const char *features;
855                 struct object_id oid_buf;
856                 char *line = packet_read_line(0, NULL);
857                 const char *arg;
858
859                 reset_timeout();
860                 if (!line)
861                         break;
862
863                 if (process_shallow(line, &shallows))
864                         continue;
865                 if (process_deepen(line, &depth))
866                         continue;
867                 if (process_deepen_since(line, &deepen_since, &deepen_rev_list))
868                         continue;
869                 if (process_deepen_not(line, &deepen_not, &deepen_rev_list))
870                         continue;
871
872                 if (!skip_prefix(line, "want ", &arg) ||
873                     get_oid_hex(arg, &oid_buf))
874                         die("git upload-pack: protocol error, "
875                             "expected to get sha, not '%s'", line);
876
877                 features = arg + 40;
878
879                 if (parse_feature_request(features, "deepen-relative"))
880                         deepen_relative = 1;
881                 if (parse_feature_request(features, "multi_ack_detailed"))
882                         multi_ack = 2;
883                 else if (parse_feature_request(features, "multi_ack"))
884                         multi_ack = 1;
885                 if (parse_feature_request(features, "no-done"))
886                         no_done = 1;
887                 if (parse_feature_request(features, "thin-pack"))
888                         use_thin_pack = 1;
889                 if (parse_feature_request(features, "ofs-delta"))
890                         use_ofs_delta = 1;
891                 if (parse_feature_request(features, "side-band-64k"))
892                         use_sideband = LARGE_PACKET_MAX;
893                 else if (parse_feature_request(features, "side-band"))
894                         use_sideband = DEFAULT_PACKET_MAX;
895                 if (parse_feature_request(features, "no-progress"))
896                         no_progress = 1;
897                 if (parse_feature_request(features, "include-tag"))
898                         use_include_tag = 1;
899
900                 o = parse_object(&oid_buf);
901                 if (!o) {
902                         packet_write_fmt(1,
903                                          "ERR upload-pack: not our ref %s",
904                                          oid_to_hex(&oid_buf));
905                         die("git upload-pack: not our ref %s",
906                             oid_to_hex(&oid_buf));
907                 }
908                 if (!(o->flags & WANTED)) {
909                         o->flags |= WANTED;
910                         if (!((allow_unadvertised_object_request & ALLOW_ANY_SHA1) == ALLOW_ANY_SHA1
911                               || is_our_ref(o)))
912                                 has_non_tip = 1;
913                         add_object_array(o, NULL, &want_obj);
914                 }
915         }
916
917         /*
918          * We have sent all our refs already, and the other end
919          * should have chosen out of them. When we are operating
920          * in the stateless RPC mode, however, their choice may
921          * have been based on the set of older refs advertised
922          * by another process that handled the initial request.
923          */
924         if (has_non_tip)
925                 check_non_tip();
926
927         if (!use_sideband && daemon_mode)
928                 no_progress = 1;
929
930         if (depth == 0 && !deepen_rev_list && shallows.nr == 0)
931                 return;
932
933         if (send_shallow_list(depth, deepen_rev_list, deepen_since,
934                               &deepen_not, &shallows))
935                 packet_flush(1);
936         object_array_clear(&shallows);
937 }
938
939 /* return non-zero if the ref is hidden, otherwise 0 */
940 static int mark_our_ref(const char *refname, const char *refname_full,
941                         const struct object_id *oid)
942 {
943         struct object *o = lookup_unknown_object(oid->hash);
944
945         if (ref_is_hidden(refname, refname_full)) {
946                 o->flags |= HIDDEN_REF;
947                 return 1;
948         }
949         o->flags |= OUR_REF;
950         return 0;
951 }
952
953 static int check_ref(const char *refname_full, const struct object_id *oid,
954                      int flag, void *cb_data)
955 {
956         const char *refname = strip_namespace(refname_full);
957
958         mark_our_ref(refname, refname_full, oid);
959         return 0;
960 }
961
962 static void format_symref_info(struct strbuf *buf, struct string_list *symref)
963 {
964         struct string_list_item *item;
965
966         if (!symref->nr)
967                 return;
968         for_each_string_list_item(item, symref)
969                 strbuf_addf(buf, " symref=%s:%s", item->string, (char *)item->util);
970 }
971
972 static int send_ref(const char *refname, const struct object_id *oid,
973                     int flag, void *cb_data)
974 {
975         static const char *capabilities = "multi_ack thin-pack side-band"
976                 " side-band-64k ofs-delta shallow deepen-since deepen-not"
977                 " deepen-relative no-progress include-tag multi_ack_detailed";
978         const char *refname_nons = strip_namespace(refname);
979         struct object_id peeled;
980
981         if (mark_our_ref(refname_nons, refname, oid))
982                 return 0;
983
984         if (capabilities) {
985                 struct strbuf symref_info = STRBUF_INIT;
986
987                 format_symref_info(&symref_info, cb_data);
988                 packet_write_fmt(1, "%s %s%c%s%s%s%s%s agent=%s\n",
989                              oid_to_hex(oid), refname_nons,
990                              0, capabilities,
991                              (allow_unadvertised_object_request & ALLOW_TIP_SHA1) ?
992                                      " allow-tip-sha1-in-want" : "",
993                              (allow_unadvertised_object_request & ALLOW_REACHABLE_SHA1) ?
994                                      " allow-reachable-sha1-in-want" : "",
995                              stateless_rpc ? " no-done" : "",
996                              symref_info.buf,
997                              git_user_agent_sanitized());
998                 strbuf_release(&symref_info);
999         } else {
1000                 packet_write_fmt(1, "%s %s\n", oid_to_hex(oid), refname_nons);
1001         }
1002         capabilities = NULL;
1003         if (!peel_ref(refname, &peeled))
1004                 packet_write_fmt(1, "%s %s^{}\n", oid_to_hex(&peeled), refname_nons);
1005         return 0;
1006 }
1007
1008 static int find_symref(const char *refname, const struct object_id *oid,
1009                        int flag, void *cb_data)
1010 {
1011         const char *symref_target;
1012         struct string_list_item *item;
1013
1014         if ((flag & REF_ISSYMREF) == 0)
1015                 return 0;
1016         symref_target = resolve_ref_unsafe(refname, 0, NULL, &flag);
1017         if (!symref_target || (flag & REF_ISSYMREF) == 0)
1018                 die("'%s' is a symref but it is not?", refname);
1019         item = string_list_append(cb_data, refname);
1020         item->util = xstrdup(symref_target);
1021         return 0;
1022 }
1023
1024 static int upload_pack_config(const char *var, const char *value, void *unused)
1025 {
1026         if (!strcmp("uploadpack.allowtipsha1inwant", var)) {
1027                 if (git_config_bool(var, value))
1028                         allow_unadvertised_object_request |= ALLOW_TIP_SHA1;
1029                 else
1030                         allow_unadvertised_object_request &= ~ALLOW_TIP_SHA1;
1031         } else if (!strcmp("uploadpack.allowreachablesha1inwant", var)) {
1032                 if (git_config_bool(var, value))
1033                         allow_unadvertised_object_request |= ALLOW_REACHABLE_SHA1;
1034                 else
1035                         allow_unadvertised_object_request &= ~ALLOW_REACHABLE_SHA1;
1036         } else if (!strcmp("uploadpack.allowanysha1inwant", var)) {
1037                 if (git_config_bool(var, value))
1038                         allow_unadvertised_object_request |= ALLOW_ANY_SHA1;
1039                 else
1040                         allow_unadvertised_object_request &= ~ALLOW_ANY_SHA1;
1041         } else if (!strcmp("uploadpack.keepalive", var)) {
1042                 keepalive = git_config_int(var, value);
1043                 if (!keepalive)
1044                         keepalive = -1;
1045         } else if (current_config_scope() != CONFIG_SCOPE_REPO) {
1046                 if (!strcmp("uploadpack.packobjectshook", var))
1047                         return git_config_string(&pack_objects_hook, var, value);
1048         }
1049         return parse_hide_refs_config(var, value, "uploadpack");
1050 }
1051
1052 void upload_pack(struct upload_pack_options *options)
1053 {
1054         struct string_list symref = STRING_LIST_INIT_DUP;
1055
1056         stateless_rpc = options->stateless_rpc;
1057         timeout = options->timeout;
1058         daemon_mode = options->daemon_mode;
1059
1060         git_config(upload_pack_config, NULL);
1061
1062         head_ref_namespaced(find_symref, &symref);
1063
1064         if (options->advertise_refs || !stateless_rpc) {
1065                 reset_timeout();
1066                 head_ref_namespaced(send_ref, &symref);
1067                 for_each_namespaced_ref(send_ref, &symref);
1068                 advertise_shallow_grafts(1);
1069                 packet_flush(1);
1070         } else {
1071                 head_ref_namespaced(check_ref, NULL);
1072                 for_each_namespaced_ref(check_ref, NULL);
1073         }
1074         string_list_clear(&symref, 1);
1075         if (options->advertise_refs)
1076                 return;
1077
1078         receive_needs();
1079         if (want_obj.nr) {
1080                 get_common_commits();
1081                 create_pack_file();
1082         }
1083 }
1084
1085 struct upload_pack_data {
1086         struct object_array wants;
1087         struct oid_array haves;
1088
1089         struct object_array shallows;
1090         struct string_list deepen_not;
1091         int depth;
1092         timestamp_t deepen_since;
1093         int deepen_rev_list;
1094         int deepen_relative;
1095
1096         unsigned stateless_rpc : 1;
1097
1098         unsigned use_thin_pack : 1;
1099         unsigned use_ofs_delta : 1;
1100         unsigned no_progress : 1;
1101         unsigned use_include_tag : 1;
1102         unsigned done : 1;
1103 };
1104
1105 static void upload_pack_data_init(struct upload_pack_data *data)
1106 {
1107         struct object_array wants = OBJECT_ARRAY_INIT;
1108         struct oid_array haves = OID_ARRAY_INIT;
1109         struct object_array shallows = OBJECT_ARRAY_INIT;
1110         struct string_list deepen_not = STRING_LIST_INIT_DUP;
1111
1112         memset(data, 0, sizeof(*data));
1113         data->wants = wants;
1114         data->haves = haves;
1115         data->shallows = shallows;
1116         data->deepen_not = deepen_not;
1117 }
1118
1119 static void upload_pack_data_clear(struct upload_pack_data *data)
1120 {
1121         object_array_clear(&data->wants);
1122         oid_array_clear(&data->haves);
1123         object_array_clear(&data->shallows);
1124         string_list_clear(&data->deepen_not, 0);
1125 }
1126
1127 static int parse_want(const char *line)
1128 {
1129         const char *arg;
1130         if (skip_prefix(line, "want ", &arg)) {
1131                 struct object_id oid;
1132                 struct object *o;
1133
1134                 if (get_oid_hex(arg, &oid))
1135                         die("git upload-pack: protocol error, "
1136                             "expected to get oid, not '%s'", line);
1137
1138                 o = parse_object(&oid);
1139                 if (!o) {
1140                         packet_write_fmt(1,
1141                                          "ERR upload-pack: not our ref %s",
1142                                          oid_to_hex(&oid));
1143                         die("git upload-pack: not our ref %s",
1144                             oid_to_hex(&oid));
1145                 }
1146
1147                 if (!(o->flags & WANTED)) {
1148                         o->flags |= WANTED;
1149                         add_object_array(o, NULL, &want_obj);
1150                 }
1151
1152                 return 1;
1153         }
1154
1155         return 0;
1156 }
1157
1158 static int parse_have(const char *line, struct oid_array *haves)
1159 {
1160         const char *arg;
1161         if (skip_prefix(line, "have ", &arg)) {
1162                 struct object_id oid;
1163
1164                 if (get_oid_hex(arg, &oid))
1165                         die("git upload-pack: expected SHA1 object, got '%s'", arg);
1166                 oid_array_append(haves, &oid);
1167                 return 1;
1168         }
1169
1170         return 0;
1171 }
1172
1173 static void process_args(struct packet_reader *request,
1174                          struct upload_pack_data *data)
1175 {
1176         while (packet_reader_read(request) != PACKET_READ_FLUSH) {
1177                 const char *arg = request->line;
1178
1179                 /* process want */
1180                 if (parse_want(arg))
1181                         continue;
1182                 /* process have line */
1183                 if (parse_have(arg, &data->haves))
1184                         continue;
1185
1186                 /* process args like thin-pack */
1187                 if (!strcmp(arg, "thin-pack")) {
1188                         use_thin_pack = 1;
1189                         continue;
1190                 }
1191                 if (!strcmp(arg, "ofs-delta")) {
1192                         use_ofs_delta = 1;
1193                         continue;
1194                 }
1195                 if (!strcmp(arg, "no-progress")) {
1196                         no_progress = 1;
1197                         continue;
1198                 }
1199                 if (!strcmp(arg, "include-tag")) {
1200                         use_include_tag = 1;
1201                         continue;
1202                 }
1203                 if (!strcmp(arg, "done")) {
1204                         data->done = 1;
1205                         continue;
1206                 }
1207
1208                 /* Shallow related arguments */
1209                 if (process_shallow(arg, &data->shallows))
1210                         continue;
1211                 if (process_deepen(arg, &data->depth))
1212                         continue;
1213                 if (process_deepen_since(arg, &data->deepen_since,
1214                                          &data->deepen_rev_list))
1215                         continue;
1216                 if (process_deepen_not(arg, &data->deepen_not,
1217                                        &data->deepen_rev_list))
1218                         continue;
1219                 if (!strcmp(arg, "deepen-relative")) {
1220                         data->deepen_relative = 1;
1221                         continue;
1222                 }
1223
1224                 /* ignore unknown lines maybe? */
1225                 die("unexpect line: '%s'", arg);
1226         }
1227 }
1228
1229 static int process_haves(struct oid_array *haves, struct oid_array *common)
1230 {
1231         int i;
1232
1233         /* Process haves */
1234         for (i = 0; i < haves->nr; i++) {
1235                 const struct object_id *oid = &haves->oid[i];
1236                 struct object *o;
1237                 int we_knew_they_have = 0;
1238
1239                 if (!has_object_file(oid))
1240                         continue;
1241
1242                 oid_array_append(common, oid);
1243
1244                 o = parse_object(oid);
1245                 if (!o)
1246                         die("oops (%s)", oid_to_hex(oid));
1247                 if (o->type == OBJ_COMMIT) {
1248                         struct commit_list *parents;
1249                         struct commit *commit = (struct commit *)o;
1250                         if (o->flags & THEY_HAVE)
1251                                 we_knew_they_have = 1;
1252                         else
1253                                 o->flags |= THEY_HAVE;
1254                         if (!oldest_have || (commit->date < oldest_have))
1255                                 oldest_have = commit->date;
1256                         for (parents = commit->parents;
1257                              parents;
1258                              parents = parents->next)
1259                                 parents->item->object.flags |= THEY_HAVE;
1260                 }
1261                 if (!we_knew_they_have)
1262                         add_object_array(o, NULL, &have_obj);
1263         }
1264
1265         return 0;
1266 }
1267
1268 static int send_acks(struct oid_array *acks, struct strbuf *response)
1269 {
1270         int i;
1271
1272         packet_buf_write(response, "acknowledgments\n");
1273
1274         /* Send Acks */
1275         if (!acks->nr)
1276                 packet_buf_write(response, "NAK\n");
1277
1278         for (i = 0; i < acks->nr; i++) {
1279                 packet_buf_write(response, "ACK %s\n",
1280                                  oid_to_hex(&acks->oid[i]));
1281         }
1282
1283         if (ok_to_give_up()) {
1284                 /* Send Ready */
1285                 packet_buf_write(response, "ready\n");
1286                 return 1;
1287         }
1288
1289         return 0;
1290 }
1291
1292 static int process_haves_and_send_acks(struct upload_pack_data *data)
1293 {
1294         struct oid_array common = OID_ARRAY_INIT;
1295         struct strbuf response = STRBUF_INIT;
1296         int ret = 0;
1297
1298         process_haves(&data->haves, &common);
1299         if (data->done) {
1300                 ret = 1;
1301         } else if (send_acks(&common, &response)) {
1302                 packet_buf_delim(&response);
1303                 ret = 1;
1304         } else {
1305                 /* Add Flush */
1306                 packet_buf_flush(&response);
1307                 ret = 0;
1308         }
1309
1310         /* Send response */
1311         write_or_die(1, response.buf, response.len);
1312         strbuf_release(&response);
1313
1314         oid_array_clear(&data->haves);
1315         oid_array_clear(&common);
1316         return ret;
1317 }
1318
1319 static void send_shallow_info(struct upload_pack_data *data)
1320 {
1321         /* No shallow info needs to be sent */
1322         if (!data->depth && !data->deepen_rev_list && !data->shallows.nr &&
1323             !is_repository_shallow())
1324                 return;
1325
1326         packet_write_fmt(1, "shallow-info\n");
1327
1328         if (!send_shallow_list(data->depth, data->deepen_rev_list,
1329                                data->deepen_since, &data->deepen_not,
1330                                &data->shallows) && is_repository_shallow())
1331                 deepen(INFINITE_DEPTH, data->deepen_relative, &data->shallows);
1332
1333         packet_delim(1);
1334 }
1335
1336 enum fetch_state {
1337         FETCH_PROCESS_ARGS = 0,
1338         FETCH_SEND_ACKS,
1339         FETCH_SEND_PACK,
1340         FETCH_DONE,
1341 };
1342
1343 int upload_pack_v2(struct repository *r, struct argv_array *keys,
1344                    struct packet_reader *request)
1345 {
1346         enum fetch_state state = FETCH_PROCESS_ARGS;
1347         struct upload_pack_data data;
1348
1349         upload_pack_data_init(&data);
1350         use_sideband = LARGE_PACKET_MAX;
1351
1352         while (state != FETCH_DONE) {
1353                 switch (state) {
1354                 case FETCH_PROCESS_ARGS:
1355                         process_args(request, &data);
1356
1357                         if (!want_obj.nr) {
1358                                 /*
1359                                  * Request didn't contain any 'want' lines,
1360                                  * guess they didn't want anything.
1361                                  */
1362                                 state = FETCH_DONE;
1363                         } else if (data.haves.nr) {
1364                                 /*
1365                                  * Request had 'have' lines, so lets ACK them.
1366                                  */
1367                                 state = FETCH_SEND_ACKS;
1368                         } else {
1369                                 /*
1370                                  * Request had 'want's but no 'have's so we can
1371                                  * immedietly go to construct and send a pack.
1372                                  */
1373                                 state = FETCH_SEND_PACK;
1374                         }
1375                         break;
1376                 case FETCH_SEND_ACKS:
1377                         if (process_haves_and_send_acks(&data))
1378                                 state = FETCH_SEND_PACK;
1379                         else
1380                                 state = FETCH_DONE;
1381                         break;
1382                 case FETCH_SEND_PACK:
1383                         send_shallow_info(&data);
1384
1385                         packet_write_fmt(1, "packfile\n");
1386                         create_pack_file();
1387                         state = FETCH_DONE;
1388                         break;
1389                 case FETCH_DONE:
1390                         continue;
1391                 }
1392         }
1393
1394         upload_pack_data_clear(&data);
1395         return 0;
1396 }
1397
1398 int upload_pack_advertise(struct repository *r,
1399                           struct strbuf *value)
1400 {
1401         if (value)
1402                 strbuf_addstr(value, "shallow");
1403         return 1;
1404 }