t2203: abstract away SHA-1-specific constants
[git] / archive-tar.c
1 /*
2  * Copyright (c) 2005, 2006 Rene Scharfe
3  */
4 #include "cache.h"
5 #include "config.h"
6 #include "tar.h"
7 #include "archive.h"
8 #include "streaming.h"
9 #include "run-command.h"
10
11 #define RECORDSIZE      (512)
12 #define BLOCKSIZE       (RECORDSIZE * 20)
13
14 static char block[BLOCKSIZE];
15 static unsigned long offset;
16
17 static int tar_umask = 002;
18
19 static int write_tar_filter_archive(const struct archiver *ar,
20                                     struct archiver_args *args);
21
22 /*
23  * This is the max value that a ustar size header can specify, as it is fixed
24  * at 11 octal digits. POSIX specifies that we switch to extended headers at
25  * this size.
26  *
27  * Likewise for the mtime (which happens to use a buffer of the same size).
28  */
29 #if ULONG_MAX == 0xFFFFFFFF
30 #define USTAR_MAX_SIZE ULONG_MAX
31 #else
32 #define USTAR_MAX_SIZE 077777777777UL
33 #endif
34 #if TIME_MAX == 0xFFFFFFFF
35 #define USTAR_MAX_MTIME TIME_MAX
36 #else
37 #define USTAR_MAX_MTIME 077777777777ULL
38 #endif
39
40 /* writes out the whole block, but only if it is full */
41 static void write_if_needed(void)
42 {
43         if (offset == BLOCKSIZE) {
44                 write_or_die(1, block, BLOCKSIZE);
45                 offset = 0;
46         }
47 }
48
49 /*
50  * queues up writes, so that all our write(2) calls write exactly one
51  * full block; pads writes to RECORDSIZE
52  */
53 static void do_write_blocked(const void *data, unsigned long size)
54 {
55         const char *buf = data;
56
57         if (offset) {
58                 unsigned long chunk = BLOCKSIZE - offset;
59                 if (size < chunk)
60                         chunk = size;
61                 memcpy(block + offset, buf, chunk);
62                 size -= chunk;
63                 offset += chunk;
64                 buf += chunk;
65                 write_if_needed();
66         }
67         while (size >= BLOCKSIZE) {
68                 write_or_die(1, buf, BLOCKSIZE);
69                 size -= BLOCKSIZE;
70                 buf += BLOCKSIZE;
71         }
72         if (size) {
73                 memcpy(block + offset, buf, size);
74                 offset += size;
75         }
76 }
77
78 static void finish_record(void)
79 {
80         unsigned long tail;
81         tail = offset % RECORDSIZE;
82         if (tail)  {
83                 memset(block + offset, 0, RECORDSIZE - tail);
84                 offset += RECORDSIZE - tail;
85         }
86         write_if_needed();
87 }
88
89 static void write_blocked(const void *data, unsigned long size)
90 {
91         do_write_blocked(data, size);
92         finish_record();
93 }
94
95 /*
96  * The end of tar archives is marked by 2*512 nul bytes and after that
97  * follows the rest of the block (if any).
98  */
99 static void write_trailer(void)
100 {
101         int tail = BLOCKSIZE - offset;
102         memset(block + offset, 0, tail);
103         write_or_die(1, block, BLOCKSIZE);
104         if (tail < 2 * RECORDSIZE) {
105                 memset(block, 0, offset);
106                 write_or_die(1, block, BLOCKSIZE);
107         }
108 }
109
110 /*
111  * queues up writes, so that all our write(2) calls write exactly one
112  * full block; pads writes to RECORDSIZE
113  */
114 static int stream_blocked(const struct object_id *oid)
115 {
116         struct git_istream *st;
117         enum object_type type;
118         unsigned long sz;
119         char buf[BLOCKSIZE];
120         ssize_t readlen;
121
122         st = open_istream(oid, &type, &sz, NULL);
123         if (!st)
124                 return error("cannot stream blob %s", oid_to_hex(oid));
125         for (;;) {
126                 readlen = read_istream(st, buf, sizeof(buf));
127                 if (readlen <= 0)
128                         break;
129                 do_write_blocked(buf, readlen);
130         }
131         close_istream(st);
132         if (!readlen)
133                 finish_record();
134         return readlen;
135 }
136
137 /*
138  * pax extended header records have the format "%u %s=%s\n".  %u contains
139  * the size of the whole string (including the %u), the first %s is the
140  * keyword, the second one is the value.  This function constructs such a
141  * string and appends it to a struct strbuf.
142  */
143 static void strbuf_append_ext_header(struct strbuf *sb, const char *keyword,
144                                      const char *value, unsigned int valuelen)
145 {
146         int len, tmp;
147
148         /* "%u %s=%s\n" */
149         len = 1 + 1 + strlen(keyword) + 1 + valuelen + 1;
150         for (tmp = len; tmp > 9; tmp /= 10)
151                 len++;
152
153         strbuf_grow(sb, len);
154         strbuf_addf(sb, "%u %s=", len, keyword);
155         strbuf_add(sb, value, valuelen);
156         strbuf_addch(sb, '\n');
157 }
158
159 /*
160  * Like strbuf_append_ext_header, but for numeric values.
161  */
162 static void strbuf_append_ext_header_uint(struct strbuf *sb,
163                                           const char *keyword,
164                                           uintmax_t value)
165 {
166         char buf[40]; /* big enough for 2^128 in decimal, plus NUL */
167         int len;
168
169         len = xsnprintf(buf, sizeof(buf), "%"PRIuMAX, value);
170         strbuf_append_ext_header(sb, keyword, buf, len);
171 }
172
173 static unsigned int ustar_header_chksum(const struct ustar_header *header)
174 {
175         const unsigned char *p = (const unsigned char *)header;
176         unsigned int chksum = 0;
177         while (p < (const unsigned char *)header->chksum)
178                 chksum += *p++;
179         chksum += sizeof(header->chksum) * ' ';
180         p += sizeof(header->chksum);
181         while (p < (const unsigned char *)header + sizeof(struct ustar_header))
182                 chksum += *p++;
183         return chksum;
184 }
185
186 static size_t get_path_prefix(const char *path, size_t pathlen, size_t maxlen)
187 {
188         size_t i = pathlen;
189         if (i > 1 && path[i - 1] == '/')
190                 i--;
191         if (i > maxlen)
192                 i = maxlen;
193         do {
194                 i--;
195         } while (i > 0 && path[i] != '/');
196         return i;
197 }
198
199 static void prepare_header(struct archiver_args *args,
200                            struct ustar_header *header,
201                            unsigned int mode, unsigned long size)
202 {
203         xsnprintf(header->mode, sizeof(header->mode), "%07o", mode & 07777);
204         xsnprintf(header->size, sizeof(header->size), "%011lo", S_ISREG(mode) ? size : 0);
205         xsnprintf(header->mtime, sizeof(header->mtime), "%011lo", (unsigned long) args->time);
206
207         xsnprintf(header->uid, sizeof(header->uid), "%07o", 0);
208         xsnprintf(header->gid, sizeof(header->gid), "%07o", 0);
209         strlcpy(header->uname, "root", sizeof(header->uname));
210         strlcpy(header->gname, "root", sizeof(header->gname));
211         xsnprintf(header->devmajor, sizeof(header->devmajor), "%07o", 0);
212         xsnprintf(header->devminor, sizeof(header->devminor), "%07o", 0);
213
214         memcpy(header->magic, "ustar", 6);
215         memcpy(header->version, "00", 2);
216
217         xsnprintf(header->chksum, sizeof(header->chksum), "%07o", ustar_header_chksum(header));
218 }
219
220 static void write_extended_header(struct archiver_args *args,
221                                   const struct object_id *oid,
222                                   const void *buffer, unsigned long size)
223 {
224         struct ustar_header header;
225         unsigned int mode;
226         memset(&header, 0, sizeof(header));
227         *header.typeflag = TYPEFLAG_EXT_HEADER;
228         mode = 0100666;
229         xsnprintf(header.name, sizeof(header.name), "%s.paxheader", oid_to_hex(oid));
230         prepare_header(args, &header, mode, size);
231         write_blocked(&header, sizeof(header));
232         write_blocked(buffer, size);
233 }
234
235 static int write_tar_entry(struct archiver_args *args,
236                            const struct object_id *oid,
237                            const char *path, size_t pathlen,
238                            unsigned int mode)
239 {
240         struct ustar_header header;
241         struct strbuf ext_header = STRBUF_INIT;
242         unsigned int old_mode = mode;
243         unsigned long size, size_in_header;
244         void *buffer;
245         int err = 0;
246
247         memset(&header, 0, sizeof(header));
248
249         if (S_ISDIR(mode) || S_ISGITLINK(mode)) {
250                 *header.typeflag = TYPEFLAG_DIR;
251                 mode = (mode | 0777) & ~tar_umask;
252         } else if (S_ISLNK(mode)) {
253                 *header.typeflag = TYPEFLAG_LNK;
254                 mode |= 0777;
255         } else if (S_ISREG(mode)) {
256                 *header.typeflag = TYPEFLAG_REG;
257                 mode = (mode | ((mode & 0100) ? 0777 : 0666)) & ~tar_umask;
258         } else {
259                 return error("unsupported file mode: 0%o (SHA1: %s)",
260                              mode, oid_to_hex(oid));
261         }
262         if (pathlen > sizeof(header.name)) {
263                 size_t plen = get_path_prefix(path, pathlen,
264                                               sizeof(header.prefix));
265                 size_t rest = pathlen - plen - 1;
266                 if (plen > 0 && rest <= sizeof(header.name)) {
267                         memcpy(header.prefix, path, plen);
268                         memcpy(header.name, path + plen + 1, rest);
269                 } else {
270                         xsnprintf(header.name, sizeof(header.name), "%s.data",
271                                   oid_to_hex(oid));
272                         strbuf_append_ext_header(&ext_header, "path",
273                                                  path, pathlen);
274                 }
275         } else
276                 memcpy(header.name, path, pathlen);
277
278         if (S_ISREG(mode) && !args->convert &&
279             oid_object_info(oid, &size) == OBJ_BLOB &&
280             size > big_file_threshold)
281                 buffer = NULL;
282         else if (S_ISLNK(mode) || S_ISREG(mode)) {
283                 enum object_type type;
284                 buffer = object_file_to_archive(args, path, oid, old_mode, &type, &size);
285                 if (!buffer)
286                         return error("cannot read %s", oid_to_hex(oid));
287         } else {
288                 buffer = NULL;
289                 size = 0;
290         }
291
292         if (S_ISLNK(mode)) {
293                 if (size > sizeof(header.linkname)) {
294                         xsnprintf(header.linkname, sizeof(header.linkname),
295                                   "see %s.paxheader", oid_to_hex(oid));
296                         strbuf_append_ext_header(&ext_header, "linkpath",
297                                                  buffer, size);
298                 } else
299                         memcpy(header.linkname, buffer, size);
300         }
301
302         size_in_header = size;
303         if (S_ISREG(mode) && size > USTAR_MAX_SIZE) {
304                 size_in_header = 0;
305                 strbuf_append_ext_header_uint(&ext_header, "size", size);
306         }
307
308         prepare_header(args, &header, mode, size_in_header);
309
310         if (ext_header.len > 0) {
311                 write_extended_header(args, oid, ext_header.buf,
312                                       ext_header.len);
313         }
314         strbuf_release(&ext_header);
315         write_blocked(&header, sizeof(header));
316         if (S_ISREG(mode) && size > 0) {
317                 if (buffer)
318                         write_blocked(buffer, size);
319                 else
320                         err = stream_blocked(oid);
321         }
322         free(buffer);
323         return err;
324 }
325
326 static void write_global_extended_header(struct archiver_args *args)
327 {
328         const unsigned char *sha1 = args->commit_sha1;
329         struct strbuf ext_header = STRBUF_INIT;
330         struct ustar_header header;
331         unsigned int mode;
332
333         if (sha1)
334                 strbuf_append_ext_header(&ext_header, "comment",
335                                          sha1_to_hex(sha1), 40);
336         if (args->time > USTAR_MAX_MTIME) {
337                 strbuf_append_ext_header_uint(&ext_header, "mtime",
338                                               args->time);
339                 args->time = USTAR_MAX_MTIME;
340         }
341
342         if (!ext_header.len)
343                 return;
344
345         memset(&header, 0, sizeof(header));
346         *header.typeflag = TYPEFLAG_GLOBAL_HEADER;
347         mode = 0100666;
348         xsnprintf(header.name, sizeof(header.name), "pax_global_header");
349         prepare_header(args, &header, mode, ext_header.len);
350         write_blocked(&header, sizeof(header));
351         write_blocked(ext_header.buf, ext_header.len);
352         strbuf_release(&ext_header);
353 }
354
355 static struct archiver **tar_filters;
356 static int nr_tar_filters;
357 static int alloc_tar_filters;
358
359 static struct archiver *find_tar_filter(const char *name, int len)
360 {
361         int i;
362         for (i = 0; i < nr_tar_filters; i++) {
363                 struct archiver *ar = tar_filters[i];
364                 if (!strncmp(ar->name, name, len) && !ar->name[len])
365                         return ar;
366         }
367         return NULL;
368 }
369
370 static int tar_filter_config(const char *var, const char *value, void *data)
371 {
372         struct archiver *ar;
373         const char *name;
374         const char *type;
375         int namelen;
376
377         if (parse_config_key(var, "tar", &name, &namelen, &type) < 0 || !name)
378                 return 0;
379
380         ar = find_tar_filter(name, namelen);
381         if (!ar) {
382                 ar = xcalloc(1, sizeof(*ar));
383                 ar->name = xmemdupz(name, namelen);
384                 ar->write_archive = write_tar_filter_archive;
385                 ar->flags = ARCHIVER_WANT_COMPRESSION_LEVELS;
386                 ALLOC_GROW(tar_filters, nr_tar_filters + 1, alloc_tar_filters);
387                 tar_filters[nr_tar_filters++] = ar;
388         }
389
390         if (!strcmp(type, "command")) {
391                 if (!value)
392                         return config_error_nonbool(var);
393                 free(ar->data);
394                 ar->data = xstrdup(value);
395                 return 0;
396         }
397         if (!strcmp(type, "remote")) {
398                 if (git_config_bool(var, value))
399                         ar->flags |= ARCHIVER_REMOTE;
400                 else
401                         ar->flags &= ~ARCHIVER_REMOTE;
402                 return 0;
403         }
404
405         return 0;
406 }
407
408 static int git_tar_config(const char *var, const char *value, void *cb)
409 {
410         if (!strcmp(var, "tar.umask")) {
411                 if (value && !strcmp(value, "user")) {
412                         tar_umask = umask(0);
413                         umask(tar_umask);
414                 } else {
415                         tar_umask = git_config_int(var, value);
416                 }
417                 return 0;
418         }
419
420         return tar_filter_config(var, value, cb);
421 }
422
423 static int write_tar_archive(const struct archiver *ar,
424                              struct archiver_args *args)
425 {
426         int err = 0;
427
428         write_global_extended_header(args);
429         err = write_archive_entries(args, write_tar_entry);
430         if (!err)
431                 write_trailer();
432         return err;
433 }
434
435 static int write_tar_filter_archive(const struct archiver *ar,
436                                     struct archiver_args *args)
437 {
438         struct strbuf cmd = STRBUF_INIT;
439         struct child_process filter = CHILD_PROCESS_INIT;
440         const char *argv[2];
441         int r;
442
443         if (!ar->data)
444                 die("BUG: tar-filter archiver called with no filter defined");
445
446         strbuf_addstr(&cmd, ar->data);
447         if (args->compression_level >= 0)
448                 strbuf_addf(&cmd, " -%d", args->compression_level);
449
450         argv[0] = cmd.buf;
451         argv[1] = NULL;
452         filter.argv = argv;
453         filter.use_shell = 1;
454         filter.in = -1;
455
456         if (start_command(&filter) < 0)
457                 die_errno("unable to start '%s' filter", argv[0]);
458         close(1);
459         if (dup2(filter.in, 1) < 0)
460                 die_errno("unable to redirect descriptor");
461         close(filter.in);
462
463         r = write_tar_archive(ar, args);
464
465         close(1);
466         if (finish_command(&filter) != 0)
467                 die("'%s' filter reported error", argv[0]);
468
469         strbuf_release(&cmd);
470         return r;
471 }
472
473 static struct archiver tar_archiver = {
474         "tar",
475         write_tar_archive,
476         ARCHIVER_REMOTE
477 };
478
479 void init_tar_archiver(void)
480 {
481         int i;
482         register_archiver(&tar_archiver);
483
484         tar_filter_config("tar.tgz.command", "gzip -cn", NULL);
485         tar_filter_config("tar.tgz.remote", "true", NULL);
486         tar_filter_config("tar.tar.gz.command", "gzip -cn", NULL);
487         tar_filter_config("tar.tar.gz.remote", "true", NULL);
488         git_config(git_tar_config, NULL);
489         for (i = 0; i < nr_tar_filters; i++) {
490                 /* omit any filters that never had a command configured */
491                 if (tar_filters[i]->data)
492                         register_archiver(tar_filters[i]);
493         }
494 }