git.oblomov.eu Git - git/blob - sha1_file.c

   1 /*
   2  * GIT - The information manager from hell
   3  *
   4  * Copyright (C) Linus Torvalds, 2005
   5  *
   6  * This handles basic git sha1 object files - packing, unpacking,
   7  * creation etc.
   8  */
   9 #include "cache.h"
  10 #include "delta.h"
  11 #include "pack.h"
  12 #include "blob.h"
  13 #include "commit.h"
  14 #include "tag.h"
  15 #include "tree.h"
  16
  17 #ifndef O_NOATIME
  18 #if defined(__linux__) && (defined(__i386__) || defined(__PPC__))
  19 #define O_NOATIME 01000000
  20 #else
  21 #define O_NOATIME 0
  22 #endif
  23 #endif
  24
  25 const unsigned char null_sha1[20] = { 0, };
  26
  27 static unsigned int sha1_file_open_flag = O_NOATIME;
  28
  29 static unsigned hexval(char c)
  30 {
  31         if (c >= '0' && c <= '9')
  32                 return c - '0';
  33         if (c >= 'a' && c <= 'f')
  34                 return c - 'a' + 10;
  35         if (c >= 'A' && c <= 'F')
  36                 return c - 'A' + 10;
  37         return ~0;
  38 }
  39
  40 int get_sha1_hex(const char *hex, unsigned char *sha1)
  41 {
  42         int i;
  43         for (i = 0; i < 20; i++) {
  44                 unsigned int val = (hexval(hex[0]) << 4) | hexval(hex[1]);
  45                 if (val & ~0xff)
  46                         return -1;
  47                 *sha1++ = val;
  48                 hex += 2;
  49         }
  50         return 0;
  51 }
  52
  53 int safe_create_leading_directories(char *path)
  54 {
  55         char *pos = path;
  56         struct stat st;
  57
  58         if (*pos == '/')
  59                 pos++;
  60
  61         while (pos) {
  62                 pos = strchr(pos, '/');
  63                 if (!pos)
  64                         break;
  65                 *pos = 0;
  66                 if (!stat(path, &st)) {
  67                         /* path exists */
  68                         if (!S_ISDIR(st.st_mode)) {
  69                                 *pos = '/';
  70                                 return -3;
  71                         }
  72                 }
  73                 else if (mkdir(path, 0777)) {
  74                         *pos = '/';
  75                         return -1;
  76                 }
  77                 else if (adjust_shared_perm(path)) {
  78                         *pos = '/';
  79                         return -2;
  80                 }
  81                 *pos++ = '/';
  82         }
  83         return 0;
  84 }
  85
  86 char * sha1_to_hex(const unsigned char *sha1)
  87 {
  88         static int bufno;
  89         static char hexbuffer[4][50];
  90         static const char hex[] = "0123456789abcdef";
  91         char *buffer = hexbuffer[3 & ++bufno], *buf = buffer;
  92         int i;
  93
  94         for (i = 0; i < 20; i++) {
  95                 unsigned int val = *sha1++;
  96                 *buf++ = hex[val >> 4];
  97                 *buf++ = hex[val & 0xf];
  98         }
  99         *buf = '\0';
 100
 101         return buffer;
 102 }
 103
 104 static void fill_sha1_path(char *pathbuf, const unsigned char *sha1)
 105 {
 106         int i;
 107         for (i = 0; i < 20; i++) {
 108                 static char hex[] = "0123456789abcdef";
 109                 unsigned int val = sha1[i];
 110                 char *pos = pathbuf + i*2 + (i > 0);
 111                 *pos++ = hex[val >> 4];
 112                 *pos = hex[val & 0xf];
 113         }
 114 }
 115
 116 /*
 117  * NOTE! This returns a statically allocated buffer, so you have to be
 118  * careful about using it. Do a "strdup()" if you need to save the
 119  * filename.
 120  *
 121  * Also note that this returns the location for creating.  Reading
 122  * SHA1 file can happen from any alternate directory listed in the
 123  * DB_ENVIRONMENT environment variable if it is not found in
 124  * the primary object database.
 125  */
 126 char *sha1_file_name(const unsigned char *sha1)
 127 {
 128         static char *name, *base;
 129         static const char *last_objdir;
 130         const char *sha1_file_directory = get_object_directory();
 131
 132         if (!last_objdir || strcmp(last_objdir, sha1_file_directory)) {
 133                 int len = strlen(sha1_file_directory);
 134                 if (base)
 135                         free(base);
 136                 base = xmalloc(len + 60);
 137                 memcpy(base, sha1_file_directory, len);
 138                 memset(base+len, 0, 60);
 139                 base[len] = '/';
 140                 base[len+3] = '/';
 141                 name = base + len + 1;
 142                 if (last_objdir)
 143                         free((char *) last_objdir);
 144                 last_objdir = strdup(sha1_file_directory);
 145         }
 146         fill_sha1_path(name, sha1);
 147         return base;
 148 }
 149
 150 char *sha1_pack_name(const unsigned char *sha1)
 151 {
 152         static const char hex[] = "0123456789abcdef";
 153         static char *name, *base, *buf;
 154         static const char *last_objdir;
 155         const char *sha1_file_directory = get_object_directory();
 156         int i;
 157
 158         if (!last_objdir || strcmp(last_objdir, sha1_file_directory)) {
 159                 int len = strlen(sha1_file_directory);
 160                 if (base)
 161                         free(base);
 162                 base = xmalloc(len + 60);
 163                 sprintf(base, "%s/pack/pack-1234567890123456789012345678901234567890.pack", sha1_file_directory);
 164                 name = base + len + 11;
 165                 if (last_objdir)
 166                         free((char *) last_objdir);
 167                 last_objdir = strdup(sha1_file_directory);
 168         }
 169
 170         buf = name;
 171
 172         for (i = 0; i < 20; i++) {
 173                 unsigned int val = *sha1++;
 174                 *buf++ = hex[val >> 4];
 175                 *buf++ = hex[val & 0xf];
 176         }
 177
 178         return base;
 179 }
 180
 181 char *sha1_pack_index_name(const unsigned char *sha1)
 182 {
 183         static const char hex[] = "0123456789abcdef";
 184         static char *name, *base, *buf;
 185         static const char *last_objdir;
 186         const char *sha1_file_directory = get_object_directory();
 187         int i;
 188
 189         if (!last_objdir || strcmp(last_objdir, sha1_file_directory)) {
 190                 int len = strlen(sha1_file_directory);
 191                 if (base)
 192                         free(base);
 193                 base = xmalloc(len + 60);
 194                 sprintf(base, "%s/pack/pack-1234567890123456789012345678901234567890.idx", sha1_file_directory);
 195                 name = base + len + 11;
 196                 if (last_objdir)
 197                         free((char *) last_objdir);
 198                 last_objdir = strdup(sha1_file_directory);
 199         }
 200
 201         buf = name;
 202
 203         for (i = 0; i < 20; i++) {
 204                 unsigned int val = *sha1++;
 205                 *buf++ = hex[val >> 4];
 206                 *buf++ = hex[val & 0xf];
 207         }
 208
 209         return base;
 210 }
 211
 212 struct alternate_object_database *alt_odb_list;
 213 static struct alternate_object_database **alt_odb_tail;
 214
 215 static void read_info_alternates(const char * alternates, int depth);
 216
 217 /*
 218  * Prepare alternate object database registry.
 219  *
 220  * The variable alt_odb_list points at the list of struct
 221  * alternate_object_database.  The elements on this list come from
 222  * non-empty elements from colon separated ALTERNATE_DB_ENVIRONMENT
 223  * environment variable, and $GIT_OBJECT_DIRECTORY/info/alternates,
 224  * whose contents is similar to that environment variable but can be
 225  * LF separated.  Its base points at a statically allocated buffer that
 226  * contains "/the/directory/corresponding/to/.git/objects/...", while
 227  * its name points just after the slash at the end of ".git/objects/"
 228  * in the example above, and has enough space to hold 40-byte hex
 229  * SHA1, an extra slash for the first level indirection, and the
 230  * terminating NUL.
 231  */
 232 static int link_alt_odb_entry(const char * entry, int len, const char * relative_base, int depth)
 233 {
 234         struct stat st;
 235         const char *objdir = get_object_directory();
 236         struct alternate_object_database *ent;
 237         struct alternate_object_database *alt;
 238         /* 43 = 40-byte + 2 '/' + terminating NUL */
 239         int pfxlen = len;
 240         int entlen = pfxlen + 43;
 241         int base_len = -1;
 242
 243         if (*entry != '/' && relative_base) {
 244                 /* Relative alt-odb */
 245                 if (base_len < 0)
 246                         base_len = strlen(relative_base) + 1;
 247                 entlen += base_len;
 248                 pfxlen += base_len;
 249         }
 250         ent = xmalloc(sizeof(*ent) + entlen);
 251
 252         if (*entry != '/' && relative_base) {
 253                 memcpy(ent->base, relative_base, base_len - 1);
 254                 ent->base[base_len - 1] = '/';
 255                 memcpy(ent->base + base_len, entry, len);
 256         }
 257         else
 258                 memcpy(ent->base, entry, pfxlen);
 259
 260         ent->name = ent->base + pfxlen + 1;
 261         ent->base[pfxlen + 3] = '/';
 262         ent->base[pfxlen] = ent->base[entlen-1] = 0;
 263
 264         /* Detect cases where alternate disappeared */
 265         if (stat(ent->base, &st) || !S_ISDIR(st.st_mode)) {
 266                 error("object directory %s does not exist; "
 267                       "check .git/objects/info/alternates.",
 268                       ent->base);
 269                 free(ent);
 270                 return -1;
 271         }
 272
 273         /* Prevent the common mistake of listing the same
 274          * thing twice, or object directory itself.
 275          */
 276         for (alt = alt_odb_list; alt; alt = alt->next) {
 277                 if (!memcmp(ent->base, alt->base, pfxlen)) {
 278                         free(ent);
 279                         return -1;
 280                 }
 281         }
 282         if (!memcmp(ent->base, objdir, pfxlen)) {
 283                 free(ent);
 284                 return -1;
 285         }
 286
 287         /* add the alternate entry */
 288         *alt_odb_tail = ent;
 289         alt_odb_tail = &(ent->next);
 290         ent->next = NULL;
 291
 292         /* recursively add alternates */
 293         read_info_alternates(ent->base, depth + 1);
 294
 295         ent->base[pfxlen] = '/';
 296
 297         return 0;
 298 }
 299
 300 static void link_alt_odb_entries(const char *alt, const char *ep, int sep,
 301                                  const char *relative_base, int depth)
 302 {
 303         const char *cp, *last;
 304
 305         if (depth > 5) {
 306                 error("%s: ignoring alternate object stores, nesting too deep.",
 307                                 relative_base);
 308                 return;
 309         }
 310
 311         last = alt;
 312         while (last < ep) {
 313                 cp = last;
 314                 if (cp < ep && *cp == '#') {
 315                         while (cp < ep && *cp != sep)
 316                                 cp++;
 317                         last = cp + 1;
 318                         continue;
 319                 }
 320                 while (cp < ep && *cp != sep)
 321                         cp++;
 322                 if (last != cp) {
 323                         if ((*last != '/') && depth) {
 324                                 error("%s: ignoring relative alternate object store %s",
 325                                                 relative_base, last);
 326                         } else {
 327                                 link_alt_odb_entry(last, cp - last,
 328                                                 relative_base, depth);
 329                         }
 330                 }
 331                 while (cp < ep && *cp == sep)
 332                         cp++;
 333                 last = cp;
 334         }
 335 }
 336
 337 static void read_info_alternates(const char * relative_base, int depth)
 338 {
 339         char *map;
 340         struct stat st;
 341         char path[PATH_MAX];
 342         int fd;
 343
 344         sprintf(path, "%s/info/alternates", relative_base);
 345         fd = open(path, O_RDONLY);
 346         if (fd < 0)
 347                 return;
 348         if (fstat(fd, &st) || (st.st_size == 0)) {
 349                 close(fd);
 350                 return;
 351         }
 352         map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
 353         close(fd);
 354         if (map == MAP_FAILED)
 355                 return;
 356
 357         link_alt_odb_entries(map, map + st.st_size, '\n', relative_base, depth);
 358
 359         munmap(map, st.st_size);
 360 }
 361
 362 void prepare_alt_odb(void)
 363 {
 364         const char *alt;
 365
 366         alt = getenv(ALTERNATE_DB_ENVIRONMENT);
 367         if (!alt) alt = "";
 368
 369         if (alt_odb_tail)
 370                 return;
 371         alt_odb_tail = &alt_odb_list;
 372         link_alt_odb_entries(alt, alt + strlen(alt), ':', NULL, 0);
 373
 374         read_info_alternates(get_object_directory(), 0);
 375 }
 376
 377 static char *find_sha1_file(const unsigned char *sha1, struct stat *st)
 378 {
 379         char *name = sha1_file_name(sha1);
 380         struct alternate_object_database *alt;
 381
 382         if (!stat(name, st))
 383                 return name;
 384         prepare_alt_odb();
 385         for (alt = alt_odb_list; alt; alt = alt->next) {
 386                 name = alt->name;
 387                 fill_sha1_path(name, sha1);
 388                 if (!stat(alt->base, st))
 389                         return alt->base;
 390         }
 391         return NULL;
 392 }
 393
 394 #define PACK_MAX_SZ (1<<26)
 395 static int pack_used_ctr;
 396 static unsigned long pack_mapped;
 397 struct packed_git *packed_git;
 398
 399 static int check_packed_git_idx(const char *path, unsigned long *idx_size_,
 400                                 void **idx_map_)
 401 {
 402         void *idx_map;
 403         unsigned int *index;
 404         unsigned long idx_size;
 405         int nr, i;
 406         int fd = open(path, O_RDONLY);
 407         struct stat st;
 408         if (fd < 0)
 409                 return -1;
 410         if (fstat(fd, &st)) {
 411                 close(fd);
 412                 return -1;
 413         }
 414         idx_size = st.st_size;
 415         idx_map = mmap(NULL, idx_size, PROT_READ, MAP_PRIVATE, fd, 0);
 416         close(fd);
 417         if (idx_map == MAP_FAILED)
 418                 return -1;
 419
 420         index = idx_map;
 421         *idx_map_ = idx_map;
 422         *idx_size_ = idx_size;
 423
 424         /* check index map */
 425         if (idx_size < 4*256 + 20 + 20)
 426                 return error("index file too small");
 427         nr = 0;
 428         for (i = 0; i < 256; i++) {
 429                 unsigned int n = ntohl(index[i]);
 430                 if (n < nr)
 431                         return error("non-monotonic index");
 432                 nr = n;
 433         }
 434
 435         /*
 436          * Total size:
 437          *  - 256 index entries 4 bytes each
 438          *  - 24-byte entries * nr (20-byte sha1 + 4-byte offset)
 439          *  - 20-byte SHA1 of the packfile
 440          *  - 20-byte SHA1 file checksum
 441          */
 442         if (idx_size != 4*256 + nr * 24 + 20 + 20)
 443                 return error("wrong index file size");
 444
 445         return 0;
 446 }
 447
 448 static int unuse_one_packed_git(void)
 449 {
 450         struct packed_git *p, *lru = NULL;
 451
 452         for (p = packed_git; p; p = p->next) {
 453                 if (p->pack_use_cnt || !p->pack_base)
 454                         continue;
 455                 if (!lru || p->pack_last_used < lru->pack_last_used)
 456                         lru = p;
 457         }
 458         if (!lru)
 459                 return 0;
 460         munmap(lru->pack_base, lru->pack_size);
 461         lru->pack_base = NULL;
 462         return 1;
 463 }
 464
 465 void unuse_packed_git(struct packed_git *p)
 466 {
 467         p->pack_use_cnt--;
 468 }
 469
 470 int use_packed_git(struct packed_git *p)
 471 {
 472         if (!p->pack_size) {
 473                 struct stat st;
 474                 // We created the struct before we had the pack
 475                 stat(p->pack_name, &st);
 476                 if (!S_ISREG(st.st_mode))
 477                         die("packfile %s not a regular file", p->pack_name);
 478                 p->pack_size = st.st_size;
 479         }
 480         if (!p->pack_base) {
 481                 int fd;
 482                 struct stat st;
 483                 void *map;
 484
 485                 pack_mapped += p->pack_size;
 486                 while (PACK_MAX_SZ < pack_mapped && unuse_one_packed_git())
 487                         ; /* nothing */
 488                 fd = open(p->pack_name, O_RDONLY);
 489                 if (fd < 0)
 490                         die("packfile %s cannot be opened", p->pack_name);
 491                 if (fstat(fd, &st)) {
 492                         close(fd);
 493                         die("packfile %s cannot be opened", p->pack_name);
 494                 }
 495                 if (st.st_size != p->pack_size)
 496                         die("packfile %s size mismatch.", p->pack_name);
 497                 map = mmap(NULL, p->pack_size, PROT_READ, MAP_PRIVATE, fd, 0);
 498                 close(fd);
 499                 if (map == MAP_FAILED)
 500                         die("packfile %s cannot be mapped.", p->pack_name);
 501                 p->pack_base = map;
 502
 503                 /* Check if the pack file matches with the index file.
 504                  * this is cheap.
 505                  */
 506                 if (memcmp((char*)(p->index_base) + p->index_size - 40,
 507                            (char *) p->pack_base + p->pack_size - 20,
 508                            20)) {
 509
 510                         die("packfile %s does not match index.", p->pack_name);
 511                 }
 512         }
 513         p->pack_last_used = pack_used_ctr++;
 514         p->pack_use_cnt++;
 515         return 0;
 516 }
 517
 518 struct packed_git *add_packed_git(char *path, int path_len, int local)
 519 {
 520         struct stat st;
 521         struct packed_git *p;
 522         unsigned long idx_size;
 523         void *idx_map;
 524         unsigned char sha1[20];
 525
 526         if (check_packed_git_idx(path, &idx_size, &idx_map))
 527                 return NULL;
 528
 529         /* do we have a corresponding .pack file? */
 530         strcpy(path + path_len - 4, ".pack");
 531         if (stat(path, &st) || !S_ISREG(st.st_mode)) {
 532                 munmap(idx_map, idx_size);
 533                 return NULL;
 534         }
 535         /* ok, it looks sane as far as we can check without
 536          * actually mapping the pack file.
 537          */
 538         p = xmalloc(sizeof(*p) + path_len + 2);
 539         strcpy(p->pack_name, path);
 540         p->index_size = idx_size;
 541         p->pack_size = st.st_size;
 542         p->index_base = idx_map;
 543         p->next = NULL;
 544         p->pack_base = NULL;
 545         p->pack_last_used = 0;
 546         p->pack_use_cnt = 0;
 547         p->pack_local = local;
 548         if ((path_len > 44) && !get_sha1_hex(path + path_len - 44, sha1))
 549                 memcpy(p->sha1, sha1, 20);
 550         return p;
 551 }
 552
 553 struct packed_git *parse_pack_index(unsigned char *sha1)
 554 {
 555         char *path = sha1_pack_index_name(sha1);
 556         return parse_pack_index_file(sha1, path);
 557 }
 558
 559 struct packed_git *parse_pack_index_file(const unsigned char *sha1, char *idx_path)
 560 {
 561         struct packed_git *p;
 562         unsigned long idx_size;
 563         void *idx_map;
 564         char *path;
 565
 566         if (check_packed_git_idx(idx_path, &idx_size, &idx_map))
 567                 return NULL;
 568
 569         path = sha1_pack_name(sha1);
 570
 571         p = xmalloc(sizeof(*p) + strlen(path) + 2);
 572         strcpy(p->pack_name, path);
 573         p->index_size = idx_size;
 574         p->pack_size = 0;
 575         p->index_base = idx_map;
 576         p->next = NULL;
 577         p->pack_base = NULL;
 578         p->pack_last_used = 0;
 579         p->pack_use_cnt = 0;
 580         memcpy(p->sha1, sha1, 20);
 581         return p;
 582 }
 583
 584 void install_packed_git(struct packed_git *pack)
 585 {
 586         pack->next = packed_git;
 587         packed_git = pack;
 588 }
 589
 590 static void prepare_packed_git_one(char *objdir, int local)
 591 {
 592         char path[PATH_MAX];
 593         int len;
 594         DIR *dir;
 595         struct dirent *de;
 596
 597         sprintf(path, "%s/pack", objdir);
 598         len = strlen(path);
 599         dir = opendir(path);
 600         if (!dir) {
 601                 if (errno != ENOENT)
 602                         error("unable to open object pack directory: %s: %s",
 603                               path, strerror(errno));
 604                 return;
 605         }
 606         path[len++] = '/';
 607         while ((de = readdir(dir)) != NULL) {
 608                 int namelen = strlen(de->d_name);
 609                 struct packed_git *p;
 610
 611                 if (strcmp(de->d_name + namelen - 4, ".idx"))
 612                         continue;
 613
 614                 /* we have .idx.  Is it a file we can map? */
 615                 strcpy(path + len, de->d_name);
 616                 for (p = packed_git; p; p = p->next) {
 617                         if (!memcmp(path, p->pack_name, len + namelen - 4))
 618                                 break;
 619                 }
 620                 if (p)
 621                         continue;
 622                 p = add_packed_git(path, len + namelen, local);
 623                 if (!p)
 624                         continue;
 625                 p->next = packed_git;
 626                 packed_git = p;
 627         }
 628         closedir(dir);
 629 }
 630
 631 static int prepare_packed_git_run_once = 0;
 632 void prepare_packed_git(void)
 633 {
 634         struct alternate_object_database *alt;
 635
 636         if (prepare_packed_git_run_once)
 637                 return;
 638         prepare_packed_git_one(get_object_directory(), 1);
 639         prepare_alt_odb();
 640         for (alt = alt_odb_list; alt; alt = alt->next) {
 641                 alt->name[-1] = 0;
 642                 prepare_packed_git_one(alt->base, 0);
 643                 alt->name[-1] = '/';
 644         }
 645         prepare_packed_git_run_once = 1;
 646 }
 647
 648 static void reprepare_packed_git(void)
 649 {
 650         prepare_packed_git_run_once = 0;
 651         prepare_packed_git();
 652 }
 653
 654 int check_sha1_signature(const unsigned char *sha1, void *map, unsigned long size, const char *type)
 655 {
 656         char header[100];
 657         unsigned char real_sha1[20];
 658         SHA_CTX c;
 659
 660         SHA1_Init(&c);
 661         SHA1_Update(&c, header, 1+sprintf(header, "%s %lu", type, size));
 662         SHA1_Update(&c, map, size);
 663         SHA1_Final(real_sha1, &c);
 664         return memcmp(sha1, real_sha1, 20) ? -1 : 0;
 665 }
 666
 667 static void *map_sha1_file_internal(const unsigned char *sha1,
 668                                     unsigned long *size)
 669 {
 670         struct stat st;
 671         void *map;
 672         int fd;
 673         char *filename = find_sha1_file(sha1, &st);
 674
 675         if (!filename) {
 676                 return NULL;
 677         }
 678
 679         fd = open(filename, O_RDONLY | sha1_file_open_flag);
 680         if (fd < 0) {
 681                 /* See if it works without O_NOATIME */
 682                 switch (sha1_file_open_flag) {
 683                 default:
 684                         fd = open(filename, O_RDONLY);
 685                         if (fd >= 0)
 686                                 break;
 687                 /* Fallthrough */
 688                 case 0:
 689                         return NULL;
 690                 }
 691
 692                 /* If it failed once, it will probably fail again.
 693                  * Stop using O_NOATIME
 694                  */
 695                 sha1_file_open_flag = 0;
 696         }
 697         map = mmap(NULL, st.st_size, PROT_READ, MAP_PRIVATE, fd, 0);
 698         close(fd);
 699         if (map == MAP_FAILED)
 700                 return NULL;
 701         *size = st.st_size;
 702         return map;
 703 }
 704
 705 int unpack_sha1_header(z_stream *stream, void *map, unsigned long mapsize, void *buffer, unsigned long size)
 706 {
 707         /* Get the data stream */
 708         memset(stream, 0, sizeof(*stream));
 709         stream->next_in = map;
 710         stream->avail_in = mapsize;
 711         stream->next_out = buffer;
 712         stream->avail_out = size;
 713
 714         inflateInit(stream);
 715         return inflate(stream, 0);
 716 }
 717
 718 static void *unpack_sha1_rest(z_stream *stream, void *buffer, unsigned long size)
 719 {
 720         int bytes = strlen(buffer) + 1;
 721         unsigned char *buf = xmalloc(1+size);
 722
 723         memcpy(buf, (char *) buffer + bytes, stream->total_out - bytes);
 724         bytes = stream->total_out - bytes;
 725         if (bytes < size) {
 726                 stream->next_out = buf + bytes;
 727                 stream->avail_out = size - bytes;
 728                 while (inflate(stream, Z_FINISH) == Z_OK)
 729                         /* nothing */;
 730         }
 731         buf[size] = 0;
 732         inflateEnd(stream);
 733         return buf;
 734 }
 735
 736 /*
 737  * We used to just use "sscanf()", but that's actually way
 738  * too permissive for what we want to check. So do an anal
 739  * object header parse by hand.
 740  */
 741 int parse_sha1_header(char *hdr, char *type, unsigned long *sizep)
 742 {
 743         int i;
 744         unsigned long size;
 745
 746         /*
 747          * The type can be at most ten bytes (including the
 748          * terminating '\0' that we add), and is followed by
 749          * a space.
 750          */
 751         i = 10;
 752         for (;;) {
 753                 char c = *hdr++;
 754                 if (c == ' ')
 755                         break;
 756                 if (!--i)
 757                         return -1;
 758                 *type++ = c;
 759         }
 760         *type = 0;
 761
 762         /*
 763          * The length must follow immediately, and be in canonical
 764          * decimal format (ie "010" is not valid).
 765          */
 766         size = *hdr++ - '0';
 767         if (size > 9)
 768                 return -1;
 769         if (size) {
 770                 for (;;) {
 771                         unsigned long c = *hdr - '0';
 772                         if (c > 9)
 773                                 break;
 774                         hdr++;
 775                         size = size * 10 + c;
 776                 }
 777         }
 778         *sizep = size;
 779
 780         /*
 781          * The length must be followed by a zero byte
 782          */
 783         return *hdr ? -1 : 0;
 784 }
 785
 786 void * unpack_sha1_file(void *map, unsigned long mapsize, char *type, unsigned long *size)
 787 {
 788         int ret;
 789         z_stream stream;
 790         char hdr[8192];
 791
 792         ret = unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr));
 793         if (ret < Z_OK || parse_sha1_header(hdr, type, size) < 0)
 794                 return NULL;
 795
 796         return unpack_sha1_rest(&stream, hdr, *size);
 797 }
 798
 799 /* forward declaration for a mutually recursive function */
 800 static int packed_object_info(struct pack_entry *entry,
 801                               char *type, unsigned long *sizep);
 802
 803 static int packed_delta_info(unsigned char *base_sha1,
 804                              unsigned long delta_size,
 805                              unsigned long left,
 806                              char *type,
 807                              unsigned long *sizep,
 808                              struct packed_git *p)
 809 {
 810         struct pack_entry base_ent;
 811
 812         if (left < 20)
 813                 die("truncated pack file");
 814
 815         /* The base entry _must_ be in the same pack */
 816         if (!find_pack_entry_one(base_sha1, &base_ent, p))
 817                 die("failed to find delta-pack base object %s",
 818                     sha1_to_hex(base_sha1));
 819
 820         /* We choose to only get the type of the base object and
 821          * ignore potentially corrupt pack file that expects the delta
 822          * based on a base with a wrong size.  This saves tons of
 823          * inflate() calls.
 824          */
 825
 826         if (packed_object_info(&base_ent, type, NULL))
 827                 die("cannot get info for delta-pack base");
 828
 829         if (sizep) {
 830                 const unsigned char *data;
 831                 unsigned char delta_head[64];
 832                 unsigned long result_size;
 833                 z_stream stream;
 834                 int st;
 835
 836                 memset(&stream, 0, sizeof(stream));
 837
 838                 data = stream.next_in = base_sha1 + 20;
 839                 stream.avail_in = left - 20;
 840                 stream.next_out = delta_head;
 841                 stream.avail_out = sizeof(delta_head);
 842
 843                 inflateInit(&stream);
 844                 st = inflate(&stream, Z_FINISH);
 845                 inflateEnd(&stream);
 846                 if ((st != Z_STREAM_END) &&
 847                     stream.total_out != sizeof(delta_head))
 848                         die("delta data unpack-initial failed");
 849
 850                 /* Examine the initial part of the delta to figure out
 851                  * the result size.
 852                  */
 853                 data = delta_head;
 854
 855                 /* ignore base size */
 856                 get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
 857
 858                 /* Read the result size */
 859                 result_size = get_delta_hdr_size(&data, delta_head+sizeof(delta_head));
 860                 *sizep = result_size;
 861         }
 862         return 0;
 863 }
 864
 865 static unsigned long unpack_object_header(struct packed_git *p, unsigned long offset,
 866         enum object_type *type, unsigned long *sizep)
 867 {
 868         unsigned shift;
 869         unsigned char *pack, c;
 870         unsigned long size;
 871
 872         if (offset >= p->pack_size)
 873                 die("object offset outside of pack file");
 874
 875         pack =  (unsigned char *) p->pack_base + offset;
 876         c = *pack++;
 877         offset++;
 878         *type = (c >> 4) & 7;
 879         size = c & 15;
 880         shift = 4;
 881         while (c & 0x80) {
 882                 if (offset >= p->pack_size)
 883                         die("object offset outside of pack file");
 884                 c = *pack++;
 885                 offset++;
 886                 size += (c & 0x7f) << shift;
 887                 shift += 7;
 888         }
 889         *sizep = size;
 890         return offset;
 891 }
 892
 893 int check_reuse_pack_delta(struct packed_git *p, unsigned long offset,
 894                            unsigned char *base, unsigned long *sizep,
 895                            enum object_type *kindp)
 896 {
 897         unsigned long ptr;
 898         int status = -1;
 899
 900         use_packed_git(p);
 901         ptr = offset;
 902         ptr = unpack_object_header(p, ptr, kindp, sizep);
 903         if (*kindp != OBJ_DELTA)
 904                 goto done;
 905         memcpy(base, (char *) p->pack_base + ptr, 20);
 906         status = 0;
 907  done:
 908         unuse_packed_git(p);
 909         return status;
 910 }
 911
 912 void packed_object_info_detail(struct pack_entry *e,
 913                                char *type,
 914                                unsigned long *size,
 915                                unsigned long *store_size,
 916                                unsigned int *delta_chain_length,
 917                                unsigned char *base_sha1)
 918 {
 919         struct packed_git *p = e->p;
 920         unsigned long offset;
 921         unsigned char *pack;
 922         enum object_type kind;
 923
 924         offset = unpack_object_header(p, e->offset, &kind, size);
 925         pack = (unsigned char *) p->pack_base + offset;
 926         if (kind != OBJ_DELTA)
 927                 *delta_chain_length = 0;
 928         else {
 929                 unsigned int chain_length = 0;
 930                 if (p->pack_size <= offset + 20)
 931                         die("pack file %s records an incomplete delta base",
 932                             p->pack_name);
 933                 memcpy(base_sha1, pack, 20);
 934                 do {
 935                         struct pack_entry base_ent;
 936                         unsigned long junk;
 937
 938                         find_pack_entry_one(pack, &base_ent, p);
 939                         offset = unpack_object_header(p, base_ent.offset,
 940                                                       &kind, &junk);
 941                         pack = (unsigned char *) p->pack_base + offset;
 942                         chain_length++;
 943                 } while (kind == OBJ_DELTA);
 944                 *delta_chain_length = chain_length;
 945         }
 946         switch (kind) {
 947         case OBJ_COMMIT:
 948                 strcpy(type, commit_type);
 949                 break;
 950         case OBJ_TREE:
 951                 strcpy(type, tree_type);
 952                 break;
 953         case OBJ_BLOB:
 954                 strcpy(type, blob_type);
 955                 break;
 956         case OBJ_TAG:
 957                 strcpy(type, tag_type);
 958                 break;
 959         default:
 960                 die("corrupted pack file %s containing object of kind %d",
 961                     p->pack_name, kind);
 962         }
 963         *store_size = 0; /* notyet */
 964 }
 965
 966 static int packed_object_info(struct pack_entry *entry,
 967                               char *type, unsigned long *sizep)
 968 {
 969         struct packed_git *p = entry->p;
 970         unsigned long offset, size, left;
 971         unsigned char *pack;
 972         enum object_type kind;
 973         int retval;
 974
 975         if (use_packed_git(p))
 976                 die("cannot map packed file");
 977
 978         offset = unpack_object_header(p, entry->offset, &kind, &size);
 979         pack = (unsigned char *) p->pack_base + offset;
 980         left = p->pack_size - offset;
 981
 982         switch (kind) {
 983         case OBJ_DELTA:
 984                 retval = packed_delta_info(pack, size, left, type, sizep, p);
 985                 unuse_packed_git(p);
 986                 return retval;
 987         case OBJ_COMMIT:
 988                 strcpy(type, commit_type);
 989                 break;
 990         case OBJ_TREE:
 991                 strcpy(type, tree_type);
 992                 break;
 993         case OBJ_BLOB:
 994                 strcpy(type, blob_type);
 995                 break;
 996         case OBJ_TAG:
 997                 strcpy(type, tag_type);
 998                 break;
 999         default:
1000                 die("corrupted pack file %s containing object of kind %d",
1001                     p->pack_name, kind);
1002         }
1003         if (sizep)
1004                 *sizep = size;
1005         unuse_packed_git(p);
1006         return 0;
1007 }
1008
1009 /* forward declaration for a mutually recursive function */
1010 static void *unpack_entry(struct pack_entry *, char *, unsigned long *);
1011
1012 static void *unpack_delta_entry(unsigned char *base_sha1,
1013                                 unsigned long delta_size,
1014                                 unsigned long left,
1015                                 char *type,
1016                                 unsigned long *sizep,
1017                                 struct packed_git *p)
1018 {
1019         struct pack_entry base_ent;
1020         void *data, *delta_data, *result, *base;
1021         unsigned long data_size, result_size, base_size;
1022         z_stream stream;
1023         int st;
1024
1025         if (left < 20)
1026                 die("truncated pack file");
1027
1028         /* The base entry _must_ be in the same pack */
1029         if (!find_pack_entry_one(base_sha1, &base_ent, p))
1030                 die("failed to find delta-pack base object %s",
1031                     sha1_to_hex(base_sha1));
1032         base = unpack_entry_gently(&base_ent, type, &base_size);
1033         if (!base)
1034                 die("failed to read delta-pack base object %s",
1035                     sha1_to_hex(base_sha1));
1036
1037         data = base_sha1 + 20;
1038         data_size = left - 20;
1039         delta_data = xmalloc(delta_size);
1040
1041         memset(&stream, 0, sizeof(stream));
1042
1043         stream.next_in = data;
1044         stream.avail_in = data_size;
1045         stream.next_out = delta_data;
1046         stream.avail_out = delta_size;
1047
1048         inflateInit(&stream);
1049         st = inflate(&stream, Z_FINISH);
1050         inflateEnd(&stream);
1051         if ((st != Z_STREAM_END) || stream.total_out != delta_size)
1052                 die("delta data unpack failed");
1053
1054         result = patch_delta(base, base_size,
1055                              delta_data, delta_size,
1056                              &result_size);
1057         if (!result)
1058                 die("failed to apply delta");
1059         free(delta_data);
1060         free(base);
1061         *sizep = result_size;
1062         return result;
1063 }
1064
1065 static void *unpack_non_delta_entry(unsigned char *data,
1066                                     unsigned long size,
1067                                     unsigned long left)
1068 {
1069         int st;
1070         z_stream stream;
1071         unsigned char *buffer;
1072
1073         buffer = xmalloc(size + 1);
1074         buffer[size] = 0;
1075         memset(&stream, 0, sizeof(stream));
1076         stream.next_in = data;
1077         stream.avail_in = left;
1078         stream.next_out = buffer;
1079         stream.avail_out = size;
1080
1081         inflateInit(&stream);
1082         st = inflate(&stream, Z_FINISH);
1083         inflateEnd(&stream);
1084         if ((st != Z_STREAM_END) || stream.total_out != size) {
1085                 free(buffer);
1086                 return NULL;
1087         }
1088
1089         return buffer;
1090 }
1091
1092 static void *unpack_entry(struct pack_entry *entry,
1093                           char *type, unsigned long *sizep)
1094 {
1095         struct packed_git *p = entry->p;
1096         void *retval;
1097
1098         if (use_packed_git(p))
1099                 die("cannot map packed file");
1100         retval = unpack_entry_gently(entry, type, sizep);
1101         unuse_packed_git(p);
1102         if (!retval)
1103                 die("corrupted pack file %s", p->pack_name);
1104         return retval;
1105 }
1106
1107 /* The caller is responsible for use_packed_git()/unuse_packed_git() pair */
1108 void *unpack_entry_gently(struct pack_entry *entry,
1109                           char *type, unsigned long *sizep)
1110 {
1111         struct packed_git *p = entry->p;
1112         unsigned long offset, size, left;
1113         unsigned char *pack;
1114         enum object_type kind;
1115         void *retval;
1116
1117         offset = unpack_object_header(p, entry->offset, &kind, &size);
1118         pack = (unsigned char *) p->pack_base + offset;
1119         left = p->pack_size - offset;
1120         switch (kind) {
1121         case OBJ_DELTA:
1122                 retval = unpack_delta_entry(pack, size, left, type, sizep, p);
1123                 return retval;
1124         case OBJ_COMMIT:
1125                 strcpy(type, commit_type);
1126                 break;
1127         case OBJ_TREE:
1128                 strcpy(type, tree_type);
1129                 break;
1130         case OBJ_BLOB:
1131                 strcpy(type, blob_type);
1132                 break;
1133         case OBJ_TAG:
1134                 strcpy(type, tag_type);
1135                 break;
1136         default:
1137                 return NULL;
1138         }
1139         *sizep = size;
1140         retval = unpack_non_delta_entry(pack, size, left);
1141         return retval;
1142 }
1143
1144 int num_packed_objects(const struct packed_git *p)
1145 {
1146         /* See check_packed_git_idx() */
1147         return (p->index_size - 20 - 20 - 4*256) / 24;
1148 }
1149
1150 int nth_packed_object_sha1(const struct packed_git *p, int n,
1151                            unsigned char* sha1)
1152 {
1153         void *index = p->index_base + 256;
1154         if (n < 0 || num_packed_objects(p) <= n)
1155                 return -1;
1156         memcpy(sha1, (char *) index + (24 * n) + 4, 20);
1157         return 0;
1158 }
1159
1160 int find_pack_entry_one(const unsigned char *sha1,
1161                         struct pack_entry *e, struct packed_git *p)
1162 {
1163         unsigned int *level1_ofs = p->index_base;
1164         int hi = ntohl(level1_ofs[*sha1]);
1165         int lo = ((*sha1 == 0x0) ? 0 : ntohl(level1_ofs[*sha1 - 1]));
1166         void *index = p->index_base + 256;
1167
1168         do {
1169                 int mi = (lo + hi) / 2;
1170                 int cmp = memcmp((char *) index + (24 * mi) + 4, sha1, 20);
1171                 if (!cmp) {
1172                         e->offset = ntohl(*((unsigned int *) ((char *) index + (24 * mi))));
1173                         memcpy(e->sha1, sha1, 20);
1174                         e->p = p;
1175                         return 1;
1176                 }
1177                 if (cmp > 0)
1178                         hi = mi;
1179                 else
1180                         lo = mi+1;
1181         } while (lo < hi);
1182         return 0;
1183 }
1184
1185 static int find_pack_entry(const unsigned char *sha1, struct pack_entry *e)
1186 {
1187         struct packed_git *p;
1188         prepare_packed_git();
1189
1190         for (p = packed_git; p; p = p->next) {
1191                 if (find_pack_entry_one(sha1, e, p))
1192                         return 1;
1193         }
1194         return 0;
1195 }
1196
1197 struct packed_git *find_sha1_pack(const unsigned char *sha1,
1198                                   struct packed_git *packs)
1199 {
1200         struct packed_git *p;
1201         struct pack_entry e;
1202
1203         for (p = packs; p; p = p->next) {
1204                 if (find_pack_entry_one(sha1, &e, p))
1205                         return p;
1206         }
1207         return NULL;
1208
1209 }
1210
1211 int sha1_object_info(const unsigned char *sha1, char *type, unsigned long *sizep)
1212 {
1213         int status;
1214         unsigned long mapsize, size;
1215         void *map;
1216         z_stream stream;
1217         char hdr[128];
1218
1219         map = map_sha1_file_internal(sha1, &mapsize);
1220         if (!map) {
1221                 struct pack_entry e;
1222
1223                 if (find_pack_entry(sha1, &e))
1224                         return packed_object_info(&e, type, sizep);
1225                 reprepare_packed_git();
1226                 if (find_pack_entry(sha1, &e))
1227                         return packed_object_info(&e, type, sizep);
1228                 return error("unable to find %s", sha1_to_hex(sha1));
1229         }
1230         if (unpack_sha1_header(&stream, map, mapsize, hdr, sizeof(hdr)) < 0)
1231                 status = error("unable to unpack %s header",
1232                                sha1_to_hex(sha1));
1233         if (parse_sha1_header(hdr, type, &size) < 0)
1234                 status = error("unable to parse %s header", sha1_to_hex(sha1));
1235         else {
1236                 status = 0;
1237                 if (sizep)
1238                         *sizep = size;
1239         }
1240         inflateEnd(&stream);
1241         munmap(map, mapsize);
1242         return status;
1243 }
1244
1245 static void *read_packed_sha1(const unsigned char *sha1, char *type, unsigned long *size)
1246 {
1247         struct pack_entry e;
1248
1249         if (!find_pack_entry(sha1, &e)) {
1250                 error("cannot read sha1_file for %s", sha1_to_hex(sha1));
1251                 return NULL;
1252         }
1253         return unpack_entry(&e, type, size);
1254 }
1255
1256 void * read_sha1_file(const unsigned char *sha1, char *type, unsigned long *size)
1257 {
1258         unsigned long mapsize;
1259         void *map, *buf;
1260         struct pack_entry e;
1261
1262         if (find_pack_entry(sha1, &e))
1263                 return read_packed_sha1(sha1, type, size);
1264         map = map_sha1_file_internal(sha1, &mapsize);
1265         if (map) {
1266                 buf = unpack_sha1_file(map, mapsize, type, size);
1267                 munmap(map, mapsize);
1268                 return buf;
1269         }
1270         reprepare_packed_git();
1271         if (find_pack_entry(sha1, &e))
1272                 return read_packed_sha1(sha1, type, size);
1273         return NULL;
1274 }
1275
1276 void *read_object_with_reference(const unsigned char *sha1,
1277                                  const char *required_type,
1278                                  unsigned long *size,
1279                                  unsigned char *actual_sha1_return)
1280 {
1281         char type[20];
1282         void *buffer;
1283         unsigned long isize;
1284         unsigned char actual_sha1[20];
1285
1286         memcpy(actual_sha1, sha1, 20);
1287         while (1) {
1288                 int ref_length = -1;
1289                 const char *ref_type = NULL;
1290
1291                 buffer = read_sha1_file(actual_sha1, type, &isize);
1292                 if (!buffer)
1293                         return NULL;
1294                 if (!strcmp(type, required_type)) {
1295                         *size = isize;
1296                         if (actual_sha1_return)
1297                                 memcpy(actual_sha1_return, actual_sha1, 20);
1298                         return buffer;
1299                 }
1300                 /* Handle references */
1301                 else if (!strcmp(type, commit_type))
1302                         ref_type = "tree ";
1303                 else if (!strcmp(type, tag_type))
1304                         ref_type = "object ";
1305                 else {
1306                         free(buffer);
1307                         return NULL;
1308                 }
1309                 ref_length = strlen(ref_type);
1310
1311                 if (memcmp(buffer, ref_type, ref_length) ||
1312                     get_sha1_hex((char *) buffer + ref_length, actual_sha1)) {
1313                         free(buffer);
1314                         return NULL;
1315                 }
1316                 free(buffer);
1317                 /* Now we have the ID of the referred-to object in
1318                  * actual_sha1.  Check again. */
1319         }
1320 }
1321
1322 char *write_sha1_file_prepare(void *buf,
1323                               unsigned long len,
1324                               const char *type,
1325                               unsigned char *sha1,
1326                               unsigned char *hdr,
1327                               int *hdrlen)
1328 {
1329         SHA_CTX c;
1330
1331         /* Generate the header */
1332         *hdrlen = sprintf((char *)hdr, "%s %lu", type, len)+1;
1333
1334         /* Sha1.. */
1335         SHA1_Init(&c);
1336         SHA1_Update(&c, hdr, *hdrlen);
1337         SHA1_Update(&c, buf, len);
1338         SHA1_Final(sha1, &c);
1339
1340         return sha1_file_name(sha1);
1341 }
1342
1343 /*
1344  * Link the tempfile to the final place, possibly creating the
1345  * last directory level as you do so.
1346  *
1347  * Returns the errno on failure, 0 on success.
1348  */
1349 static int link_temp_to_file(const char *tmpfile, char *filename)
1350 {
1351         int ret;
1352
1353         if (!link(tmpfile, filename))
1354                 return 0;
1355
1356         /*
1357          * Try to mkdir the last path component if that failed
1358          * with an ENOENT.
1359          *
1360          * Re-try the "link()" regardless of whether the mkdir
1361          * succeeds, since a race might mean that somebody
1362          * else succeeded.
1363          */
1364         ret = errno;
1365         if (ret == ENOENT) {
1366                 char *dir = strrchr(filename, '/');
1367                 if (dir) {
1368                         *dir = 0;
1369                         mkdir(filename, 0777);
1370                         if (adjust_shared_perm(filename))
1371                                 return -2;
1372                         *dir = '/';
1373                         if (!link(tmpfile, filename))
1374                                 return 0;
1375                         ret = errno;
1376                 }
1377         }
1378         return ret;
1379 }
1380
1381 /*
1382  * Move the just written object into its final resting place
1383  */
1384 int move_temp_to_file(const char *tmpfile, char *filename)
1385 {
1386         int ret = link_temp_to_file(tmpfile, filename);
1387
1388         /*
1389          * Coda hack - coda doesn't like cross-directory links,
1390          * so we fall back to a rename, which will mean that it
1391          * won't be able to check collisions, but that's not a
1392          * big deal.
1393          *
1394          * The same holds for FAT formatted media.
1395          *
1396          * When this succeeds, we just return 0. We have nothing
1397          * left to unlink.
1398          */
1399         if (ret && ret != EEXIST) {
1400                 if (!rename(tmpfile, filename))
1401                         return 0;
1402                 ret = errno;
1403         }
1404         unlink(tmpfile);
1405         if (ret) {
1406                 if (ret != EEXIST) {
1407                         fprintf(stderr, "unable to write sha1 filename %s: %s\n", filename, strerror(ret));
1408                         return -1;
1409                 }
1410                 /* FIXME!!! Collision check here ? */
1411         }
1412
1413         return 0;
1414 }
1415
1416 static int write_buffer(int fd, const void *buf, size_t len)
1417 {
1418         while (len) {
1419                 ssize_t size;
1420
1421                 size = write(fd, buf, len);
1422                 if (!size)
1423                         return error("file write: disk full");
1424                 if (size < 0) {
1425                         if (errno == EINTR || errno == EAGAIN)
1426                                 continue;
1427                         return error("file write error (%s)", strerror(errno));
1428                 }
1429                 len -= size;
1430                 buf = (char *) buf + size;
1431         }
1432         return 0;
1433 }
1434
1435 int write_sha1_file(void *buf, unsigned long len, const char *type, unsigned char *returnsha1)
1436 {
1437         int size;
1438         unsigned char *compressed;
1439         z_stream stream;
1440         unsigned char sha1[20];
1441         char *filename;
1442         static char tmpfile[PATH_MAX];
1443         unsigned char hdr[50];
1444         int fd, hdrlen;
1445
1446         /* Normally if we have it in the pack then we do not bother writing
1447          * it out into .git/objects/??/?{38} file.
1448          */
1449         filename = write_sha1_file_prepare(buf, len, type, sha1, hdr, &hdrlen);
1450         if (returnsha1)
1451                 memcpy(returnsha1, sha1, 20);
1452         if (has_sha1_file(sha1))
1453                 return 0;
1454         fd = open(filename, O_RDONLY);
1455         if (fd >= 0) {
1456                 /*
1457                  * FIXME!!! We might do collision checking here, but we'd
1458                  * need to uncompress the old file and check it. Later.
1459                  */
1460                 close(fd);
1461                 return 0;
1462         }
1463
1464         if (errno != ENOENT) {
1465                 fprintf(stderr, "sha1 file %s: %s\n", filename, strerror(errno));
1466                 return -1;
1467         }
1468
1469         snprintf(tmpfile, sizeof(tmpfile), "%s/obj_XXXXXX", get_object_directory());
1470
1471         fd = mkstemp(tmpfile);
1472         if (fd < 0) {
1473                 fprintf(stderr, "unable to create temporary sha1 filename %s: %s\n", tmpfile, strerror(errno));
1474                 return -1;
1475         }
1476
1477         /* Set it up */
1478         memset(&stream, 0, sizeof(stream));
1479         deflateInit(&stream, Z_BEST_COMPRESSION);
1480         size = deflateBound(&stream, len+hdrlen);
1481         compressed = xmalloc(size);
1482
1483         /* Compress it */
1484         stream.next_out = compressed;
1485         stream.avail_out = size;
1486
1487         /* First header.. */
1488         stream.next_in = hdr;
1489         stream.avail_in = hdrlen;
1490         while (deflate(&stream, 0) == Z_OK)
1491                 /* nothing */;
1492
1493         /* Then the data itself.. */
1494         stream.next_in = buf;
1495         stream.avail_in = len;
1496         while (deflate(&stream, Z_FINISH) == Z_OK)
1497                 /* nothing */;
1498         deflateEnd(&stream);
1499         size = stream.total_out;
1500
1501         if (write_buffer(fd, compressed, size) < 0)
1502                 die("unable to write sha1 file");
1503         fchmod(fd, 0444);
1504         close(fd);
1505         free(compressed);
1506
1507         return move_temp_to_file(tmpfile, filename);
1508 }
1509
1510 /*
1511  * We need to unpack and recompress the object for writing
1512  * it out to a different file.
1513  */
1514 static void *repack_object(const unsigned char *sha1, unsigned long *objsize)
1515 {
1516         size_t size;
1517         z_stream stream;
1518         unsigned char *unpacked;
1519         unsigned long len;
1520         char type[20];
1521         char hdr[50];
1522         int hdrlen;
1523         void *buf;
1524
1525         // need to unpack and recompress it by itself
1526         unpacked = read_packed_sha1(sha1, type, &len);
1527
1528         hdrlen = sprintf(hdr, "%s %lu", type, len) + 1;
1529
1530         /* Set it up */
1531         memset(&stream, 0, sizeof(stream));
1532         deflateInit(&stream, Z_BEST_COMPRESSION);
1533         size = deflateBound(&stream, len + hdrlen);
1534         buf = xmalloc(size);
1535
1536         /* Compress it */
1537         stream.next_out = buf;
1538         stream.avail_out = size;
1539
1540         /* First header.. */
1541         stream.next_in = (void *)hdr;
1542         stream.avail_in = hdrlen;
1543         while (deflate(&stream, 0) == Z_OK)
1544                 /* nothing */;
1545
1546         /* Then the data itself.. */
1547         stream.next_in = unpacked;
1548         stream.avail_in = len;
1549         while (deflate(&stream, Z_FINISH) == Z_OK)
1550                 /* nothing */;
1551         deflateEnd(&stream);
1552         free(unpacked);
1553
1554         *objsize = stream.total_out;
1555         return buf;
1556 }
1557
1558 int write_sha1_to_fd(int fd, const unsigned char *sha1)
1559 {
1560         int retval;
1561         unsigned long objsize;
1562         void *buf = map_sha1_file_internal(sha1, &objsize);
1563
1564         if (buf) {
1565                 retval = write_buffer(fd, buf, objsize);
1566                 munmap(buf, objsize);
1567                 return retval;
1568         }
1569
1570         buf = repack_object(sha1, &objsize);
1571         retval = write_buffer(fd, buf, objsize);
1572         free(buf);
1573         return retval;
1574 }
1575
1576 int write_sha1_from_fd(const unsigned char *sha1, int fd, char *buffer,
1577                        size_t bufsize, size_t *bufposn)
1578 {
1579         char tmpfile[PATH_MAX];
1580         int local;
1581         z_stream stream;
1582         unsigned char real_sha1[20];
1583         unsigned char discard[4096];
1584         int ret;
1585         SHA_CTX c;
1586
1587         snprintf(tmpfile, sizeof(tmpfile), "%s/obj_XXXXXX", get_object_directory());
1588
1589         local = mkstemp(tmpfile);
1590         if (local < 0)
1591                 return error("Couldn't open %s for %s",
1592                              tmpfile, sha1_to_hex(sha1));
1593
1594         memset(&stream, 0, sizeof(stream));
1595
1596         inflateInit(&stream);
1597
1598         SHA1_Init(&c);
1599
1600         do {
1601                 ssize_t size;
1602                 if (*bufposn) {
1603                         stream.avail_in = *bufposn;
1604                         stream.next_in = (unsigned char *) buffer;
1605                         do {
1606                                 stream.next_out = discard;
1607                                 stream.avail_out = sizeof(discard);
1608                                 ret = inflate(&stream, Z_SYNC_FLUSH);
1609                                 SHA1_Update(&c, discard, sizeof(discard) -
1610                                             stream.avail_out);
1611                         } while (stream.avail_in && ret == Z_OK);
1612                         if (write_buffer(local, buffer, *bufposn - stream.avail_in) < 0)
1613                                 die("unable to write sha1 file");
1614                         memmove(buffer, buffer + *bufposn - stream.avail_in,
1615                                 stream.avail_in);
1616                         *bufposn = stream.avail_in;
1617                         if (ret != Z_OK)
1618                                 break;
1619                 }
1620                 size = read(fd, buffer + *bufposn, bufsize - *bufposn);
1621                 if (size <= 0) {
1622                         close(local);
1623                         unlink(tmpfile);
1624                         if (!size)
1625                                 return error("Connection closed?");
1626                         perror("Reading from connection");
1627                         return -1;
1628                 }
1629                 *bufposn += size;
1630         } while (1);
1631         inflateEnd(&stream);
1632
1633         close(local);
1634         SHA1_Final(real_sha1, &c);
1635         if (ret != Z_STREAM_END) {
1636                 unlink(tmpfile);
1637                 return error("File %s corrupted", sha1_to_hex(sha1));
1638         }
1639         if (memcmp(sha1, real_sha1, 20)) {
1640                 unlink(tmpfile);
1641                 return error("File %s has bad hash", sha1_to_hex(sha1));
1642         }
1643
1644         return move_temp_to_file(tmpfile, sha1_file_name(sha1));
1645 }
1646
1647 int has_pack_index(const unsigned char *sha1)
1648 {
1649         struct stat st;
1650         if (stat(sha1_pack_index_name(sha1), &st))
1651                 return 0;
1652         return 1;
1653 }
1654
1655 int has_pack_file(const unsigned char *sha1)
1656 {
1657         struct stat st;
1658         if (stat(sha1_pack_name(sha1), &st))
1659                 return 0;
1660         return 1;
1661 }
1662
1663 int has_sha1_pack(const unsigned char *sha1)
1664 {
1665         struct pack_entry e;
1666         return find_pack_entry(sha1, &e);
1667 }
1668
1669 int has_sha1_file(const unsigned char *sha1)
1670 {
1671         struct stat st;
1672         struct pack_entry e;
1673
1674         if (find_pack_entry(sha1, &e))
1675                 return 1;
1676         return find_sha1_file(sha1, &st) ? 1 : 0;
1677 }
1678
1679 /*
1680  * reads from fd as long as possible into a supplied buffer of size bytes.
1681  * If neccessary the buffer's size is increased using realloc()
1682  *
1683  * returns 0 if anything went fine and -1 otherwise
1684  *
1685  * NOTE: both buf and size may change, but even when -1 is returned
1686  * you still have to free() it yourself.
1687  */
1688 int read_pipe(int fd, char** return_buf, unsigned long* return_size)
1689 {
1690         char* buf = *return_buf;
1691         unsigned long size = *return_size;
1692         int iret;
1693         unsigned long off = 0;
1694
1695         do {
1696                 iret = xread(fd, buf + off, size - off);
1697                 if (iret > 0) {
1698                         off += iret;
1699                         if (off == size) {
1700                                 size *= 2;
1701                                 buf = realloc(buf, size);
1702                         }
1703                 }
1704         } while (iret > 0);
1705
1706         *return_buf = buf;
1707         *return_size = off;
1708
1709         if (iret < 0)
1710                 return -1;
1711         return 0;
1712 }
1713
1714 int index_pipe(unsigned char *sha1, int fd, const char *type, int write_object)
1715 {
1716         unsigned long size = 4096;
1717         char *buf = malloc(size);
1718         int ret;
1719         unsigned char hdr[50];
1720         int hdrlen;
1721
1722         if (read_pipe(fd, &buf, &size)) {
1723                 free(buf);
1724                 return -1;
1725         }
1726
1727         if (!type)
1728                 type = blob_type;
1729         if (write_object)
1730                 ret = write_sha1_file(buf, size, type, sha1);
1731         else {
1732                 write_sha1_file_prepare(buf, size, type, sha1, hdr, &hdrlen);
1733                 ret = 0;
1734         }
1735         free(buf);
1736         return ret;
1737 }
1738
1739 int index_fd(unsigned char *sha1, int fd, struct stat *st, int write_object, const char *type)
1740 {
1741         unsigned long size = st->st_size;
1742         void *buf;
1743         int ret;
1744         unsigned char hdr[50];
1745         int hdrlen;
1746
1747         buf = "";
1748         if (size)
1749                 buf = mmap(NULL, size, PROT_READ, MAP_PRIVATE, fd, 0);
1750         close(fd);
1751         if (buf == MAP_FAILED)
1752                 return -1;
1753
1754         if (!type)
1755                 type = blob_type;
1756         if (write_object)
1757                 ret = write_sha1_file(buf, size, type, sha1);
1758         else {
1759                 write_sha1_file_prepare(buf, size, type, sha1, hdr, &hdrlen);
1760                 ret = 0;
1761         }
1762         if (size)
1763                 munmap(buf, size);
1764         return ret;
1765 }
1766
1767 int index_path(unsigned char *sha1, const char *path, struct stat *st, int write_object)
1768 {
1769         int fd;
1770         char *target;
1771
1772         switch (st->st_mode & S_IFMT) {
1773         case S_IFREG:
1774                 fd = open(path, O_RDONLY);
1775                 if (fd < 0)
1776                         return error("open(\"%s\"): %s", path,
1777                                      strerror(errno));
1778                 if (index_fd(sha1, fd, st, write_object, NULL) < 0)
1779                         return error("%s: failed to insert into database",
1780                                      path);
1781                 break;
1782         case S_IFLNK:
1783                 target = xmalloc(st->st_size+1);
1784                 if (readlink(path, target, st->st_size+1) != st->st_size) {
1785                         char *errstr = strerror(errno);
1786                         free(target);
1787                         return error("readlink(\"%s\"): %s", path,
1788                                      errstr);
1789                 }
1790                 if (!write_object) {
1791                         unsigned char hdr[50];
1792                         int hdrlen;
1793                         write_sha1_file_prepare(target, st->st_size, blob_type,
1794                                                 sha1, hdr, &hdrlen);
1795                 } else if (write_sha1_file(target, st->st_size, blob_type, sha1))
1796                         return error("%s: failed to insert into database",
1797                                      path);
1798                 free(target);
1799                 break;
1800         default:
1801                 return error("%s: unsupported file type", path);
1802         }
1803         return 0;
1804 }