git.oblomov.eu Git - git/blob - fast-import.c

   1 /*
   2 Format of STDIN stream:
   3
   4   stream ::= cmd*;
   5
   6   cmd ::= new_blob
   7         | new_branch
   8         | new_commit
   9         | new_tag
  10         ;
  11
  12   new_blob ::= 'blob' lf
  13         mark?
  14     file_content;
  15   file_content ::= data;
  16
  17   new_branch ::= 'branch' sp ref_str lf
  18     ('from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf)?
  19     lf;
  20
  21   new_commit ::= 'commit' sp ref_str lf
  22         mark?
  23         ('author' sp name '<' email '>' ts tz lf)?
  24         'committer' sp name '<' email '>' ts tz lf
  25         commit_msg
  26     file_change*
  27     lf;
  28   commit_msg ::= data;
  29
  30   file_change ::= 'M' sp mode sp (hexsha1 | idnum) sp path_str lf
  31                 | 'D' sp path_str lf
  32                 ;
  33   mode ::= '644' | '755';
  34
  35   new_tag ::= 'tag' sp tag_str lf
  36     'from' sp (ref_str | hexsha1 | sha1exp_str | idnum) lf
  37         'tagger' sp name '<' email '>' ts tz lf
  38     tag_msg;
  39   tag_msg ::= data;
  40
  41      # note: the first idnum in a stream should be 1 and subsequent
  42      # idnums should not have gaps between values as this will cause
  43      # the stream parser to reserve space for the gapped values.  An
  44          # idnum can be updated in the future to a new object by issuing
  45      # a new mark directive with the old idnum.
  46          #
  47   mark ::= 'mark' sp idnum lf;
  48
  49      # note: declen indicates the length of binary_data in bytes.
  50      # declen does not include the lf preceeding or trailing the
  51      # binary data.
  52      #
  53   data ::= 'data' sp declen lf
  54     binary_data
  55         lf;
  56
  57      # note: quoted strings are C-style quoting supporting \c for
  58      # common escapes of 'c' (e..g \n, \t, \\, \") or \nnn where nnn
  59          # is the signed byte value in octal.  Note that the only
  60      # characters which must actually be escaped to protect the
  61      # stream formatting is: \, " and LF.  Otherwise these values
  62          # are UTF8.
  63      #
  64   ref_str     ::= ref     | '"' quoted(ref)     '"' ;
  65   sha1exp_str ::= sha1exp | '"' quoted(sha1exp) '"' ;
  66   tag_str     ::= tag     | '"' quoted(tag)     '"' ;
  67   path_str    ::= path    | '"' quoted(path)    '"' ;
  68
  69   declen ::= # unsigned 32 bit value, ascii base10 notation;
  70   binary_data ::= # file content, not interpreted;
  71
  72   sp ::= # ASCII space character;
  73   lf ::= # ASCII newline (LF) character;
  74
  75      # note: a colon (':') must precede the numerical value assigned to
  76          # an idnum.  This is to distinguish it from a ref or tag name as
  77      # GIT does not permit ':' in ref or tag strings.
  78          #
  79   idnum   ::= ':' declen;
  80   path    ::= # GIT style file path, e.g. "a/b/c";
  81   ref     ::= # GIT ref name, e.g. "refs/heads/MOZ_GECKO_EXPERIMENT";
  82   tag     ::= # GIT tag name, e.g. "FIREFOX_1_5";
  83   sha1exp ::= # Any valid GIT SHA1 expression;
  84   hexsha1 ::= # SHA1 in hexadecimal format;
  85
  86      # note: name and email are UTF8 strings, however name must not
  87          # contain '<' or lf and email must not contain any of the
  88      # following: '<', '>', lf.
  89          #
  90   name  ::= # valid GIT author/committer name;
  91   email ::= # valid GIT author/committer email;
  92   ts    ::= # time since the epoch in seconds, ascii base10 notation;
  93   tz    ::= # GIT style timezone;
  94 */
  95
  96 #include "builtin.h"
  97 #include "cache.h"
  98 #include "object.h"
  99 #include "blob.h"
 100 #include "tree.h"
 101 #include "delta.h"
 102 #include "pack.h"
 103 #include "refs.h"
 104 #include "csum-file.h"
 105 #include "strbuf.h"
 106 #include "quote.h"
 107
 108 struct object_entry
 109 {
 110         struct object_entry *next;
 111         enum object_type type;
 112         unsigned long offset;
 113         unsigned char sha1[20];
 114 };
 115
 116 struct object_entry_pool
 117 {
 118         struct object_entry_pool *next_pool;
 119         struct object_entry *next_free;
 120         struct object_entry *end;
 121         struct object_entry entries[FLEX_ARRAY]; /* more */
 122 };
 123
 124 struct last_object
 125 {
 126         void *data;
 127         unsigned int len;
 128         unsigned int depth;
 129         unsigned char sha1[20];
 130 };
 131
 132 struct mem_pool
 133 {
 134         struct mem_pool *next_pool;
 135         char *next_free;
 136         char *end;
 137         char space[FLEX_ARRAY]; /* more */
 138 };
 139
 140 struct atom_str
 141 {
 142         struct atom_str *next_atom;
 143         int str_len;
 144         char str_dat[FLEX_ARRAY]; /* more */
 145 };
 146
 147 struct tree_content;
 148 struct tree_entry
 149 {
 150         struct tree_content *tree;
 151         struct atom_str* name;
 152         unsigned int mode;
 153         unsigned char sha1[20];
 154 };
 155
 156 struct tree_content
 157 {
 158         unsigned int entry_capacity; /* must match avail_tree_content */
 159         unsigned int entry_count;
 160         struct tree_entry *entries[FLEX_ARRAY]; /* more */
 161 };
 162
 163 struct avail_tree_content
 164 {
 165         unsigned int entry_capacity; /* must match tree_content */
 166         struct avail_tree_content *next_avail;
 167 };
 168
 169 struct branch
 170 {
 171         struct branch *table_next_branch;
 172         struct branch *active_next_branch;
 173         const char *name;
 174         unsigned long last_commit;
 175         struct tree_entry branch_tree;
 176         unsigned char sha1[20];
 177 };
 178
 179
 180 /* Stats and misc. counters */
 181 static int max_depth = 10;
 182 static unsigned long alloc_count;
 183 static unsigned long branch_count;
 184 static unsigned long object_count;
 185 static unsigned long duplicate_count;
 186 static unsigned long object_count_by_type[9];
 187 static unsigned long duplicate_count_by_type[9];
 188
 189 /* Memory pools */
 190 static size_t mem_pool_alloc = 2*1024*1024 - sizeof(struct mem_pool);
 191 static size_t total_allocd;
 192 static struct mem_pool *mem_pool;
 193
 194 /* Atom management */
 195 static unsigned int atom_table_sz = 4451;
 196 static unsigned int atom_cnt;
 197 static struct atom_str **atom_table;
 198
 199 /* The .pack file being generated */
 200 static int pack_fd;
 201 static unsigned long pack_offset;
 202 static unsigned char pack_sha1[20];
 203
 204 /* Table of objects we've written. */
 205 static unsigned int object_entry_alloc = 1000;
 206 static struct object_entry_pool *blocks;
 207 static struct object_entry *object_table[1 << 16];
 208
 209 /* Our last blob */
 210 static struct last_object last_blob;
 211
 212 /* Tree management */
 213 static unsigned int tree_entry_alloc = 1000;
 214 static void *avail_tree_entry;
 215 static unsigned int avail_tree_table_sz = 100;
 216 static struct avail_tree_content **avail_tree_table;
 217
 218 /* Branch data */
 219 static unsigned int max_active_branches = 5;
 220 static unsigned int cur_active_branches;
 221 static unsigned int branch_table_sz = 1039;
 222 static struct branch **branch_table;
 223 static struct branch *active_branches;
 224
 225 /* Input stream parsing */
 226 static struct strbuf command_buf;
 227 static unsigned long command_mark;
 228
 229
 230 static void alloc_objects(int cnt)
 231 {
 232         struct object_entry_pool *b;
 233
 234         b = xmalloc(sizeof(struct object_entry_pool)
 235                 + cnt * sizeof(struct object_entry));
 236         b->next_pool = blocks;
 237         b->next_free = b->entries;
 238         b->end = b->entries + cnt;
 239         blocks = b;
 240         alloc_count += cnt;
 241 }
 242
 243 static struct object_entry* new_object(unsigned char *sha1)
 244 {
 245         struct object_entry *e;
 246
 247         if (blocks->next_free == blocks->end)
 248                 alloc_objects(object_entry_alloc);
 249
 250         e = blocks->next_free++;
 251         memcpy(e->sha1, sha1, sizeof(e->sha1));
 252         return e;
 253 }
 254
 255 static struct object_entry* find_object(unsigned char *sha1)
 256 {
 257         unsigned int h = sha1[0] << 8 | sha1[1];
 258         struct object_entry *e;
 259         for (e = object_table[h]; e; e = e->next)
 260                 if (!memcmp(sha1, e->sha1, sizeof(e->sha1)))
 261                         return e;
 262         return NULL;
 263 }
 264
 265 static struct object_entry* insert_object(unsigned char *sha1)
 266 {
 267         unsigned int h = sha1[0] << 8 | sha1[1];
 268         struct object_entry *e = object_table[h];
 269         struct object_entry *p = NULL;
 270
 271         while (e) {
 272                 if (!memcmp(sha1, e->sha1, sizeof(e->sha1)))
 273                         return e;
 274                 p = e;
 275                 e = e->next;
 276         }
 277
 278         e = new_object(sha1);
 279         e->next = NULL;
 280         e->offset = 0;
 281         if (p)
 282                 p->next = e;
 283         else
 284                 object_table[h] = e;
 285         return e;
 286 }
 287
 288 static unsigned int hc_str(const char *s, size_t len)
 289 {
 290         unsigned int r = 0;
 291         while (len-- > 0)
 292                 r = r * 31 + *s++;
 293         return r;
 294 }
 295
 296 static void* pool_alloc(size_t len)
 297 {
 298         struct mem_pool *p;
 299         void *r;
 300
 301         for (p = mem_pool; p; p = p->next_pool)
 302                 if ((p->end - p->next_free >= len))
 303                         break;
 304
 305         if (!p) {
 306                 if (len >= (mem_pool_alloc/2)) {
 307                         total_allocd += len;
 308                         return xmalloc(len);
 309                 }
 310                 total_allocd += sizeof(struct mem_pool) + mem_pool_alloc;
 311                 p = xmalloc(sizeof(struct mem_pool) + mem_pool_alloc);
 312                 p->next_pool = mem_pool;
 313                 p->next_free = p->space;
 314                 p->end = p->next_free + mem_pool_alloc;
 315                 mem_pool = p;
 316         }
 317
 318         r = p->next_free;
 319         p->next_free += len;
 320         return r;
 321 }
 322
 323 static void* pool_calloc(size_t count, size_t size)
 324 {
 325         size_t len = count * size;
 326         void *r = pool_alloc(len);
 327         memset(r, 0, len);
 328         return r;
 329 }
 330
 331 static char* pool_strdup(const char *s)
 332 {
 333         char *r = pool_alloc(strlen(s) + 1);
 334         strcpy(r, s);
 335         return r;
 336 }
 337
 338 static struct atom_str* to_atom(const char *s, size_t len)
 339 {
 340         unsigned int hc = hc_str(s, len) % atom_table_sz;
 341         struct atom_str *c;
 342
 343         for (c = atom_table[hc]; c; c = c->next_atom)
 344                 if (c->str_len == len && !strncmp(s, c->str_dat, len))
 345                         return c;
 346
 347         c = pool_alloc(sizeof(struct atom_str) + len + 1);
 348         c->str_len = len;
 349         strncpy(c->str_dat, s, len);
 350         c->str_dat[len] = 0;
 351         c->next_atom = atom_table[hc];
 352         atom_table[hc] = c;
 353         atom_cnt++;
 354         return c;
 355 }
 356
 357 static struct branch* lookup_branch(const char *name)
 358 {
 359         unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
 360         struct branch *b;
 361
 362         for (b = branch_table[hc]; b; b = b->table_next_branch)
 363                 if (!strcmp(name, b->name))
 364                         return b;
 365         return NULL;
 366 }
 367
 368 static struct branch* new_branch(const char *name)
 369 {
 370         unsigned int hc = hc_str(name, strlen(name)) % branch_table_sz;
 371         struct branch* b = lookup_branch(name);
 372
 373         if (b)
 374                 die("Invalid attempt to create duplicate branch: %s", name);
 375         if (check_ref_format(name))
 376                 die("Branch name doesn't conform to GIT standards: %s", name);
 377
 378         b = pool_calloc(1, sizeof(struct branch));
 379         b->name = pool_strdup(name);
 380         b->table_next_branch = branch_table[hc];
 381         branch_table[hc] = b;
 382         branch_count++;
 383         return b;
 384 }
 385
 386 static unsigned int hc_entries(unsigned int cnt)
 387 {
 388         cnt = cnt & 7 ? (cnt / 8) + 1 : cnt / 8;
 389         return cnt < avail_tree_table_sz ? cnt : avail_tree_table_sz - 1;
 390 }
 391
 392 static struct tree_content* new_tree_content(unsigned int cnt)
 393 {
 394         struct avail_tree_content *f, *l = NULL;
 395         struct tree_content *t;
 396         unsigned int hc = hc_entries(cnt);
 397
 398         for (f = avail_tree_table[hc]; f; l = f, f = f->next_avail)
 399                 if (f->entry_capacity >= cnt)
 400                         break;
 401
 402         if (f) {
 403                 if (l)
 404                         l->next_avail = f->next_avail;
 405                 else
 406                         avail_tree_table[hc] = f->next_avail;
 407         } else {
 408                 cnt = cnt & 7 ? ((cnt / 8) + 1) * 8 : cnt;
 409                 f = pool_alloc(sizeof(*t) + sizeof(t->entries[0]) * cnt);
 410                 f->entry_capacity = cnt;
 411         }
 412
 413         t = (struct tree_content*)f;
 414         t->entry_count = 0;
 415         return t;
 416 }
 417
 418 static void release_tree_entry(struct tree_entry *e);
 419 static void release_tree_content(struct tree_content *t)
 420 {
 421         struct avail_tree_content *f = (struct avail_tree_content*)t;
 422         unsigned int hc = hc_entries(f->entry_capacity);
 423         unsigned int i;
 424         for (i = 0; i < t->entry_count; i++)
 425                 release_tree_entry(t->entries[i]);
 426         f->next_avail = avail_tree_table[hc];
 427         avail_tree_table[hc] = f;
 428 }
 429
 430 static struct tree_content* grow_tree_content(
 431         struct tree_content *t,
 432         int amt)
 433 {
 434         struct tree_content *r = new_tree_content(t->entry_count + amt);
 435         r->entry_count = t->entry_count;
 436         memcpy(r->entries,t->entries,t->entry_count*sizeof(t->entries[0]));
 437         release_tree_content(t);
 438         return r;
 439 }
 440
 441 static struct tree_entry* new_tree_entry()
 442 {
 443         struct tree_entry *e;
 444
 445         if (!avail_tree_entry) {
 446                 unsigned int n = tree_entry_alloc;
 447                 avail_tree_entry = e = xmalloc(n * sizeof(struct tree_entry));
 448                 while (n--) {
 449                         *((void**)e) = e + 1;
 450                         e++;
 451                 }
 452         }
 453
 454         e = avail_tree_entry;
 455         avail_tree_entry = *((void**)e);
 456         return e;
 457 }
 458
 459 static void release_tree_entry(struct tree_entry *e)
 460 {
 461         if (e->tree)
 462                 release_tree_content(e->tree);
 463         *((void**)e) = avail_tree_entry;
 464         avail_tree_entry = e;
 465 }
 466
 467 static void yread(int fd, void *buffer, size_t length)
 468 {
 469         ssize_t ret = 0;
 470         while (ret < length) {
 471                 ssize_t size = xread(fd, (char *) buffer + ret, length - ret);
 472                 if (!size)
 473                         die("Read from descriptor %i: end of stream", fd);
 474                 if (size < 0)
 475                         die("Read from descriptor %i: %s", fd, strerror(errno));
 476                 ret += size;
 477         }
 478 }
 479
 480 static void ywrite(int fd, void *buffer, size_t length)
 481 {
 482         ssize_t ret = 0;
 483         while (ret < length) {
 484                 ssize_t size = xwrite(fd, (char *) buffer + ret, length - ret);
 485                 if (!size)
 486                         die("Write to descriptor %i: end of file", fd);
 487                 if (size < 0)
 488                         die("Write to descriptor %i: %s", fd, strerror(errno));
 489                 ret += size;
 490         }
 491 }
 492
 493 static size_t encode_header(
 494         enum object_type type,
 495         size_t size,
 496         unsigned char *hdr)
 497 {
 498         int n = 1;
 499         unsigned char c;
 500
 501         if (type < OBJ_COMMIT || type > OBJ_DELTA)
 502                 die("bad type %d", type);
 503
 504         c = (type << 4) | (size & 15);
 505         size >>= 4;
 506         while (size) {
 507                 *hdr++ = c | 0x80;
 508                 c = size & 0x7f;
 509                 size >>= 7;
 510                 n++;
 511         }
 512         *hdr = c;
 513         return n;
 514 }
 515
 516 static int store_object(
 517         enum object_type type,
 518         void *dat,
 519         size_t datlen,
 520         struct last_object *last,
 521         unsigned char *sha1out)
 522 {
 523         void *out, *delta;
 524         struct object_entry *e;
 525         unsigned char hdr[96];
 526         unsigned char sha1[20];
 527         unsigned long hdrlen, deltalen;
 528         SHA_CTX c;
 529         z_stream s;
 530
 531         hdrlen = sprintf((char*)hdr,"%s %lu",type_names[type],datlen) + 1;
 532         SHA1_Init(&c);
 533         SHA1_Update(&c, hdr, hdrlen);
 534         SHA1_Update(&c, dat, datlen);
 535         SHA1_Final(sha1, &c);
 536         if (sha1out)
 537                 memcpy(sha1out, sha1, sizeof(sha1));
 538
 539         e = insert_object(sha1);
 540         if (e->offset) {
 541                 duplicate_count++;
 542                 duplicate_count_by_type[type]++;
 543                 return 1;
 544         }
 545         e->type = type;
 546         e->offset = pack_offset;
 547         object_count++;
 548         object_count_by_type[type]++;
 549
 550         if (last && last->data && last->depth < max_depth)
 551                 delta = diff_delta(last->data, last->len,
 552                         dat, datlen,
 553                         &deltalen, 0);
 554         else
 555                 delta = 0;
 556
 557         memset(&s, 0, sizeof(s));
 558         deflateInit(&s, zlib_compression_level);
 559
 560         if (delta) {
 561                 last->depth++;
 562                 s.next_in = delta;
 563                 s.avail_in = deltalen;
 564                 hdrlen = encode_header(OBJ_DELTA, deltalen, hdr);
 565                 ywrite(pack_fd, hdr, hdrlen);
 566                 ywrite(pack_fd, last->sha1, sizeof(sha1));
 567                 pack_offset += hdrlen + sizeof(sha1);
 568         } else {
 569                 if (last)
 570                         last->depth = 0;
 571                 s.next_in = dat;
 572                 s.avail_in = datlen;
 573                 hdrlen = encode_header(type, datlen, hdr);
 574                 ywrite(pack_fd, hdr, hdrlen);
 575                 pack_offset += hdrlen;
 576         }
 577
 578         s.avail_out = deflateBound(&s, s.avail_in);
 579         s.next_out = out = xmalloc(s.avail_out);
 580         while (deflate(&s, Z_FINISH) == Z_OK)
 581                 /* nothing */;
 582         deflateEnd(&s);
 583
 584         ywrite(pack_fd, out, s.total_out);
 585         pack_offset += s.total_out;
 586
 587         free(out);
 588         if (delta)
 589                 free(delta);
 590         if (last) {
 591                 if (last->data)
 592                         free(last->data);
 593                 last->data = dat;
 594                 last->len = datlen;
 595                 memcpy(last->sha1, sha1, sizeof(sha1));
 596         }
 597         return 0;
 598 }
 599
 600 static const char *get_mode(const char *str, unsigned int *modep)
 601 {
 602         unsigned char c;
 603         unsigned int mode = 0;
 604
 605         while ((c = *str++) != ' ') {
 606                 if (c < '0' || c > '7')
 607                         return NULL;
 608                 mode = (mode << 3) + (c - '0');
 609         }
 610         *modep = mode;
 611         return str;
 612 }
 613
 614 static void load_tree(struct tree_entry *root)
 615 {
 616         struct object_entry *myoe;
 617         struct tree_content *t;
 618         unsigned long size;
 619         char *buf;
 620         const char *c;
 621         char type[20];
 622
 623         root->tree = t = new_tree_content(8);
 624         if (!memcmp(root->sha1, null_sha1, 20))
 625                 return;
 626
 627         myoe = find_object(root->sha1);
 628         if (myoe) {
 629                 die("FIXME");
 630         } else {
 631                 buf = read_sha1_file(root->sha1, type, &size);
 632                 if (!buf || strcmp(type, tree_type))
 633                         die("Can't load existing tree %s", sha1_to_hex(root->sha1));
 634         }
 635
 636         c = buf;
 637         while (c != (buf + size)) {
 638                 struct tree_entry *e = new_tree_entry();
 639
 640                 if (t->entry_count == t->entry_capacity)
 641                         root->tree = t = grow_tree_content(t, 8);
 642                 t->entries[t->entry_count++] = e;
 643
 644                 e->tree = NULL;
 645                 c = get_mode(c, &e->mode);
 646                 if (!c)
 647                         die("Corrupt mode in %s", sha1_to_hex(root->sha1));
 648                 e->name = to_atom(c, strlen(c));
 649                 c += e->name->str_len + 1;
 650                 memcpy(e->sha1, c, sizeof(e->sha1));
 651                 c += 20;
 652         }
 653         free(buf);
 654 }
 655
 656 static int tecmp (const void *_a, const void *_b)
 657 {
 658         struct tree_entry *a = *((struct tree_entry**)_a);
 659         struct tree_entry *b = *((struct tree_entry**)_b);
 660         return base_name_compare(
 661                 a->name->str_dat, a->name->str_len, a->mode,
 662                 b->name->str_dat, b->name->str_len, b->mode);
 663 }
 664
 665 static void store_tree(struct tree_entry *root)
 666 {
 667         struct tree_content *t = root->tree;
 668         unsigned int i;
 669         size_t maxlen;
 670         char *buf, *c;
 671
 672         if (memcmp(root->sha1, null_sha1, 20))
 673                 return;
 674
 675         maxlen = 0;
 676         for (i = 0; i < t->entry_count; i++) {
 677                 maxlen += t->entries[i]->name->str_len + 34;
 678                 if (t->entries[i]->tree)
 679                         store_tree(t->entries[i]);
 680         }
 681
 682         qsort(t->entries, t->entry_count, sizeof(t->entries[0]), tecmp);
 683         buf = c = xmalloc(maxlen);
 684         for (i = 0; i < t->entry_count; i++) {
 685                 struct tree_entry *e = t->entries[i];
 686                 c += sprintf(c, "%o", e->mode);
 687                 *c++ = ' ';
 688                 strcpy(c, e->name->str_dat);
 689                 c += e->name->str_len + 1;
 690                 memcpy(c, e->sha1, 20);
 691                 c += 20;
 692         }
 693         store_object(OBJ_TREE, buf, c - buf, NULL, root->sha1);
 694         free(buf);
 695 }
 696
 697 static int tree_content_set(
 698         struct tree_entry *root,
 699         const char *p,
 700         const unsigned char *sha1,
 701         const unsigned int mode)
 702 {
 703         struct tree_content *t = root->tree;
 704         const char *slash1;
 705         unsigned int i, n;
 706         struct tree_entry *e;
 707
 708         slash1 = strchr(p, '/');
 709         if (slash1)
 710                 n = slash1 - p;
 711         else
 712                 n = strlen(p);
 713
 714         for (i = 0; i < t->entry_count; i++) {
 715                 e = t->entries[i];
 716                 if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
 717                         if (!slash1) {
 718                                 if (e->mode == mode && !memcmp(e->sha1, sha1, 20))
 719                                         return 0;
 720                                 e->mode = mode;
 721                                 memcpy(e->sha1, sha1, 20);
 722                                 if (e->tree) {
 723                                         release_tree_content(e->tree);
 724                                         e->tree = NULL;
 725                                 }
 726                                 memcpy(root->sha1, null_sha1, 20);
 727                                 return 1;
 728                         }
 729                         if (!S_ISDIR(e->mode)) {
 730                                 e->tree = new_tree_content(8);
 731                                 e->mode = S_IFDIR;
 732                         }
 733                         if (!e->tree)
 734                                 load_tree(e);
 735                         if (tree_content_set(e, slash1 + 1, sha1, mode)) {
 736                                 memcpy(root->sha1, null_sha1, 20);
 737                                 return 1;
 738                         }
 739                         return 0;
 740                 }
 741         }
 742
 743         if (t->entry_count == t->entry_capacity)
 744                 root->tree = t = grow_tree_content(t, 8);
 745         e = new_tree_entry();
 746         e->name = to_atom(p, n);
 747         t->entries[t->entry_count++] = e;
 748         if (slash1) {
 749                 e->tree = new_tree_content(8);
 750                 e->mode = S_IFDIR;
 751                 tree_content_set(e, slash1 + 1, sha1, mode);
 752         } else {
 753                 e->tree = NULL;
 754                 e->mode = mode;
 755                 memcpy(e->sha1, sha1, 20);
 756         }
 757         memcpy(root->sha1, null_sha1, 20);
 758         return 1;
 759 }
 760
 761 static int tree_content_remove(struct tree_entry *root, const char *p)
 762 {
 763         struct tree_content *t = root->tree;
 764         const char *slash1;
 765         unsigned int i, n;
 766         struct tree_entry *e;
 767
 768         slash1 = strchr(p, '/');
 769         if (slash1)
 770                 n = slash1 - p;
 771         else
 772                 n = strlen(p);
 773
 774         for (i = 0; i < t->entry_count; i++) {
 775                 e = t->entries[i];
 776                 if (e->name->str_len == n && !strncmp(p, e->name->str_dat, n)) {
 777                         if (!slash1 || !S_ISDIR(e->mode))
 778                                 goto del_entry;
 779                         if (!e->tree)
 780                                 load_tree(e);
 781                         if (tree_content_remove(e, slash1 + 1)) {
 782                                 if (!e->tree->entry_count)
 783                                         goto del_entry;
 784                                 memcpy(root->sha1, null_sha1, 20);
 785                                 return 1;
 786                         }
 787                         return 0;
 788                 }
 789         }
 790         return 0;
 791
 792 del_entry:
 793         for (i++; i < t->entry_count; i++)
 794                 t->entries[i-1] = t->entries[i];
 795         t->entry_count--;
 796         release_tree_entry(e);
 797         memcpy(root->sha1, null_sha1, 20);
 798         return 1;
 799 }
 800
 801 static void init_pack_header()
 802 {
 803         struct pack_header hdr;
 804
 805         hdr.hdr_signature = htonl(PACK_SIGNATURE);
 806         hdr.hdr_version = htonl(2);
 807         hdr.hdr_entries = 0;
 808
 809         ywrite(pack_fd, &hdr, sizeof(hdr));
 810         pack_offset = sizeof(hdr);
 811 }
 812
 813 static void fixup_header_footer()
 814 {
 815         SHA_CTX c;
 816         char hdr[8];
 817         unsigned long cnt;
 818         char *buf;
 819         size_t n;
 820
 821         if (lseek(pack_fd, 0, SEEK_SET) != 0)
 822                 die("Failed seeking to start: %s", strerror(errno));
 823
 824         SHA1_Init(&c);
 825         yread(pack_fd, hdr, 8);
 826         SHA1_Update(&c, hdr, 8);
 827
 828         cnt = htonl(object_count);
 829         SHA1_Update(&c, &cnt, 4);
 830         ywrite(pack_fd, &cnt, 4);
 831
 832         buf = xmalloc(128 * 1024);
 833         for (;;) {
 834                 n = xread(pack_fd, buf, 128 * 1024);
 835                 if (n <= 0)
 836                         break;
 837                 SHA1_Update(&c, buf, n);
 838         }
 839         free(buf);
 840
 841         SHA1_Final(pack_sha1, &c);
 842         ywrite(pack_fd, pack_sha1, sizeof(pack_sha1));
 843 }
 844
 845 static int oecmp (const void *_a, const void *_b)
 846 {
 847         struct object_entry *a = *((struct object_entry**)_a);
 848         struct object_entry *b = *((struct object_entry**)_b);
 849         return memcmp(a->sha1, b->sha1, sizeof(a->sha1));
 850 }
 851
 852 static void write_index(const char *idx_name)
 853 {
 854         struct sha1file *f;
 855         struct object_entry **idx, **c, **last;
 856         struct object_entry *e;
 857         struct object_entry_pool *o;
 858         unsigned int array[256];
 859         int i;
 860
 861         /* Build the sorted table of object IDs. */
 862         idx = xmalloc(object_count * sizeof(struct object_entry*));
 863         c = idx;
 864         for (o = blocks; o; o = o->next_pool)
 865                 for (e = o->entries; e != o->next_free; e++)
 866                         *c++ = e;
 867         last = idx + object_count;
 868         qsort(idx, object_count, sizeof(struct object_entry*), oecmp);
 869
 870         /* Generate the fan-out array. */
 871         c = idx;
 872         for (i = 0; i < 256; i++) {
 873                 struct object_entry **next = c;;
 874                 while (next < last) {
 875                         if ((*next)->sha1[0] != i)
 876                                 break;
 877                         next++;
 878                 }
 879                 array[i] = htonl(next - idx);
 880                 c = next;
 881         }
 882
 883         f = sha1create("%s", idx_name);
 884         sha1write(f, array, 256 * sizeof(int));
 885         for (c = idx; c != last; c++) {
 886                 unsigned int offset = htonl((*c)->offset);
 887                 sha1write(f, &offset, 4);
 888                 sha1write(f, (*c)->sha1, sizeof((*c)->sha1));
 889         }
 890         sha1write(f, pack_sha1, sizeof(pack_sha1));
 891         sha1close(f, NULL, 1);
 892         free(idx);
 893 }
 894
 895 static void dump_branches()
 896 {
 897         static const char *msg = "fast-import";
 898         unsigned int i;
 899         struct branch *b;
 900         struct ref_lock *lock;
 901
 902         for (i = 0; i < branch_table_sz; i++) {
 903                 for (b = branch_table[i]; b; b = b->table_next_branch) {
 904                         lock = lock_any_ref_for_update(b->name, NULL, 0);
 905                         if (!lock || write_ref_sha1(lock, b->sha1, msg) < 0)
 906                                 die("Can't write %s", b->name);
 907                 }
 908         }
 909 }
 910
 911 static void read_next_command()
 912 {
 913         read_line(&command_buf, stdin, '\n');
 914 }
 915
 916 static void cmd_mark()
 917 {
 918         if (!strncmp("mark :", command_buf.buf, 6)) {
 919                 command_mark = strtoul(command_buf.buf + 6, NULL, 10);
 920                 read_next_command();
 921         }
 922         else
 923                 command_mark = 0;
 924 }
 925
 926 static void* cmd_data (size_t *size)
 927 {
 928         size_t n = 0;
 929         void *buffer;
 930         size_t length;
 931
 932         if (strncmp("data ", command_buf.buf, 5))
 933                 die("Expected 'data n' command, found: %s", command_buf.buf);
 934
 935         length = strtoul(command_buf.buf + 5, NULL, 10);
 936         buffer = xmalloc(length);
 937
 938         while (n < length) {
 939                 size_t s = fread((char*)buffer + n, 1, length - n, stdin);
 940                 if (!s && feof(stdin))
 941                         die("EOF in data (%lu bytes remaining)", length - n);
 942                 n += s;
 943         }
 944
 945         if (fgetc(stdin) != '\n')
 946                 die("An lf did not trail the binary data as expected.");
 947
 948         *size = length;
 949         return buffer;
 950 }
 951
 952 static void cmd_new_blob()
 953 {
 954         size_t datlen;
 955         void *dat;
 956         unsigned char sha1[20];
 957
 958         read_next_command();
 959         cmd_mark();
 960         dat = cmd_data(&datlen);
 961
 962         if (store_object(OBJ_BLOB, dat, datlen, &last_blob, sha1))
 963                 free(dat);
 964 }
 965
 966 static void unload_one_branch()
 967 {
 968         while (cur_active_branches >= max_active_branches) {
 969                 unsigned long min_commit = ULONG_MAX;
 970                 struct branch *e, *l = NULL, *p = NULL;
 971
 972                 for (e = active_branches; e; e = e->active_next_branch) {
 973                         if (e->last_commit < min_commit) {
 974                                 p = l;
 975                                 min_commit = e->last_commit;
 976                         }
 977                         l = e;
 978                 }
 979
 980                 if (p) {
 981                         e = p->active_next_branch;
 982                         p->active_next_branch = e->active_next_branch;
 983                 } else {
 984                         e = active_branches;
 985                         active_branches = e->active_next_branch;
 986                 }
 987                 e->active_next_branch = NULL;
 988                 if (e->branch_tree.tree) {
 989                         release_tree_content(e->branch_tree.tree);
 990                         e->branch_tree.tree = NULL;
 991                 }
 992                 cur_active_branches--;
 993         }
 994 }
 995
 996 static void load_branch(struct branch *b)
 997 {
 998         load_tree(&b->branch_tree);
 999         b->active_next_branch = active_branches;
1000         active_branches = b;
1001         cur_active_branches++;
1002 }
1003
1004 static void file_change_m(struct branch *b)
1005 {
1006         const char *p = command_buf.buf + 2;
1007         char *p_uq;
1008         const char *endp;
1009         struct object_entry *oe;
1010         unsigned char sha1[20];
1011         unsigned int mode;
1012         char type[20];
1013
1014         p = get_mode(p, &mode);
1015         if (!p)
1016                 die("Corrupt mode: %s", command_buf.buf);
1017         switch (mode) {
1018         case S_IFREG | 0644:
1019         case S_IFREG | 0755:
1020         case 0644:
1021         case 0755:
1022                 /* ok */
1023                 break;
1024         default:
1025                 die("Corrupt mode: %s", command_buf.buf);
1026         }
1027
1028         if (get_sha1_hex(p, sha1))
1029                 die("Invalid SHA1: %s", command_buf.buf);
1030         p += 40;
1031         if (*p++ != ' ')
1032                 die("Missing space after SHA1: %s", command_buf.buf);
1033
1034         p_uq = unquote_c_style(p, &endp);
1035         if (p_uq) {
1036                 if (*endp)
1037                         die("Garbage after path in: %s", command_buf.buf);
1038                 p = p_uq;
1039         }
1040
1041         oe = find_object(sha1);
1042         if (oe) {
1043                 if (oe->type != OBJ_BLOB)
1044                         die("Not a blob (actually a %s): %s",
1045                                 command_buf.buf, type_names[oe->type]);
1046         } else {
1047                 if (sha1_object_info(sha1, type, NULL))
1048                         die("Blob not found: %s", command_buf.buf);
1049                 if (strcmp(blob_type, type))
1050                         die("Not a blob (actually a %s): %s",
1051                                 command_buf.buf, type);
1052         }
1053
1054         tree_content_set(&b->branch_tree, p, sha1, S_IFREG | mode);
1055
1056         if (p_uq)
1057                 free(p_uq);
1058 }
1059
1060 static void file_change_d(struct branch *b)
1061 {
1062         const char *p = command_buf.buf + 2;
1063         char *p_uq;
1064         const char *endp;
1065
1066         p_uq = unquote_c_style(p, &endp);
1067         if (p_uq) {
1068                 if (*endp)
1069                         die("Garbage after path in: %s", command_buf.buf);
1070                 p = p_uq;
1071         }
1072         tree_content_remove(&b->branch_tree, p);
1073         if (p_uq)
1074                 free(p_uq);
1075 }
1076
1077 static void cmd_new_commit()
1078 {
1079         struct branch *b;
1080         void *msg;
1081         size_t msglen;
1082         char *str_uq;
1083         const char *endp;
1084         char *sp;
1085         char *author = NULL;
1086         char *committer = NULL;
1087         char *body;
1088
1089         /* Obtain the branch name from the rest of our command */
1090         sp = strchr(command_buf.buf, ' ') + 1;
1091         str_uq = unquote_c_style(sp, &endp);
1092         if (str_uq) {
1093                 if (*endp)
1094                         die("Garbage after ref in: %s", command_buf.buf);
1095                 sp = str_uq;
1096         }
1097         b = lookup_branch(sp);
1098         if (!b)
1099                 die("Branch not declared: %s", sp);
1100         if (str_uq)
1101                 free(str_uq);
1102
1103         read_next_command();
1104         cmd_mark();
1105         if (!strncmp("author ", command_buf.buf, 7)) {
1106                 author = strdup(command_buf.buf);
1107                 read_next_command();
1108         }
1109         if (!strncmp("committer ", command_buf.buf, 10)) {
1110                 committer = strdup(command_buf.buf);
1111                 read_next_command();
1112         }
1113         if (!committer)
1114                 die("Expected committer but didn't get one");
1115         msg = cmd_data(&msglen);
1116
1117         /* ensure the branch is active/loaded */
1118         if (!b->branch_tree.tree) {
1119                 unload_one_branch();
1120                 load_branch(b);
1121         }
1122
1123         /* file_change* */
1124         for (;;) {
1125                 read_next_command();
1126                 if (1 == command_buf.len)
1127                         break;
1128                 else if (!strncmp("M ", command_buf.buf, 2))
1129                         file_change_m(b);
1130                 else if (!strncmp("D ", command_buf.buf, 2))
1131                         file_change_d(b);
1132                 else
1133                         die("Unsupported file_change: %s", command_buf.buf);
1134         }
1135
1136         /* build the tree and the commit */
1137         store_tree(&b->branch_tree);
1138         body = xmalloc(97 + msglen
1139                 + (author
1140                         ? strlen(author) + strlen(committer)
1141                         : 2 * strlen(committer)));
1142         sp = body;
1143         sp += sprintf(sp, "tree %s\n", sha1_to_hex(b->branch_tree.sha1));
1144         if (memcmp(b->sha1, null_sha1, 20))
1145                 sp += sprintf(sp, "parent %s\n", sha1_to_hex(b->sha1));
1146         if (author)
1147                 sp += sprintf(sp, "%s\n", author);
1148         else
1149                 sp += sprintf(sp, "author %s\n", committer + 10);
1150         sp += sprintf(sp, "%s\n\n", committer);
1151         memcpy(sp, msg, msglen);
1152         sp += msglen;
1153         if (author)
1154                 free(author);
1155         free(committer);
1156         free(msg);
1157
1158         store_object(OBJ_COMMIT, body, sp - body, NULL, b->sha1);
1159         free(body);
1160         b->last_commit = object_count_by_type[OBJ_COMMIT];
1161 }
1162
1163 static void cmd_new_branch()
1164 {
1165         struct branch *b;
1166         char *str_uq;
1167         const char *endp;
1168         char *sp;
1169
1170         /* Obtain the new branch name from the rest of our command */
1171         sp = strchr(command_buf.buf, ' ') + 1;
1172         str_uq = unquote_c_style(sp, &endp);
1173         if (str_uq) {
1174                 if (*endp)
1175                         die("Garbage after ref in: %s", command_buf.buf);
1176                 sp = str_uq;
1177         }
1178         b = new_branch(sp);
1179         if (str_uq)
1180                 free(str_uq);
1181         read_next_command();
1182
1183         /* from ... */
1184         if (!strncmp("from ", command_buf.buf, 5)) {
1185                 const char *from;
1186                 struct branch *s;
1187
1188                 from = strchr(command_buf.buf, ' ') + 1;
1189                 str_uq = unquote_c_style(from, &endp);
1190                 if (str_uq) {
1191                         if (*endp)
1192                                 die("Garbage after string in: %s", command_buf.buf);
1193                         from = str_uq;
1194                 }
1195
1196                 s = lookup_branch(from);
1197                 if (b == s)
1198                         die("Can't create a branch from itself: %s", b->name);
1199                 else if (s) {
1200                         memcpy(b->sha1, s->sha1, 20);
1201                         memcpy(b->branch_tree.sha1, s->branch_tree.sha1, 20);
1202                 } else if (!get_sha1(from, b->sha1)) {
1203                         if (!memcmp(b->sha1, null_sha1, 20))
1204                                 memcpy(b->branch_tree.sha1, null_sha1, 20);
1205                         else {
1206                                 unsigned long size;
1207                                 char *buf;
1208
1209                                 buf = read_object_with_reference(b->sha1,
1210                                         type_names[OBJ_COMMIT], &size, b->sha1);
1211                                 if (!buf || size < 46)
1212                                         die("Not a valid commit: %s", from);
1213                                 if (memcmp("tree ", buf, 5)
1214                                         || get_sha1_hex(buf + 5, b->branch_tree.sha1))
1215                                         die("The commit %s is corrupt", sha1_to_hex(b->sha1));
1216                                 free(buf);
1217                         }
1218                 } else
1219                         die("Invalid ref name or SHA1 expression: %s", from);
1220
1221                 if (str_uq)
1222                         free(str_uq);
1223                 read_next_command();
1224         } else {
1225                 memcpy(b->sha1, null_sha1, 20);
1226                 memcpy(b->branch_tree.sha1, null_sha1, 20);
1227         }
1228
1229         if (command_buf.eof || command_buf.len > 1)
1230                 die("An lf did not terminate the branch command as expected.");
1231 }
1232
1233 int main(int argc, const char **argv)
1234 {
1235         const char *base_name = argv[1];
1236         int est_obj_cnt = atoi(argv[2]);
1237         char *pack_name;
1238         char *idx_name;
1239         struct stat sb;
1240
1241         setup_ident();
1242         git_config(git_default_config);
1243
1244         pack_name = xmalloc(strlen(base_name) + 6);
1245         sprintf(pack_name, "%s.pack", base_name);
1246         idx_name = xmalloc(strlen(base_name) + 5);
1247         sprintf(idx_name, "%s.idx", base_name);
1248
1249         pack_fd = open(pack_name, O_RDWR|O_CREAT|O_EXCL, 0666);
1250         if (pack_fd < 0)
1251                 die("Can't create %s: %s", pack_name, strerror(errno));
1252
1253         init_pack_header();
1254         alloc_objects(est_obj_cnt);
1255         strbuf_init(&command_buf);
1256
1257         atom_table = xcalloc(atom_table_sz, sizeof(struct atom_str*));
1258         branch_table = xcalloc(branch_table_sz, sizeof(struct branch*));
1259         avail_tree_table = xcalloc(avail_tree_table_sz, sizeof(struct avail_tree_content*));
1260
1261         for (;;) {
1262                 read_next_command();
1263                 if (command_buf.eof)
1264                         break;
1265                 else if (!strcmp("blob", command_buf.buf))
1266                         cmd_new_blob();
1267                 else if (!strncmp("branch ", command_buf.buf, 7))
1268                         cmd_new_branch();
1269                 else if (!strncmp("commit ", command_buf.buf, 7))
1270                         cmd_new_commit();
1271                 else
1272                         die("Unsupported command: %s", command_buf.buf);
1273         }
1274
1275         fixup_header_footer();
1276         close(pack_fd);
1277         write_index(idx_name);
1278         dump_branches();
1279
1280         fprintf(stderr, "%s statistics:\n", argv[0]);
1281         fprintf(stderr, "---------------------------------------------------\n");
1282         fprintf(stderr, "Alloc'd objects: %10lu (%10lu overflow  )\n", alloc_count, alloc_count - est_obj_cnt);
1283         fprintf(stderr, "Total objects:   %10lu (%10lu duplicates)\n", object_count, duplicate_count);
1284         fprintf(stderr, "      blobs  :   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_BLOB], duplicate_count_by_type[OBJ_BLOB]);
1285         fprintf(stderr, "      trees  :   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_TREE], duplicate_count_by_type[OBJ_TREE]);
1286         fprintf(stderr, "      commits:   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_COMMIT], duplicate_count_by_type[OBJ_COMMIT]);
1287         fprintf(stderr, "      tags   :   %10lu (%10lu duplicates)\n", object_count_by_type[OBJ_TAG], duplicate_count_by_type[OBJ_TAG]);
1288         fprintf(stderr, "Total branches:  %10lu\n", branch_count);
1289         fprintf(stderr, "Total atoms:     %10u\n", atom_cnt);
1290         fprintf(stderr, "Memory total:    %10lu KiB\n", (total_allocd + alloc_count*sizeof(struct object_entry))/1024);
1291         fprintf(stderr, "       pools:    %10lu KiB\n", total_allocd/1024);
1292         fprintf(stderr, "     objects:    %10lu KiB\n", (alloc_count*sizeof(struct object_entry))/1024);
1293         fprintf(stderr, "---------------------------------------------------\n");
1294
1295         stat(pack_name, &sb);
1296         fprintf(stderr, "Pack size:       %10lu KiB\n", (unsigned long)(sb.st_size/1024));
1297         stat(idx_name, &sb);
1298         fprintf(stderr, "Index size:      %10lu KiB\n", (unsigned long)(sb.st_size/1024));
1299
1300         fprintf(stderr, "\n");
1301
1302         return 0;
1303 }