git.oblomov.eu Git - linux-2.6/blob - fs/btrfs/extent_io.c

   1 #include <linux/bitops.h>
   2 #include <linux/slab.h>
   3 #include <linux/bio.h>
   4 #include <linux/mm.h>
   5 #include <linux/gfp.h>
   6 #include <linux/pagemap.h>
   7 #include <linux/page-flags.h>
   8 #include <linux/module.h>
   9 #include <linux/spinlock.h>
  10 #include <linux/blkdev.h>
  11 #include <linux/swap.h>
  12 #include <linux/version.h>
  13 #include <linux/writeback.h>
  14 #include <linux/pagevec.h>
  15 #include "extent_io.h"
  16 #include "extent_map.h"
  17 #include "compat.h"
  18 #include "ctree.h"
  19 #include "btrfs_inode.h"
  20
  21 /* temporary define until extent_map moves out of btrfs */
  22 struct kmem_cache *btrfs_cache_create(const char *name, size_t size,
  23                                        unsigned long extra_flags,
  24                                        void (*ctor)(void *, struct kmem_cache *,
  25                                                     unsigned long));
  26
  27 static struct kmem_cache *extent_state_cache;
  28 static struct kmem_cache *extent_buffer_cache;
  29
  30 static LIST_HEAD(buffers);
  31 static LIST_HEAD(states);
  32
  33 #define LEAK_DEBUG 1
  34 #ifdef LEAK_DEBUG
  35 static spinlock_t leak_lock = SPIN_LOCK_UNLOCKED;
  36 #endif
  37
  38 #define BUFFER_LRU_MAX 64
  39
  40 struct tree_entry {
  41         u64 start;
  42         u64 end;
  43         struct rb_node rb_node;
  44 };
  45
  46 struct extent_page_data {
  47         struct bio *bio;
  48         struct extent_io_tree *tree;
  49         get_extent_t *get_extent;
  50 };
  51
  52 int __init extent_io_init(void)
  53 {
  54         extent_state_cache = btrfs_cache_create("extent_state",
  55                                             sizeof(struct extent_state), 0,
  56                                             NULL);
  57         if (!extent_state_cache)
  58                 return -ENOMEM;
  59
  60         extent_buffer_cache = btrfs_cache_create("extent_buffers",
  61                                             sizeof(struct extent_buffer), 0,
  62                                             NULL);
  63         if (!extent_buffer_cache)
  64                 goto free_state_cache;
  65         return 0;
  66
  67 free_state_cache:
  68         kmem_cache_destroy(extent_state_cache);
  69         return -ENOMEM;
  70 }
  71
  72 void extent_io_exit(void)
  73 {
  74         struct extent_state *state;
  75         struct extent_buffer *eb;
  76
  77         while (!list_empty(&states)) {
  78                 state = list_entry(states.next, struct extent_state, leak_list);
  79                 printk("state leak: start %Lu end %Lu state %lu in tree %p refs %d\n", state->start, state->end, state->state, state->tree, atomic_read(&state->refs));
  80                 list_del(&state->leak_list);
  81                 kmem_cache_free(extent_state_cache, state);
  82
  83         }
  84
  85         while (!list_empty(&buffers)) {
  86                 eb = list_entry(buffers.next, struct extent_buffer, leak_list);
  87                 printk("buffer leak start %Lu len %lu refs %d\n", eb->start, eb->len, atomic_read(&eb->refs));
  88                 list_del(&eb->leak_list);
  89                 kmem_cache_free(extent_buffer_cache, eb);
  90         }
  91         if (extent_state_cache)
  92                 kmem_cache_destroy(extent_state_cache);
  93         if (extent_buffer_cache)
  94                 kmem_cache_destroy(extent_buffer_cache);
  95 }
  96
  97 void extent_io_tree_init(struct extent_io_tree *tree,
  98                           struct address_space *mapping, gfp_t mask)
  99 {
 100         tree->state.rb_node = NULL;
 101         tree->buffer.rb_node = NULL;
 102         tree->ops = NULL;
 103         tree->dirty_bytes = 0;
 104         spin_lock_init(&tree->lock);
 105         spin_lock_init(&tree->buffer_lock);
 106         tree->mapping = mapping;
 107 }
 108 EXPORT_SYMBOL(extent_io_tree_init);
 109
 110 struct extent_state *alloc_extent_state(gfp_t mask)
 111 {
 112         struct extent_state *state;
 113 #ifdef LEAK_DEBUG
 114         unsigned long flags;
 115 #endif
 116
 117         state = kmem_cache_alloc(extent_state_cache, mask);
 118         if (!state)
 119                 return state;
 120         state->state = 0;
 121         state->private = 0;
 122         state->tree = NULL;
 123 #ifdef LEAK_DEBUG
 124         spin_lock_irqsave(&leak_lock, flags);
 125         list_add(&state->leak_list, &states);
 126         spin_unlock_irqrestore(&leak_lock, flags);
 127 #endif
 128         atomic_set(&state->refs, 1);
 129         init_waitqueue_head(&state->wq);
 130         return state;
 131 }
 132 EXPORT_SYMBOL(alloc_extent_state);
 133
 134 void free_extent_state(struct extent_state *state)
 135 {
 136         if (!state)
 137                 return;
 138         if (atomic_dec_and_test(&state->refs)) {
 139 #ifdef LEAK_DEBUG
 140                 unsigned long flags;
 141 #endif
 142                 WARN_ON(state->tree);
 143 #ifdef LEAK_DEBUG
 144                 spin_lock_irqsave(&leak_lock, flags);
 145                 list_del(&state->leak_list);
 146                 spin_unlock_irqrestore(&leak_lock, flags);
 147 #endif
 148                 kmem_cache_free(extent_state_cache, state);
 149         }
 150 }
 151 EXPORT_SYMBOL(free_extent_state);
 152
 153 static struct rb_node *tree_insert(struct rb_root *root, u64 offset,
 154                                    struct rb_node *node)
 155 {
 156         struct rb_node ** p = &root->rb_node;
 157         struct rb_node * parent = NULL;
 158         struct tree_entry *entry;
 159
 160         while(*p) {
 161                 parent = *p;
 162                 entry = rb_entry(parent, struct tree_entry, rb_node);
 163
 164                 if (offset < entry->start)
 165                         p = &(*p)->rb_left;
 166                 else if (offset > entry->end)
 167                         p = &(*p)->rb_right;
 168                 else
 169                         return parent;
 170         }
 171
 172         entry = rb_entry(node, struct tree_entry, rb_node);
 173         rb_link_node(node, parent, p);
 174         rb_insert_color(node, root);
 175         return NULL;
 176 }
 177
 178 static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
 179                                      struct rb_node **prev_ret,
 180                                      struct rb_node **next_ret)
 181 {
 182         struct rb_root *root = &tree->state;
 183         struct rb_node * n = root->rb_node;
 184         struct rb_node *prev = NULL;
 185         struct rb_node *orig_prev = NULL;
 186         struct tree_entry *entry;
 187         struct tree_entry *prev_entry = NULL;
 188
 189         while(n) {
 190                 entry = rb_entry(n, struct tree_entry, rb_node);
 191                 prev = n;
 192                 prev_entry = entry;
 193
 194                 if (offset < entry->start)
 195                         n = n->rb_left;
 196                 else if (offset > entry->end)
 197                         n = n->rb_right;
 198                 else {
 199                         return n;
 200                 }
 201         }
 202
 203         if (prev_ret) {
 204                 orig_prev = prev;
 205                 while(prev && offset > prev_entry->end) {
 206                         prev = rb_next(prev);
 207                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 208                 }
 209                 *prev_ret = prev;
 210                 prev = orig_prev;
 211         }
 212
 213         if (next_ret) {
 214                 prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 215                 while(prev && offset < prev_entry->start) {
 216                         prev = rb_prev(prev);
 217                         prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 218                 }
 219                 *next_ret = prev;
 220         }
 221         return NULL;
 222 }
 223
 224 static inline struct rb_node *tree_search(struct extent_io_tree *tree,
 225                                           u64 offset)
 226 {
 227         struct rb_node *prev = NULL;
 228         struct rb_node *ret;
 229
 230         ret = __etree_search(tree, offset, &prev, NULL);
 231         if (!ret) {
 232                 return prev;
 233         }
 234         return ret;
 235 }
 236
 237 static struct extent_buffer *buffer_tree_insert(struct extent_io_tree *tree,
 238                                           u64 offset, struct rb_node *node)
 239 {
 240         struct rb_root *root = &tree->buffer;
 241         struct rb_node ** p = &root->rb_node;
 242         struct rb_node * parent = NULL;
 243         struct extent_buffer *eb;
 244
 245         while(*p) {
 246                 parent = *p;
 247                 eb = rb_entry(parent, struct extent_buffer, rb_node);
 248
 249                 if (offset < eb->start)
 250                         p = &(*p)->rb_left;
 251                 else if (offset > eb->start)
 252                         p = &(*p)->rb_right;
 253                 else
 254                         return eb;
 255         }
 256
 257         rb_link_node(node, parent, p);
 258         rb_insert_color(node, root);
 259         return NULL;
 260 }
 261
 262 static struct extent_buffer *buffer_search(struct extent_io_tree *tree,
 263                                            u64 offset)
 264 {
 265         struct rb_root *root = &tree->buffer;
 266         struct rb_node * n = root->rb_node;
 267         struct extent_buffer *eb;
 268
 269         while(n) {
 270                 eb = rb_entry(n, struct extent_buffer, rb_node);
 271                 if (offset < eb->start)
 272                         n = n->rb_left;
 273                 else if (offset > eb->start)
 274                         n = n->rb_right;
 275                 else
 276                         return eb;
 277         }
 278         return NULL;
 279 }
 280
 281 /*
 282  * utility function to look for merge candidates inside a given range.
 283  * Any extents with matching state are merged together into a single
 284  * extent in the tree.  Extents with EXTENT_IO in their state field
 285  * are not merged because the end_io handlers need to be able to do
 286  * operations on them without sleeping (or doing allocations/splits).
 287  *
 288  * This should be called with the tree lock held.
 289  */
 290 static int merge_state(struct extent_io_tree *tree,
 291                        struct extent_state *state)
 292 {
 293         struct extent_state *other;
 294         struct rb_node *other_node;
 295
 296         if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
 297                 return 0;
 298
 299         other_node = rb_prev(&state->rb_node);
 300         if (other_node) {
 301                 other = rb_entry(other_node, struct extent_state, rb_node);
 302                 if (other->end == state->start - 1 &&
 303                     other->state == state->state) {
 304                         state->start = other->start;
 305                         other->tree = NULL;
 306                         rb_erase(&other->rb_node, &tree->state);
 307                         free_extent_state(other);
 308                 }
 309         }
 310         other_node = rb_next(&state->rb_node);
 311         if (other_node) {
 312                 other = rb_entry(other_node, struct extent_state, rb_node);
 313                 if (other->start == state->end + 1 &&
 314                     other->state == state->state) {
 315                         other->start = state->start;
 316                         state->tree = NULL;
 317                         rb_erase(&state->rb_node, &tree->state);
 318                         free_extent_state(state);
 319                 }
 320         }
 321         return 0;
 322 }
 323
 324 static void set_state_cb(struct extent_io_tree *tree,
 325                          struct extent_state *state,
 326                          unsigned long bits)
 327 {
 328         if (tree->ops && tree->ops->set_bit_hook) {
 329                 tree->ops->set_bit_hook(tree->mapping->host, state->start,
 330                                         state->end, state->state, bits);
 331         }
 332 }
 333
 334 static void clear_state_cb(struct extent_io_tree *tree,
 335                            struct extent_state *state,
 336                            unsigned long bits)
 337 {
 338         if (tree->ops && tree->ops->set_bit_hook) {
 339                 tree->ops->clear_bit_hook(tree->mapping->host, state->start,
 340                                           state->end, state->state, bits);
 341         }
 342 }
 343
 344 /*
 345  * insert an extent_state struct into the tree.  'bits' are set on the
 346  * struct before it is inserted.
 347  *
 348  * This may return -EEXIST if the extent is already there, in which case the
 349  * state struct is freed.
 350  *
 351  * The tree lock is not taken internally.  This is a utility function and
 352  * probably isn't what you want to call (see set/clear_extent_bit).
 353  */
 354 static int insert_state(struct extent_io_tree *tree,
 355                         struct extent_state *state, u64 start, u64 end,
 356                         int bits)
 357 {
 358         struct rb_node *node;
 359
 360         if (end < start) {
 361                 printk("end < start %Lu %Lu\n", end, start);
 362                 WARN_ON(1);
 363         }
 364         if (bits & EXTENT_DIRTY)
 365                 tree->dirty_bytes += end - start + 1;
 366         set_state_cb(tree, state, bits);
 367         state->state |= bits;
 368         state->start = start;
 369         state->end = end;
 370         node = tree_insert(&tree->state, end, &state->rb_node);
 371         if (node) {
 372                 struct extent_state *found;
 373                 found = rb_entry(node, struct extent_state, rb_node);
 374                 printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, start, end);
 375                 free_extent_state(state);
 376                 return -EEXIST;
 377         }
 378         state->tree = tree;
 379         merge_state(tree, state);
 380         return 0;
 381 }
 382
 383 /*
 384  * split a given extent state struct in two, inserting the preallocated
 385  * struct 'prealloc' as the newly created second half.  'split' indicates an
 386  * offset inside 'orig' where it should be split.
 387  *
 388  * Before calling,
 389  * the tree has 'orig' at [orig->start, orig->end].  After calling, there
 390  * are two extent state structs in the tree:
 391  * prealloc: [orig->start, split - 1]
 392  * orig: [ split, orig->end ]
 393  *
 394  * The tree locks are not taken by this function. They need to be held
 395  * by the caller.
 396  */
 397 static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
 398                        struct extent_state *prealloc, u64 split)
 399 {
 400         struct rb_node *node;
 401         prealloc->start = orig->start;
 402         prealloc->end = split - 1;
 403         prealloc->state = orig->state;
 404         orig->start = split;
 405
 406         node = tree_insert(&tree->state, prealloc->end, &prealloc->rb_node);
 407         if (node) {
 408                 struct extent_state *found;
 409                 found = rb_entry(node, struct extent_state, rb_node);
 410                 printk("found node %Lu %Lu on insert of %Lu %Lu\n", found->start, found->end, prealloc->start, prealloc->end);
 411                 free_extent_state(prealloc);
 412                 return -EEXIST;
 413         }
 414         prealloc->tree = tree;
 415         return 0;
 416 }
 417
 418 /*
 419  * utility function to clear some bits in an extent state struct.
 420  * it will optionally wake up any one waiting on this state (wake == 1), or
 421  * forcibly remove the state from the tree (delete == 1).
 422  *
 423  * If no bits are set on the state struct after clearing things, the
 424  * struct is freed and removed from the tree
 425  */
 426 static int clear_state_bit(struct extent_io_tree *tree,
 427                             struct extent_state *state, int bits, int wake,
 428                             int delete)
 429 {
 430         int ret = state->state & bits;
 431
 432         if ((bits & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
 433                 u64 range = state->end - state->start + 1;
 434                 WARN_ON(range > tree->dirty_bytes);
 435                 tree->dirty_bytes -= range;
 436         }
 437         clear_state_cb(tree, state, bits);
 438         state->state &= ~bits;
 439         if (wake)
 440                 wake_up(&state->wq);
 441         if (delete || state->state == 0) {
 442                 if (state->tree) {
 443                         clear_state_cb(tree, state, state->state);
 444                         rb_erase(&state->rb_node, &tree->state);
 445                         state->tree = NULL;
 446                         free_extent_state(state);
 447                 } else {
 448                         WARN_ON(1);
 449                 }
 450         } else {
 451                 merge_state(tree, state);
 452         }
 453         return ret;
 454 }
 455
 456 /*
 457  * clear some bits on a range in the tree.  This may require splitting
 458  * or inserting elements in the tree, so the gfp mask is used to
 459  * indicate which allocations or sleeping are allowed.
 460  *
 461  * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
 462  * the given range from the tree regardless of state (ie for truncate).
 463  *
 464  * the range [start, end] is inclusive.
 465  *
 466  * This takes the tree lock, and returns < 0 on error, > 0 if any of the
 467  * bits were already set, or zero if none of the bits were already set.
 468  */
 469 int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
 470                      int bits, int wake, int delete, gfp_t mask)
 471 {
 472         struct extent_state *state;
 473         struct extent_state *prealloc = NULL;
 474         struct rb_node *node;
 475         unsigned long flags;
 476         int err;
 477         int set = 0;
 478
 479 again:
 480         if (!prealloc && (mask & __GFP_WAIT)) {
 481                 prealloc = alloc_extent_state(mask);
 482                 if (!prealloc)
 483                         return -ENOMEM;
 484         }
 485
 486         spin_lock_irqsave(&tree->lock, flags);
 487         /*
 488          * this search will find the extents that end after
 489          * our range starts
 490          */
 491         node = tree_search(tree, start);
 492         if (!node)
 493                 goto out;
 494         state = rb_entry(node, struct extent_state, rb_node);
 495         if (state->start > end)
 496                 goto out;
 497         WARN_ON(state->end < start);
 498
 499         /*
 500          *     | ---- desired range ---- |
 501          *  | state | or
 502          *  | ------------- state -------------- |
 503          *
 504          * We need to split the extent we found, and may flip
 505          * bits on second half.
 506          *
 507          * If the extent we found extends past our range, we
 508          * just split and search again.  It'll get split again
 509          * the next time though.
 510          *
 511          * If the extent we found is inside our range, we clear
 512          * the desired bit on it.
 513          */
 514
 515         if (state->start < start) {
 516                 if (!prealloc)
 517                         prealloc = alloc_extent_state(GFP_ATOMIC);
 518                 err = split_state(tree, state, prealloc, start);
 519                 BUG_ON(err == -EEXIST);
 520                 prealloc = NULL;
 521                 if (err)
 522                         goto out;
 523                 if (state->end <= end) {
 524                         start = state->end + 1;
 525                         set |= clear_state_bit(tree, state, bits,
 526                                         wake, delete);
 527                 } else {
 528                         start = state->start;
 529                 }
 530                 goto search_again;
 531         }
 532         /*
 533          * | ---- desired range ---- |
 534          *                        | state |
 535          * We need to split the extent, and clear the bit
 536          * on the first half
 537          */
 538         if (state->start <= end && state->end > end) {
 539                 if (!prealloc)
 540                         prealloc = alloc_extent_state(GFP_ATOMIC);
 541                 err = split_state(tree, state, prealloc, end + 1);
 542                 BUG_ON(err == -EEXIST);
 543
 544                 if (wake)
 545                         wake_up(&state->wq);
 546                 set |= clear_state_bit(tree, prealloc, bits,
 547                                        wake, delete);
 548                 prealloc = NULL;
 549                 goto out;
 550         }
 551
 552         start = state->end + 1;
 553         set |= clear_state_bit(tree, state, bits, wake, delete);
 554         goto search_again;
 555
 556 out:
 557         spin_unlock_irqrestore(&tree->lock, flags);
 558         if (prealloc)
 559                 free_extent_state(prealloc);
 560
 561         return set;
 562
 563 search_again:
 564         if (start > end)
 565                 goto out;
 566         spin_unlock_irqrestore(&tree->lock, flags);
 567         if (mask & __GFP_WAIT)
 568                 cond_resched();
 569         goto again;
 570 }
 571 EXPORT_SYMBOL(clear_extent_bit);
 572
 573 static int wait_on_state(struct extent_io_tree *tree,
 574                          struct extent_state *state)
 575 {
 576         DEFINE_WAIT(wait);
 577         prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
 578         spin_unlock_irq(&tree->lock);
 579         schedule();
 580         spin_lock_irq(&tree->lock);
 581         finish_wait(&state->wq, &wait);
 582         return 0;
 583 }
 584
 585 /*
 586  * waits for one or more bits to clear on a range in the state tree.
 587  * The range [start, end] is inclusive.
 588  * The tree lock is taken by this function
 589  */
 590 int wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits)
 591 {
 592         struct extent_state *state;
 593         struct rb_node *node;
 594
 595         spin_lock_irq(&tree->lock);
 596 again:
 597         while (1) {
 598                 /*
 599                  * this search will find all the extents that end after
 600                  * our range starts
 601                  */
 602                 node = tree_search(tree, start);
 603                 if (!node)
 604                         break;
 605
 606                 state = rb_entry(node, struct extent_state, rb_node);
 607
 608                 if (state->start > end)
 609                         goto out;
 610
 611                 if (state->state & bits) {
 612                         start = state->start;
 613                         atomic_inc(&state->refs);
 614                         wait_on_state(tree, state);
 615                         free_extent_state(state);
 616                         goto again;
 617                 }
 618                 start = state->end + 1;
 619
 620                 if (start > end)
 621                         break;
 622
 623                 if (need_resched()) {
 624                         spin_unlock_irq(&tree->lock);
 625                         cond_resched();
 626                         spin_lock_irq(&tree->lock);
 627                 }
 628         }
 629 out:
 630         spin_unlock_irq(&tree->lock);
 631         return 0;
 632 }
 633 EXPORT_SYMBOL(wait_extent_bit);
 634
 635 static void set_state_bits(struct extent_io_tree *tree,
 636                            struct extent_state *state,
 637                            int bits)
 638 {
 639         if ((bits & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
 640                 u64 range = state->end - state->start + 1;
 641                 tree->dirty_bytes += range;
 642         }
 643         set_state_cb(tree, state, bits);
 644         state->state |= bits;
 645 }
 646
 647 /*
 648  * set some bits on a range in the tree.  This may require allocations
 649  * or sleeping, so the gfp mask is used to indicate what is allowed.
 650  *
 651  * If 'exclusive' == 1, this will fail with -EEXIST if some part of the
 652  * range already has the desired bits set.  The start of the existing
 653  * range is returned in failed_start in this case.
 654  *
 655  * [start, end] is inclusive
 656  * This takes the tree lock.
 657  */
 658 int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end, int bits,
 659                    int exclusive, u64 *failed_start, gfp_t mask)
 660 {
 661         struct extent_state *state;
 662         struct extent_state *prealloc = NULL;
 663         struct rb_node *node;
 664         unsigned long flags;
 665         int err = 0;
 666         int set;
 667         u64 last_start;
 668         u64 last_end;
 669 again:
 670         if (!prealloc && (mask & __GFP_WAIT)) {
 671                 prealloc = alloc_extent_state(mask);
 672                 if (!prealloc)
 673                         return -ENOMEM;
 674         }
 675
 676         spin_lock_irqsave(&tree->lock, flags);
 677         /*
 678          * this search will find all the extents that end after
 679          * our range starts.
 680          */
 681         node = tree_search(tree, start);
 682         if (!node) {
 683                 err = insert_state(tree, prealloc, start, end, bits);
 684                 prealloc = NULL;
 685                 BUG_ON(err == -EEXIST);
 686                 goto out;
 687         }
 688
 689         state = rb_entry(node, struct extent_state, rb_node);
 690         last_start = state->start;
 691         last_end = state->end;
 692
 693         /*
 694          * | ---- desired range ---- |
 695          * | state |
 696          *
 697          * Just lock what we found and keep going
 698          */
 699         if (state->start == start && state->end <= end) {
 700                 set = state->state & bits;
 701                 if (set && exclusive) {
 702                         *failed_start = state->start;
 703                         err = -EEXIST;
 704                         goto out;
 705                 }
 706                 set_state_bits(tree, state, bits);
 707                 start = state->end + 1;
 708                 merge_state(tree, state);
 709                 goto search_again;
 710         }
 711
 712         /*
 713          *     | ---- desired range ---- |
 714          * | state |
 715          *   or
 716          * | ------------- state -------------- |
 717          *
 718          * We need to split the extent we found, and may flip bits on
 719          * second half.
 720          *
 721          * If the extent we found extends past our
 722          * range, we just split and search again.  It'll get split
 723          * again the next time though.
 724          *
 725          * If the extent we found is inside our range, we set the
 726          * desired bit on it.
 727          */
 728         if (state->start < start) {
 729                 set = state->state & bits;
 730                 if (exclusive && set) {
 731                         *failed_start = start;
 732                         err = -EEXIST;
 733                         goto out;
 734                 }
 735                 err = split_state(tree, state, prealloc, start);
 736                 BUG_ON(err == -EEXIST);
 737                 prealloc = NULL;
 738                 if (err)
 739                         goto out;
 740                 if (state->end <= end) {
 741                         set_state_bits(tree, state, bits);
 742                         start = state->end + 1;
 743                         merge_state(tree, state);
 744                 } else {
 745                         start = state->start;
 746                 }
 747                 goto search_again;
 748         }
 749         /*
 750          * | ---- desired range ---- |
 751          *     | state | or               | state |
 752          *
 753          * There's a hole, we need to insert something in it and
 754          * ignore the extent we found.
 755          */
 756         if (state->start > start) {
 757                 u64 this_end;
 758                 if (end < last_start)
 759                         this_end = end;
 760                 else
 761                         this_end = last_start -1;
 762                 err = insert_state(tree, prealloc, start, this_end,
 763                                    bits);
 764                 prealloc = NULL;
 765                 BUG_ON(err == -EEXIST);
 766                 if (err)
 767                         goto out;
 768                 start = this_end + 1;
 769                 goto search_again;
 770         }
 771         /*
 772          * | ---- desired range ---- |
 773          *                        | state |
 774          * We need to split the extent, and set the bit
 775          * on the first half
 776          */
 777         if (state->start <= end && state->end > end) {
 778                 set = state->state & bits;
 779                 if (exclusive && set) {
 780                         *failed_start = start;
 781                         err = -EEXIST;
 782                         goto out;
 783                 }
 784                 err = split_state(tree, state, prealloc, end + 1);
 785                 BUG_ON(err == -EEXIST);
 786
 787                 set_state_bits(tree, prealloc, bits);
 788                 merge_state(tree, prealloc);
 789                 prealloc = NULL;
 790                 goto out;
 791         }
 792
 793         goto search_again;
 794
 795 out:
 796         spin_unlock_irqrestore(&tree->lock, flags);
 797         if (prealloc)
 798                 free_extent_state(prealloc);
 799
 800         return err;
 801
 802 search_again:
 803         if (start > end)
 804                 goto out;
 805         spin_unlock_irqrestore(&tree->lock, flags);
 806         if (mask & __GFP_WAIT)
 807                 cond_resched();
 808         goto again;
 809 }
 810 EXPORT_SYMBOL(set_extent_bit);
 811
 812 /* wrappers around set/clear extent bit */
 813 int set_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
 814                      gfp_t mask)
 815 {
 816         return set_extent_bit(tree, start, end, EXTENT_DIRTY, 0, NULL,
 817                               mask);
 818 }
 819 EXPORT_SYMBOL(set_extent_dirty);
 820
 821 int set_extent_ordered(struct extent_io_tree *tree, u64 start, u64 end,
 822                        gfp_t mask)
 823 {
 824         return set_extent_bit(tree, start, end, EXTENT_ORDERED, 0, NULL, mask);
 825 }
 826 EXPORT_SYMBOL(set_extent_ordered);
 827
 828 int set_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
 829                     int bits, gfp_t mask)
 830 {
 831         return set_extent_bit(tree, start, end, bits, 0, NULL,
 832                               mask);
 833 }
 834 EXPORT_SYMBOL(set_extent_bits);
 835
 836 int clear_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
 837                       int bits, gfp_t mask)
 838 {
 839         return clear_extent_bit(tree, start, end, bits, 0, 0, mask);
 840 }
 841 EXPORT_SYMBOL(clear_extent_bits);
 842
 843 int set_extent_delalloc(struct extent_io_tree *tree, u64 start, u64 end,
 844                      gfp_t mask)
 845 {
 846         return set_extent_bit(tree, start, end,
 847                               EXTENT_DELALLOC | EXTENT_DIRTY,
 848                               0, NULL, mask);
 849 }
 850 EXPORT_SYMBOL(set_extent_delalloc);
 851
 852 int clear_extent_dirty(struct extent_io_tree *tree, u64 start, u64 end,
 853                        gfp_t mask)
 854 {
 855         return clear_extent_bit(tree, start, end,
 856                                 EXTENT_DIRTY | EXTENT_DELALLOC, 0, 0, mask);
 857 }
 858 EXPORT_SYMBOL(clear_extent_dirty);
 859
 860 int clear_extent_ordered(struct extent_io_tree *tree, u64 start, u64 end,
 861                          gfp_t mask)
 862 {
 863         return clear_extent_bit(tree, start, end, EXTENT_ORDERED, 1, 0, mask);
 864 }
 865 EXPORT_SYMBOL(clear_extent_ordered);
 866
 867 int set_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
 868                      gfp_t mask)
 869 {
 870         return set_extent_bit(tree, start, end, EXTENT_NEW, 0, NULL,
 871                               mask);
 872 }
 873 EXPORT_SYMBOL(set_extent_new);
 874
 875 int clear_extent_new(struct extent_io_tree *tree, u64 start, u64 end,
 876                        gfp_t mask)
 877 {
 878         return clear_extent_bit(tree, start, end, EXTENT_NEW, 0, 0, mask);
 879 }
 880 EXPORT_SYMBOL(clear_extent_new);
 881
 882 int set_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
 883                         gfp_t mask)
 884 {
 885         return set_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, NULL,
 886                               mask);
 887 }
 888 EXPORT_SYMBOL(set_extent_uptodate);
 889
 890 int clear_extent_uptodate(struct extent_io_tree *tree, u64 start, u64 end,
 891                           gfp_t mask)
 892 {
 893         return clear_extent_bit(tree, start, end, EXTENT_UPTODATE, 0, 0, mask);
 894 }
 895 EXPORT_SYMBOL(clear_extent_uptodate);
 896
 897 int set_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end,
 898                          gfp_t mask)
 899 {
 900         return set_extent_bit(tree, start, end, EXTENT_WRITEBACK,
 901                               0, NULL, mask);
 902 }
 903 EXPORT_SYMBOL(set_extent_writeback);
 904
 905 int clear_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end,
 906                            gfp_t mask)
 907 {
 908         return clear_extent_bit(tree, start, end, EXTENT_WRITEBACK, 1, 0, mask);
 909 }
 910 EXPORT_SYMBOL(clear_extent_writeback);
 911
 912 int wait_on_extent_writeback(struct extent_io_tree *tree, u64 start, u64 end)
 913 {
 914         return wait_extent_bit(tree, start, end, EXTENT_WRITEBACK);
 915 }
 916 EXPORT_SYMBOL(wait_on_extent_writeback);
 917
 918 /*
 919  * either insert or lock state struct between start and end use mask to tell
 920  * us if waiting is desired.
 921  */
 922 int lock_extent(struct extent_io_tree *tree, u64 start, u64 end, gfp_t mask)
 923 {
 924         int err;
 925         u64 failed_start;
 926         while (1) {
 927                 err = set_extent_bit(tree, start, end, EXTENT_LOCKED, 1,
 928                                      &failed_start, mask);
 929                 if (err == -EEXIST && (mask & __GFP_WAIT)) {
 930                         wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
 931                         start = failed_start;
 932                 } else {
 933                         break;
 934                 }
 935                 WARN_ON(start > end);
 936         }
 937         return err;
 938 }
 939 EXPORT_SYMBOL(lock_extent);
 940
 941 int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end,
 942                     gfp_t mask)
 943 {
 944         int err;
 945         u64 failed_start;
 946
 947         err = set_extent_bit(tree, start, end, EXTENT_LOCKED, 1,
 948                              &failed_start, mask);
 949         if (err == -EEXIST)
 950                 return 0;
 951         return 1;
 952 }
 953 EXPORT_SYMBOL(try_lock_extent);
 954
 955 int unlock_extent(struct extent_io_tree *tree, u64 start, u64 end,
 956                   gfp_t mask)
 957 {
 958         return clear_extent_bit(tree, start, end, EXTENT_LOCKED, 1, 0, mask);
 959 }
 960 EXPORT_SYMBOL(unlock_extent);
 961
 962 /*
 963  * helper function to set pages and extents in the tree dirty
 964  */
 965 int set_range_dirty(struct extent_io_tree *tree, u64 start, u64 end)
 966 {
 967         unsigned long index = start >> PAGE_CACHE_SHIFT;
 968         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
 969         struct page *page;
 970
 971         while (index <= end_index) {
 972                 page = find_get_page(tree->mapping, index);
 973                 BUG_ON(!page);
 974                 __set_page_dirty_nobuffers(page);
 975                 page_cache_release(page);
 976                 index++;
 977         }
 978         set_extent_dirty(tree, start, end, GFP_NOFS);
 979         return 0;
 980 }
 981 EXPORT_SYMBOL(set_range_dirty);
 982
 983 /*
 984  * helper function to set both pages and extents in the tree writeback
 985  */
 986 int set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
 987 {
 988         unsigned long index = start >> PAGE_CACHE_SHIFT;
 989         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
 990         struct page *page;
 991
 992         while (index <= end_index) {
 993                 page = find_get_page(tree->mapping, index);
 994                 BUG_ON(!page);
 995                 set_page_writeback(page);
 996                 page_cache_release(page);
 997                 index++;
 998         }
 999         set_extent_writeback(tree, start, end, GFP_NOFS);
1000         return 0;
1001 }
1002 EXPORT_SYMBOL(set_range_writeback);
1003
1004 /*
1005  * find the first offset in the io tree with 'bits' set. zero is
1006  * returned if we find something, and *start_ret and *end_ret are
1007  * set to reflect the state struct that was found.
1008  *
1009  * If nothing was found, 1 is returned, < 0 on error
1010  */
1011 int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1012                           u64 *start_ret, u64 *end_ret, int bits)
1013 {
1014         struct rb_node *node;
1015         struct extent_state *state;
1016         int ret = 1;
1017
1018         spin_lock_irq(&tree->lock);
1019         /*
1020          * this search will find all the extents that end after
1021          * our range starts.
1022          */
1023         node = tree_search(tree, start);
1024         if (!node) {
1025                 goto out;
1026         }
1027
1028         while(1) {
1029                 state = rb_entry(node, struct extent_state, rb_node);
1030                 if (state->end >= start && (state->state & bits)) {
1031                         *start_ret = state->start;
1032                         *end_ret = state->end;
1033                         ret = 0;
1034                         break;
1035                 }
1036                 node = rb_next(node);
1037                 if (!node)
1038                         break;
1039         }
1040 out:
1041         spin_unlock_irq(&tree->lock);
1042         return ret;
1043 }
1044 EXPORT_SYMBOL(find_first_extent_bit);
1045
1046 /* find the first state struct with 'bits' set after 'start', and
1047  * return it.  tree->lock must be held.  NULL will returned if
1048  * nothing was found after 'start'
1049  */
1050 struct extent_state *find_first_extent_bit_state(struct extent_io_tree *tree,
1051                                                  u64 start, int bits)
1052 {
1053         struct rb_node *node;
1054         struct extent_state *state;
1055
1056         /*
1057          * this search will find all the extents that end after
1058          * our range starts.
1059          */
1060         node = tree_search(tree, start);
1061         if (!node) {
1062                 goto out;
1063         }
1064
1065         while(1) {
1066                 state = rb_entry(node, struct extent_state, rb_node);
1067                 if (state->end >= start && (state->state & bits)) {
1068                         return state;
1069                 }
1070                 node = rb_next(node);
1071                 if (!node)
1072                         break;
1073         }
1074 out:
1075         return NULL;
1076 }
1077 EXPORT_SYMBOL(find_first_extent_bit_state);
1078
1079 /*
1080  * find a contiguous range of bytes in the file marked as delalloc, not
1081  * more than 'max_bytes'.  start and end are used to return the range,
1082  *
1083  * 1 is returned if we find something, 0 if nothing was in the tree
1084  */
1085 static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
1086                                         u64 *start, u64 *end, u64 max_bytes)
1087 {
1088         struct rb_node *node;
1089         struct extent_state *state;
1090         u64 cur_start = *start;
1091         u64 found = 0;
1092         u64 total_bytes = 0;
1093
1094         spin_lock_irq(&tree->lock);
1095
1096         /*
1097          * this search will find all the extents that end after
1098          * our range starts.
1099          */
1100         node = tree_search(tree, cur_start);
1101         if (!node) {
1102                 if (!found)
1103                         *end = (u64)-1;
1104                 goto out;
1105         }
1106
1107         while(1) {
1108                 state = rb_entry(node, struct extent_state, rb_node);
1109                 if (found && (state->start != cur_start ||
1110                               (state->state & EXTENT_BOUNDARY))) {
1111                         goto out;
1112                 }
1113                 if (!(state->state & EXTENT_DELALLOC)) {
1114                         if (!found)
1115                                 *end = state->end;
1116                         goto out;
1117                 }
1118                 if (!found)
1119                         *start = state->start;
1120                 found++;
1121                 *end = state->end;
1122                 cur_start = state->end + 1;
1123                 node = rb_next(node);
1124                 if (!node)
1125                         break;
1126                 total_bytes += state->end - state->start + 1;
1127                 if (total_bytes >= max_bytes)
1128                         break;
1129         }
1130 out:
1131         spin_unlock_irq(&tree->lock);
1132         return found;
1133 }
1134
1135 static noinline int __unlock_for_delalloc(struct inode *inode,
1136                                           struct page *locked_page,
1137                                           u64 start, u64 end)
1138 {
1139         int ret;
1140         struct page *pages[16];
1141         unsigned long index = start >> PAGE_CACHE_SHIFT;
1142         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1143         unsigned long nr_pages = end_index - index + 1;
1144         int i;
1145
1146         if (index == locked_page->index && end_index == index)
1147                 return 0;
1148
1149         while(nr_pages > 0) {
1150                 ret = find_get_pages_contig(inode->i_mapping, index,
1151                                      min(nr_pages, ARRAY_SIZE(pages)), pages);
1152                 for (i = 0; i < ret; i++) {
1153                         if (pages[i] != locked_page)
1154                                 unlock_page(pages[i]);
1155                         page_cache_release(pages[i]);
1156                 }
1157                 nr_pages -= ret;
1158                 index += ret;
1159                 cond_resched();
1160         }
1161         return 0;
1162 }
1163
1164 static noinline int lock_delalloc_pages(struct inode *inode,
1165                                         struct page *locked_page,
1166                                         u64 delalloc_start,
1167                                         u64 delalloc_end)
1168 {
1169         unsigned long index = delalloc_start >> PAGE_CACHE_SHIFT;
1170         unsigned long start_index = index;
1171         unsigned long end_index = delalloc_end >> PAGE_CACHE_SHIFT;
1172         unsigned long pages_locked = 0;
1173         struct page *pages[16];
1174         unsigned long nrpages;
1175         int ret;
1176         int i;
1177
1178         /* the caller is responsible for locking the start index */
1179         if (index == locked_page->index && index == end_index)
1180                 return 0;
1181
1182         /* skip the page at the start index */
1183         nrpages = end_index - index + 1;
1184         while(nrpages > 0) {
1185                 ret = find_get_pages_contig(inode->i_mapping, index,
1186                                      min(nrpages, ARRAY_SIZE(pages)), pages);
1187                 if (ret == 0) {
1188                         ret = -EAGAIN;
1189                         goto done;
1190                 }
1191                 /* now we have an array of pages, lock them all */
1192                 for (i = 0; i < ret; i++) {
1193                         /*
1194                          * the caller is taking responsibility for
1195                          * locked_page
1196                          */
1197                         if (pages[i] != locked_page)
1198                                 lock_page(pages[i]);
1199                         page_cache_release(pages[i]);
1200                 }
1201                 pages_locked += ret;
1202                 nrpages -= ret;
1203                 index += ret;
1204                 cond_resched();
1205         }
1206         ret = 0;
1207 done:
1208         if (ret && pages_locked) {
1209                 __unlock_for_delalloc(inode, locked_page,
1210                               delalloc_start,
1211                               ((u64)(start_index + pages_locked - 1)) <<
1212                               PAGE_CACHE_SHIFT);
1213         }
1214         return ret;
1215 }
1216
1217 /*
1218  * find a contiguous range of bytes in the file marked as delalloc, not
1219  * more than 'max_bytes'.  start and end are used to return the range,
1220  *
1221  * 1 is returned if we find something, 0 if nothing was in the tree
1222  */
1223 static noinline u64 find_lock_delalloc_range(struct inode *inode,
1224                                              struct extent_io_tree *tree,
1225                                              struct page *locked_page,
1226                                              u64 *start, u64 *end,
1227                                              u64 max_bytes)
1228 {
1229         u64 delalloc_start;
1230         u64 delalloc_end;
1231         u64 found;
1232         int ret;
1233         int loops = 0;
1234
1235 again:
1236         /* step one, find a bunch of delalloc bytes starting at start */
1237         delalloc_start = *start;
1238         delalloc_end = 0;
1239         found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1240                                     max_bytes);
1241         if (!found) {
1242                 *start = delalloc_start;
1243                 *end = delalloc_end;
1244                 return found;
1245         }
1246
1247         /*
1248          * make sure to limit the number of pages we try to lock down
1249          * if we're looping.
1250          */
1251         if (delalloc_end + 1 - delalloc_start > max_bytes && loops) {
1252                 delalloc_end = (delalloc_start + PAGE_CACHE_SIZE - 1) &
1253                         ~((u64)PAGE_CACHE_SIZE - 1);
1254         }
1255         /* step two, lock all the pages after the page that has start */
1256         ret = lock_delalloc_pages(inode, locked_page,
1257                                   delalloc_start, delalloc_end);
1258         if (ret == -EAGAIN) {
1259                 /* some of the pages are gone, lets avoid looping by
1260                  * shortening the size of the delalloc range we're searching
1261                  */
1262                 if (!loops) {
1263                         unsigned long offset = (*start) & (PAGE_CACHE_SIZE - 1);
1264                         max_bytes = PAGE_CACHE_SIZE - offset;
1265                         loops = 1;
1266                         goto again;
1267                 } else {
1268                         found = 0;
1269                         goto out_failed;
1270                 }
1271         }
1272         BUG_ON(ret);
1273
1274         /* step three, lock the state bits for the whole range */
1275         lock_extent(tree, delalloc_start, delalloc_end, GFP_NOFS);
1276
1277         /* then test to make sure it is all still delalloc */
1278         ret = test_range_bit(tree, delalloc_start, delalloc_end,
1279                              EXTENT_DELALLOC, 1);
1280         if (!ret) {
1281                 unlock_extent(tree, delalloc_start, delalloc_end, GFP_NOFS);
1282                 __unlock_for_delalloc(inode, locked_page,
1283                               delalloc_start, delalloc_end);
1284                 cond_resched();
1285                 goto again;
1286         }
1287         *start = delalloc_start;
1288         *end = delalloc_end;
1289 out_failed:
1290         return found;
1291 }
1292
1293 int extent_clear_unlock_delalloc(struct inode *inode,
1294                                 struct extent_io_tree *tree,
1295                                 u64 start, u64 end, struct page *locked_page,
1296                                 int clear_dirty, int set_writeback,
1297                                 int end_writeback)
1298 {
1299         int ret;
1300         struct page *pages[16];
1301         unsigned long index = start >> PAGE_CACHE_SHIFT;
1302         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1303         unsigned long nr_pages = end_index - index + 1;
1304         int i;
1305         int clear_bits = EXTENT_LOCKED | EXTENT_DELALLOC;
1306
1307         if (clear_dirty)
1308                 clear_bits |= EXTENT_DIRTY;
1309
1310         clear_extent_bit(tree, start, end, clear_bits, 1, 0, GFP_NOFS);
1311
1312         while(nr_pages > 0) {
1313                 ret = find_get_pages_contig(inode->i_mapping, index,
1314                                      min(nr_pages, ARRAY_SIZE(pages)), pages);
1315                 for (i = 0; i < ret; i++) {
1316                         if (pages[i] == locked_page) {
1317                                 page_cache_release(pages[i]);
1318                                 continue;
1319                         }
1320                         if (clear_dirty)
1321                                 clear_page_dirty_for_io(pages[i]);
1322                         if (set_writeback)
1323                                 set_page_writeback(pages[i]);
1324                         if (end_writeback)
1325                                 end_page_writeback(pages[i]);
1326                         unlock_page(pages[i]);
1327                         page_cache_release(pages[i]);
1328                 }
1329                 nr_pages -= ret;
1330                 index += ret;
1331                 cond_resched();
1332         }
1333         return 0;
1334 }
1335 EXPORT_SYMBOL(extent_clear_unlock_delalloc);
1336
1337 /*
1338  * count the number of bytes in the tree that have a given bit(s)
1339  * set.  This can be fairly slow, except for EXTENT_DIRTY which is
1340  * cached.  The total number found is returned.
1341  */
1342 u64 count_range_bits(struct extent_io_tree *tree,
1343                      u64 *start, u64 search_end, u64 max_bytes,
1344                      unsigned long bits)
1345 {
1346         struct rb_node *node;
1347         struct extent_state *state;
1348         u64 cur_start = *start;
1349         u64 total_bytes = 0;
1350         int found = 0;
1351
1352         if (search_end <= cur_start) {
1353                 printk("search_end %Lu start %Lu\n", search_end, cur_start);
1354                 WARN_ON(1);
1355                 return 0;
1356         }
1357
1358         spin_lock_irq(&tree->lock);
1359         if (cur_start == 0 && bits == EXTENT_DIRTY) {
1360                 total_bytes = tree->dirty_bytes;
1361                 goto out;
1362         }
1363         /*
1364          * this search will find all the extents that end after
1365          * our range starts.
1366          */
1367         node = tree_search(tree, cur_start);
1368         if (!node) {
1369                 goto out;
1370         }
1371
1372         while(1) {
1373                 state = rb_entry(node, struct extent_state, rb_node);
1374                 if (state->start > search_end)
1375                         break;
1376                 if (state->end >= cur_start && (state->state & bits)) {
1377                         total_bytes += min(search_end, state->end) + 1 -
1378                                        max(cur_start, state->start);
1379                         if (total_bytes >= max_bytes)
1380                                 break;
1381                         if (!found) {
1382                                 *start = state->start;
1383                                 found = 1;
1384                         }
1385                 }
1386                 node = rb_next(node);
1387                 if (!node)
1388                         break;
1389         }
1390 out:
1391         spin_unlock_irq(&tree->lock);
1392         return total_bytes;
1393 }
1394 /*
1395  * helper function to lock both pages and extents in the tree.
1396  * pages must be locked first.
1397  */
1398 int lock_range(struct extent_io_tree *tree, u64 start, u64 end)
1399 {
1400         unsigned long index = start >> PAGE_CACHE_SHIFT;
1401         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1402         struct page *page;
1403         int err;
1404
1405         while (index <= end_index) {
1406                 page = grab_cache_page(tree->mapping, index);
1407                 if (!page) {
1408                         err = -ENOMEM;
1409                         goto failed;
1410                 }
1411                 if (IS_ERR(page)) {
1412                         err = PTR_ERR(page);
1413                         goto failed;
1414                 }
1415                 index++;
1416         }
1417         lock_extent(tree, start, end, GFP_NOFS);
1418         return 0;
1419
1420 failed:
1421         /*
1422          * we failed above in getting the page at 'index', so we undo here
1423          * up to but not including the page at 'index'
1424          */
1425         end_index = index;
1426         index = start >> PAGE_CACHE_SHIFT;
1427         while (index < end_index) {
1428                 page = find_get_page(tree->mapping, index);
1429                 unlock_page(page);
1430                 page_cache_release(page);
1431                 index++;
1432         }
1433         return err;
1434 }
1435 EXPORT_SYMBOL(lock_range);
1436
1437 /*
1438  * helper function to unlock both pages and extents in the tree.
1439  */
1440 int unlock_range(struct extent_io_tree *tree, u64 start, u64 end)
1441 {
1442         unsigned long index = start >> PAGE_CACHE_SHIFT;
1443         unsigned long end_index = end >> PAGE_CACHE_SHIFT;
1444         struct page *page;
1445
1446         while (index <= end_index) {
1447                 page = find_get_page(tree->mapping, index);
1448                 unlock_page(page);
1449                 page_cache_release(page);
1450                 index++;
1451         }
1452         unlock_extent(tree, start, end, GFP_NOFS);
1453         return 0;
1454 }
1455 EXPORT_SYMBOL(unlock_range);
1456
1457 /*
1458  * set the private field for a given byte offset in the tree.  If there isn't
1459  * an extent_state there already, this does nothing.
1460  */
1461 int set_state_private(struct extent_io_tree *tree, u64 start, u64 private)
1462 {
1463         struct rb_node *node;
1464         struct extent_state *state;
1465         int ret = 0;
1466
1467         spin_lock_irq(&tree->lock);
1468         /*
1469          * this search will find all the extents that end after
1470          * our range starts.
1471          */
1472         node = tree_search(tree, start);
1473         if (!node) {
1474                 ret = -ENOENT;
1475                 goto out;
1476         }
1477         state = rb_entry(node, struct extent_state, rb_node);
1478         if (state->start != start) {
1479                 ret = -ENOENT;
1480                 goto out;
1481         }
1482         state->private = private;
1483 out:
1484         spin_unlock_irq(&tree->lock);
1485         return ret;
1486 }
1487
1488 int get_state_private(struct extent_io_tree *tree, u64 start, u64 *private)
1489 {
1490         struct rb_node *node;
1491         struct extent_state *state;
1492         int ret = 0;
1493
1494         spin_lock_irq(&tree->lock);
1495         /*
1496          * this search will find all the extents that end after
1497          * our range starts.
1498          */
1499         node = tree_search(tree, start);
1500         if (!node) {
1501                 ret = -ENOENT;
1502                 goto out;
1503         }
1504         state = rb_entry(node, struct extent_state, rb_node);
1505         if (state->start != start) {
1506                 ret = -ENOENT;
1507                 goto out;
1508         }
1509         *private = state->private;
1510 out:
1511         spin_unlock_irq(&tree->lock);
1512         return ret;
1513 }
1514
1515 /*
1516  * searches a range in the state tree for a given mask.
1517  * If 'filled' == 1, this returns 1 only if every extent in the tree
1518  * has the bits set.  Otherwise, 1 is returned if any bit in the
1519  * range is found set.
1520  */
1521 int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1522                    int bits, int filled)
1523 {
1524         struct extent_state *state = NULL;
1525         struct rb_node *node;
1526         int bitset = 0;
1527         unsigned long flags;
1528
1529         spin_lock_irqsave(&tree->lock, flags);
1530         node = tree_search(tree, start);
1531         while (node && start <= end) {
1532                 state = rb_entry(node, struct extent_state, rb_node);
1533
1534                 if (filled && state->start > start) {
1535                         bitset = 0;
1536                         break;
1537                 }
1538
1539                 if (state->start > end)
1540                         break;
1541
1542                 if (state->state & bits) {
1543                         bitset = 1;
1544                         if (!filled)
1545                                 break;
1546                 } else if (filled) {
1547                         bitset = 0;
1548                         break;
1549                 }
1550                 start = state->end + 1;
1551                 if (start > end)
1552                         break;
1553                 node = rb_next(node);
1554                 if (!node) {
1555                         if (filled)
1556                                 bitset = 0;
1557                         break;
1558                 }
1559         }
1560         spin_unlock_irqrestore(&tree->lock, flags);
1561         return bitset;
1562 }
1563 EXPORT_SYMBOL(test_range_bit);
1564
1565 /*
1566  * helper function to set a given page up to date if all the
1567  * extents in the tree for that page are up to date
1568  */
1569 static int check_page_uptodate(struct extent_io_tree *tree,
1570                                struct page *page)
1571 {
1572         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1573         u64 end = start + PAGE_CACHE_SIZE - 1;
1574         if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1))
1575                 SetPageUptodate(page);
1576         return 0;
1577 }
1578
1579 /*
1580  * helper function to unlock a page if all the extents in the tree
1581  * for that page are unlocked
1582  */
1583 static int check_page_locked(struct extent_io_tree *tree,
1584                              struct page *page)
1585 {
1586         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1587         u64 end = start + PAGE_CACHE_SIZE - 1;
1588         if (!test_range_bit(tree, start, end, EXTENT_LOCKED, 0))
1589                 unlock_page(page);
1590         return 0;
1591 }
1592
1593 /*
1594  * helper function to end page writeback if all the extents
1595  * in the tree for that page are done with writeback
1596  */
1597 static int check_page_writeback(struct extent_io_tree *tree,
1598                              struct page *page)
1599 {
1600         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1601         u64 end = start + PAGE_CACHE_SIZE - 1;
1602         if (!test_range_bit(tree, start, end, EXTENT_WRITEBACK, 0))
1603                 end_page_writeback(page);
1604         return 0;
1605 }
1606
1607 /* lots and lots of room for performance fixes in the end_bio funcs */
1608
1609 /*
1610  * after a writepage IO is done, we need to:
1611  * clear the uptodate bits on error
1612  * clear the writeback bits in the extent tree for this IO
1613  * end_page_writeback if the page has no more pending IO
1614  *
1615  * Scheduling is not allowed, so the extent state tree is expected
1616  * to have one and only one object corresponding to this IO.
1617  */
1618 static void end_bio_extent_writepage(struct bio *bio, int err)
1619 {
1620         int uptodate = err == 0;
1621         struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1622         struct extent_io_tree *tree;
1623         u64 start;
1624         u64 end;
1625         int whole_page;
1626         int ret;
1627
1628         do {
1629                 struct page *page = bvec->bv_page;
1630                 tree = &BTRFS_I(page->mapping->host)->io_tree;
1631
1632                 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
1633                          bvec->bv_offset;
1634                 end = start + bvec->bv_len - 1;
1635
1636                 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
1637                         whole_page = 1;
1638                 else
1639                         whole_page = 0;
1640
1641                 if (--bvec >= bio->bi_io_vec)
1642                         prefetchw(&bvec->bv_page->flags);
1643                 if (tree->ops && tree->ops->writepage_end_io_hook) {
1644                         ret = tree->ops->writepage_end_io_hook(page, start,
1645                                                        end, NULL, uptodate);
1646                         if (ret)
1647                                 uptodate = 0;
1648                 }
1649
1650                 if (!uptodate && tree->ops &&
1651                     tree->ops->writepage_io_failed_hook) {
1652                         ret = tree->ops->writepage_io_failed_hook(bio, page,
1653                                                          start, end, NULL);
1654                         if (ret == 0) {
1655                                 uptodate = (err == 0);
1656                                 continue;
1657                         }
1658                 }
1659
1660                 if (!uptodate) {
1661                         clear_extent_uptodate(tree, start, end, GFP_ATOMIC);
1662                         ClearPageUptodate(page);
1663                         SetPageError(page);
1664                 }
1665
1666                 clear_extent_writeback(tree, start, end, GFP_ATOMIC);
1667
1668                 if (whole_page)
1669                         end_page_writeback(page);
1670                 else
1671                         check_page_writeback(tree, page);
1672         } while (bvec >= bio->bi_io_vec);
1673
1674         bio_put(bio);
1675 }
1676
1677 /*
1678  * after a readpage IO is done, we need to:
1679  * clear the uptodate bits on error
1680  * set the uptodate bits if things worked
1681  * set the page up to date if all extents in the tree are uptodate
1682  * clear the lock bit in the extent tree
1683  * unlock the page if there are no other extents locked for it
1684  *
1685  * Scheduling is not allowed, so the extent state tree is expected
1686  * to have one and only one object corresponding to this IO.
1687  */
1688 static void end_bio_extent_readpage(struct bio *bio, int err)
1689 {
1690         int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1691         struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1692         struct extent_io_tree *tree;
1693         u64 start;
1694         u64 end;
1695         int whole_page;
1696         int ret;
1697
1698         do {
1699                 struct page *page = bvec->bv_page;
1700                 tree = &BTRFS_I(page->mapping->host)->io_tree;
1701
1702                 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
1703                         bvec->bv_offset;
1704                 end = start + bvec->bv_len - 1;
1705
1706                 if (bvec->bv_offset == 0 && bvec->bv_len == PAGE_CACHE_SIZE)
1707                         whole_page = 1;
1708                 else
1709                         whole_page = 0;
1710
1711                 if (--bvec >= bio->bi_io_vec)
1712                         prefetchw(&bvec->bv_page->flags);
1713
1714                 if (uptodate && tree->ops && tree->ops->readpage_end_io_hook) {
1715                         ret = tree->ops->readpage_end_io_hook(page, start, end,
1716                                                               NULL);
1717                         if (ret)
1718                                 uptodate = 0;
1719                 }
1720                 if (!uptodate && tree->ops &&
1721                     tree->ops->readpage_io_failed_hook) {
1722                         ret = tree->ops->readpage_io_failed_hook(bio, page,
1723                                                          start, end, NULL);
1724                         if (ret == 0) {
1725                                 uptodate =
1726                                         test_bit(BIO_UPTODATE, &bio->bi_flags);
1727                                 continue;
1728                         }
1729                 }
1730
1731                 if (uptodate)
1732                         set_extent_uptodate(tree, start, end,
1733                                             GFP_ATOMIC);
1734                 unlock_extent(tree, start, end, GFP_ATOMIC);
1735
1736                 if (whole_page) {
1737                         if (uptodate) {
1738                                 SetPageUptodate(page);
1739                         } else {
1740                                 ClearPageUptodate(page);
1741                                 SetPageError(page);
1742                         }
1743                         unlock_page(page);
1744                 } else {
1745                         if (uptodate) {
1746                                 check_page_uptodate(tree, page);
1747                         } else {
1748                                 ClearPageUptodate(page);
1749                                 SetPageError(page);
1750                         }
1751                         check_page_locked(tree, page);
1752                 }
1753         } while (bvec >= bio->bi_io_vec);
1754
1755         bio_put(bio);
1756 }
1757
1758 /*
1759  * IO done from prepare_write is pretty simple, we just unlock
1760  * the structs in the extent tree when done, and set the uptodate bits
1761  * as appropriate.
1762  */
1763 static void end_bio_extent_preparewrite(struct bio *bio, int err)
1764 {
1765         const int uptodate = test_bit(BIO_UPTODATE, &bio->bi_flags);
1766         struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1767         struct extent_io_tree *tree;
1768         u64 start;
1769         u64 end;
1770
1771         do {
1772                 struct page *page = bvec->bv_page;
1773                 tree = &BTRFS_I(page->mapping->host)->io_tree;
1774
1775                 start = ((u64)page->index << PAGE_CACHE_SHIFT) +
1776                         bvec->bv_offset;
1777                 end = start + bvec->bv_len - 1;
1778
1779                 if (--bvec >= bio->bi_io_vec)
1780                         prefetchw(&bvec->bv_page->flags);
1781
1782                 if (uptodate) {
1783                         set_extent_uptodate(tree, start, end, GFP_ATOMIC);
1784                 } else {
1785                         ClearPageUptodate(page);
1786                         SetPageError(page);
1787                 }
1788
1789                 unlock_extent(tree, start, end, GFP_ATOMIC);
1790
1791         } while (bvec >= bio->bi_io_vec);
1792
1793         bio_put(bio);
1794 }
1795
1796 static struct bio *
1797 extent_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
1798                  gfp_t gfp_flags)
1799 {
1800         struct bio *bio;
1801
1802         bio = bio_alloc(gfp_flags, nr_vecs);
1803
1804         if (bio == NULL && (current->flags & PF_MEMALLOC)) {
1805                 while (!bio && (nr_vecs /= 2))
1806                         bio = bio_alloc(gfp_flags, nr_vecs);
1807         }
1808
1809         if (bio) {
1810                 bio->bi_size = 0;
1811                 bio->bi_bdev = bdev;
1812                 bio->bi_sector = first_sector;
1813         }
1814         return bio;
1815 }
1816
1817 static int submit_one_bio(int rw, struct bio *bio, int mirror_num,
1818                           unsigned long bio_flags)
1819 {
1820         int ret = 0;
1821         struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
1822         struct page *page = bvec->bv_page;
1823         struct extent_io_tree *tree = bio->bi_private;
1824         u64 start;
1825         u64 end;
1826
1827         start = ((u64)page->index << PAGE_CACHE_SHIFT) + bvec->bv_offset;
1828         end = start + bvec->bv_len - 1;
1829
1830         bio->bi_private = NULL;
1831
1832         bio_get(bio);
1833
1834         if (tree->ops && tree->ops->submit_bio_hook)
1835                 tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
1836                                            mirror_num, bio_flags);
1837         else
1838                 submit_bio(rw, bio);
1839         if (bio_flagged(bio, BIO_EOPNOTSUPP))
1840                 ret = -EOPNOTSUPP;
1841         bio_put(bio);
1842         return ret;
1843 }
1844
1845 static int submit_extent_page(int rw, struct extent_io_tree *tree,
1846                               struct page *page, sector_t sector,
1847                               size_t size, unsigned long offset,
1848                               struct block_device *bdev,
1849                               struct bio **bio_ret,
1850                               unsigned long max_pages,
1851                               bio_end_io_t end_io_func,
1852                               int mirror_num,
1853                               unsigned long prev_bio_flags,
1854                               unsigned long bio_flags)
1855 {
1856         int ret = 0;
1857         struct bio *bio;
1858         int nr;
1859         int contig = 0;
1860         int this_compressed = bio_flags & EXTENT_BIO_COMPRESSED;
1861         int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
1862         size_t page_size = min(size, PAGE_CACHE_SIZE);
1863
1864         if (bio_ret && *bio_ret) {
1865                 bio = *bio_ret;
1866                 if (old_compressed)
1867                         contig = bio->bi_sector == sector;
1868                 else
1869                         contig = bio->bi_sector + (bio->bi_size >> 9) ==
1870                                 sector;
1871
1872                 if (prev_bio_flags != bio_flags || !contig ||
1873                     (tree->ops && tree->ops->merge_bio_hook &&
1874                      tree->ops->merge_bio_hook(page, offset, page_size, bio,
1875                                                bio_flags)) ||
1876                     bio_add_page(bio, page, page_size, offset) < page_size) {
1877                         ret = submit_one_bio(rw, bio, mirror_num,
1878                                              prev_bio_flags);
1879                         bio = NULL;
1880                 } else {
1881                         return 0;
1882                 }
1883         }
1884         if (this_compressed)
1885                 nr = BIO_MAX_PAGES;
1886         else
1887                 nr = bio_get_nr_vecs(bdev);
1888
1889         bio = extent_bio_alloc(bdev, sector, nr, GFP_NOFS | __GFP_HIGH);
1890         if (!bio) {
1891                 printk("failed to allocate bio nr %d\n", nr);
1892         }
1893
1894         bio_add_page(bio, page, page_size, offset);
1895         bio->bi_end_io = end_io_func;
1896         bio->bi_private = tree;
1897
1898         if (bio_ret) {
1899                 *bio_ret = bio;
1900         } else {
1901                 ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
1902         }
1903
1904         return ret;
1905 }
1906
1907 void set_page_extent_mapped(struct page *page)
1908 {
1909         if (!PagePrivate(page)) {
1910                 SetPagePrivate(page);
1911                 page_cache_get(page);
1912                 set_page_private(page, EXTENT_PAGE_PRIVATE);
1913         }
1914 }
1915
1916 void set_page_extent_head(struct page *page, unsigned long len)
1917 {
1918         set_page_private(page, EXTENT_PAGE_PRIVATE_FIRST_PAGE | len << 2);
1919 }
1920
1921 /*
1922  * basic readpage implementation.  Locked extent state structs are inserted
1923  * into the tree that are removed when the IO is done (by the end_io
1924  * handlers)
1925  */
1926 static int __extent_read_full_page(struct extent_io_tree *tree,
1927                                    struct page *page,
1928                                    get_extent_t *get_extent,
1929                                    struct bio **bio, int mirror_num,
1930                                    unsigned long *bio_flags)
1931 {
1932         struct inode *inode = page->mapping->host;
1933         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
1934         u64 page_end = start + PAGE_CACHE_SIZE - 1;
1935         u64 end;
1936         u64 cur = start;
1937         u64 extent_offset;
1938         u64 last_byte = i_size_read(inode);
1939         u64 block_start;
1940         u64 cur_end;
1941         sector_t sector;
1942         struct extent_map *em;
1943         struct block_device *bdev;
1944         int ret;
1945         int nr = 0;
1946         size_t page_offset = 0;
1947         size_t iosize;
1948         size_t disk_io_size;
1949         size_t blocksize = inode->i_sb->s_blocksize;
1950         unsigned long this_bio_flag = 0;
1951
1952         set_page_extent_mapped(page);
1953
1954         end = page_end;
1955         lock_extent(tree, start, end, GFP_NOFS);
1956
1957         if (page->index == last_byte >> PAGE_CACHE_SHIFT) {
1958                 char *userpage;
1959                 size_t zero_offset = last_byte & (PAGE_CACHE_SIZE - 1);
1960
1961                 if (zero_offset) {
1962                         iosize = PAGE_CACHE_SIZE - zero_offset;
1963                         userpage = kmap_atomic(page, KM_USER0);
1964                         memset(userpage + zero_offset, 0, iosize);
1965                         flush_dcache_page(page);
1966                         kunmap_atomic(userpage, KM_USER0);
1967                 }
1968         }
1969         while (cur <= end) {
1970                 if (cur >= last_byte) {
1971                         char *userpage;
1972                         iosize = PAGE_CACHE_SIZE - page_offset;
1973                         userpage = kmap_atomic(page, KM_USER0);
1974                         memset(userpage + page_offset, 0, iosize);
1975                         flush_dcache_page(page);
1976                         kunmap_atomic(userpage, KM_USER0);
1977                         set_extent_uptodate(tree, cur, cur + iosize - 1,
1978                                             GFP_NOFS);
1979                         unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
1980                         break;
1981                 }
1982                 em = get_extent(inode, page, page_offset, cur,
1983                                 end - cur + 1, 0);
1984                 if (IS_ERR(em) || !em) {
1985                         SetPageError(page);
1986                         unlock_extent(tree, cur, end, GFP_NOFS);
1987                         break;
1988                 }
1989                 extent_offset = cur - em->start;
1990                 if (extent_map_end(em) <= cur) {
1991 printk("bad mapping em [%Lu %Lu] cur %Lu\n", em->start, extent_map_end(em), cur);
1992                 }
1993                 BUG_ON(extent_map_end(em) <= cur);
1994                 if (end < cur) {
1995 printk("2bad mapping end %Lu cur %Lu\n", end, cur);
1996                 }
1997                 BUG_ON(end < cur);
1998
1999                 if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
2000                         this_bio_flag = EXTENT_BIO_COMPRESSED;
2001
2002                 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2003                 cur_end = min(extent_map_end(em) - 1, end);
2004                 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
2005                 if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2006                         disk_io_size = em->block_len;
2007                         sector = em->block_start >> 9;
2008                 } else {
2009                         sector = (em->block_start + extent_offset) >> 9;
2010                         disk_io_size = iosize;
2011                 }
2012                 bdev = em->bdev;
2013                 block_start = em->block_start;
2014                 free_extent_map(em);
2015                 em = NULL;
2016
2017                 /* we've found a hole, just zero and go on */
2018                 if (block_start == EXTENT_MAP_HOLE) {
2019                         char *userpage;
2020                         userpage = kmap_atomic(page, KM_USER0);
2021                         memset(userpage + page_offset, 0, iosize);
2022                         flush_dcache_page(page);
2023                         kunmap_atomic(userpage, KM_USER0);
2024
2025                         set_extent_uptodate(tree, cur, cur + iosize - 1,
2026                                             GFP_NOFS);
2027                         unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
2028                         cur = cur + iosize;
2029                         page_offset += iosize;
2030                         continue;
2031                 }
2032                 /* the get_extent function already copied into the page */
2033                 if (test_range_bit(tree, cur, cur_end, EXTENT_UPTODATE, 1)) {
2034                         check_page_uptodate(tree, page);
2035                         unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
2036                         cur = cur + iosize;
2037                         page_offset += iosize;
2038                         continue;
2039                 }
2040                 /* we have an inline extent but it didn't get marked up
2041                  * to date.  Error out
2042                  */
2043                 if (block_start == EXTENT_MAP_INLINE) {
2044                         SetPageError(page);
2045                         unlock_extent(tree, cur, cur + iosize - 1, GFP_NOFS);
2046                         cur = cur + iosize;
2047                         page_offset += iosize;
2048                         continue;
2049                 }
2050
2051                 ret = 0;
2052                 if (tree->ops && tree->ops->readpage_io_hook) {
2053                         ret = tree->ops->readpage_io_hook(page, cur,
2054                                                           cur + iosize - 1);
2055                 }
2056                 if (!ret) {
2057                         unsigned long pnr = (last_byte >> PAGE_CACHE_SHIFT) + 1;
2058                         pnr -= page->index;
2059                         ret = submit_extent_page(READ, tree, page,
2060                                          sector, disk_io_size, page_offset,
2061                                          bdev, bio, pnr,
2062                                          end_bio_extent_readpage, mirror_num,
2063                                          *bio_flags,
2064                                          this_bio_flag);
2065                         nr++;
2066                         *bio_flags = this_bio_flag;
2067                 }
2068                 if (ret)
2069                         SetPageError(page);
2070                 cur = cur + iosize;
2071                 page_offset += iosize;
2072         }
2073         if (!nr) {
2074                 if (!PageError(page))
2075                         SetPageUptodate(page);
2076                 unlock_page(page);
2077         }
2078         return 0;
2079 }
2080
2081 int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
2082                             get_extent_t *get_extent)
2083 {
2084         struct bio *bio = NULL;
2085         unsigned long bio_flags = 0;
2086         int ret;
2087
2088         ret = __extent_read_full_page(tree, page, get_extent, &bio, 0,
2089                                       &bio_flags);
2090         if (bio)
2091                 submit_one_bio(READ, bio, 0, bio_flags);
2092         return ret;
2093 }
2094 EXPORT_SYMBOL(extent_read_full_page);
2095
2096 /*
2097  * the writepage semantics are similar to regular writepage.  extent
2098  * records are inserted to lock ranges in the tree, and as dirty areas
2099  * are found, they are marked writeback.  Then the lock bits are removed
2100  * and the end_io handler clears the writeback ranges
2101  */
2102 static int __extent_writepage(struct page *page, struct writeback_control *wbc,
2103                               void *data)
2104 {
2105         struct inode *inode = page->mapping->host;
2106         struct extent_page_data *epd = data;
2107         struct extent_io_tree *tree = epd->tree;
2108         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2109         u64 delalloc_start;
2110         u64 page_end = start + PAGE_CACHE_SIZE - 1;
2111         u64 end;
2112         u64 cur = start;
2113         u64 extent_offset;
2114         u64 last_byte = i_size_read(inode);
2115         u64 block_start;
2116         u64 iosize;
2117         u64 unlock_start;
2118         sector_t sector;
2119         struct extent_map *em;
2120         struct block_device *bdev;
2121         int ret;
2122         int nr = 0;
2123         size_t pg_offset = 0;
2124         size_t blocksize;
2125         loff_t i_size = i_size_read(inode);
2126         unsigned long end_index = i_size >> PAGE_CACHE_SHIFT;
2127         u64 nr_delalloc;
2128         u64 delalloc_end;
2129         int page_started;
2130         int compressed;
2131
2132         WARN_ON(!PageLocked(page));
2133         pg_offset = i_size & (PAGE_CACHE_SIZE - 1);
2134         if (page->index > end_index ||
2135            (page->index == end_index && !pg_offset)) {
2136                 page->mapping->a_ops->invalidatepage(page, 0);
2137                 unlock_page(page);
2138                 return 0;
2139         }
2140
2141         if (page->index == end_index) {
2142                 char *userpage;
2143
2144                 userpage = kmap_atomic(page, KM_USER0);
2145                 memset(userpage + pg_offset, 0,
2146                        PAGE_CACHE_SIZE - pg_offset);
2147                 kunmap_atomic(userpage, KM_USER0);
2148                 flush_dcache_page(page);
2149         }
2150         pg_offset = 0;
2151
2152         set_page_extent_mapped(page);
2153
2154         delalloc_start = start;
2155         delalloc_end = 0;
2156         page_started = 0;
2157         while(delalloc_end < page_end) {
2158                 nr_delalloc = find_lock_delalloc_range(inode, tree,
2159                                                        page,
2160                                                        &delalloc_start,
2161                                                        &delalloc_end,
2162                                                        128 * 1024 * 1024);
2163                 if (nr_delalloc == 0) {
2164                         delalloc_start = delalloc_end + 1;
2165                         continue;
2166                 }
2167                 tree->ops->fill_delalloc(inode, page, delalloc_start,
2168                                          delalloc_end, &page_started);
2169                 delalloc_start = delalloc_end + 1;
2170         }
2171
2172         /* did the fill delalloc function already unlock and start the IO? */
2173         if (page_started) {
2174                 return 0;
2175         }
2176
2177         lock_extent(tree, start, page_end, GFP_NOFS);
2178         unlock_start = start;
2179
2180         if (tree->ops && tree->ops->writepage_start_hook) {
2181                 ret = tree->ops->writepage_start_hook(page, start,
2182                                                       page_end);
2183                 if (ret == -EAGAIN) {
2184                         unlock_extent(tree, start, page_end, GFP_NOFS);
2185                         redirty_page_for_writepage(wbc, page);
2186                         unlock_page(page);
2187                         return 0;
2188                 }
2189         }
2190
2191         end = page_end;
2192         if (test_range_bit(tree, start, page_end, EXTENT_DELALLOC, 0)) {
2193                 printk("found delalloc bits after lock_extent\n");
2194         }
2195
2196         if (last_byte <= start) {
2197                 clear_extent_dirty(tree, start, page_end, GFP_NOFS);
2198                 unlock_extent(tree, start, page_end, GFP_NOFS);
2199                 if (tree->ops && tree->ops->writepage_end_io_hook)
2200                         tree->ops->writepage_end_io_hook(page, start,
2201                                                          page_end, NULL, 1);
2202                 unlock_start = page_end + 1;
2203                 goto done;
2204         }
2205
2206         set_extent_uptodate(tree, start, page_end, GFP_NOFS);
2207         blocksize = inode->i_sb->s_blocksize;
2208
2209         while (cur <= end) {
2210                 if (cur >= last_byte) {
2211                         clear_extent_dirty(tree, cur, page_end, GFP_NOFS);
2212                         unlock_extent(tree, unlock_start, page_end, GFP_NOFS);
2213                         if (tree->ops && tree->ops->writepage_end_io_hook)
2214                                 tree->ops->writepage_end_io_hook(page, cur,
2215                                                          page_end, NULL, 1);
2216                         unlock_start = page_end + 1;
2217                         break;
2218                 }
2219                 em = epd->get_extent(inode, page, pg_offset, cur,
2220                                      end - cur + 1, 1);
2221                 if (IS_ERR(em) || !em) {
2222                         SetPageError(page);
2223                         break;
2224                 }
2225
2226                 extent_offset = cur - em->start;
2227                 BUG_ON(extent_map_end(em) <= cur);
2228                 BUG_ON(end < cur);
2229                 iosize = min(extent_map_end(em) - cur, end - cur + 1);
2230                 iosize = (iosize + blocksize - 1) & ~((u64)blocksize - 1);
2231                 sector = (em->block_start + extent_offset) >> 9;
2232                 bdev = em->bdev;
2233                 block_start = em->block_start;
2234                 compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
2235                 free_extent_map(em);
2236                 em = NULL;
2237
2238                 /*
2239                  * compressed and inline extents are written through other
2240                  * paths in the FS
2241                  */
2242                 if (compressed || block_start == EXTENT_MAP_HOLE ||
2243                     block_start == EXTENT_MAP_INLINE) {
2244                         clear_extent_dirty(tree, cur,
2245                                            cur + iosize - 1, GFP_NOFS);
2246
2247                         unlock_extent(tree, unlock_start, cur + iosize -1,
2248                                       GFP_NOFS);
2249
2250                         /*
2251                          * end_io notification does not happen here for
2252                          * compressed extents
2253                          */
2254                         if (!compressed && tree->ops &&
2255                             tree->ops->writepage_end_io_hook)
2256                                 tree->ops->writepage_end_io_hook(page, cur,
2257                                                          cur + iosize - 1,
2258                                                          NULL, 1);
2259                         else if (compressed) {
2260                                 /* we don't want to end_page_writeback on
2261                                  * a compressed extent.  this happens
2262                                  * elsewhere
2263                                  */
2264                                 nr++;
2265                         }
2266
2267                         cur += iosize;
2268                         pg_offset += iosize;
2269                         unlock_start = cur;
2270                         continue;
2271                 }
2272                 /* leave this out until we have a page_mkwrite call */
2273                 if (0 && !test_range_bit(tree, cur, cur + iosize - 1,
2274                                    EXTENT_DIRTY, 0)) {
2275                         cur = cur + iosize;
2276                         pg_offset += iosize;
2277                         continue;
2278                 }
2279
2280                 clear_extent_dirty(tree, cur, cur + iosize - 1, GFP_NOFS);
2281                 if (tree->ops && tree->ops->writepage_io_hook) {
2282                         ret = tree->ops->writepage_io_hook(page, cur,
2283                                                 cur + iosize - 1);
2284                 } else {
2285                         ret = 0;
2286                 }
2287                 if (ret) {
2288                         SetPageError(page);
2289                 } else {
2290                         unsigned long max_nr = end_index + 1;
2291
2292                         set_range_writeback(tree, cur, cur + iosize - 1);
2293                         if (!PageWriteback(page)) {
2294                                 printk("warning page %lu not writeback, "
2295                                        "cur %llu end %llu\n", page->index,
2296                                        (unsigned long long)cur,
2297                                        (unsigned long long)end);
2298                         }
2299
2300                         ret = submit_extent_page(WRITE, tree, page, sector,
2301                                                  iosize, pg_offset, bdev,
2302                                                  &epd->bio, max_nr,
2303                                                  end_bio_extent_writepage,
2304                                                  0, 0, 0);
2305                         if (ret)
2306                                 SetPageError(page);
2307                 }
2308                 cur = cur + iosize;
2309                 pg_offset += iosize;
2310                 nr++;
2311         }
2312 done:
2313         if (nr == 0) {
2314                 /* make sure the mapping tag for page dirty gets cleared */
2315                 set_page_writeback(page);
2316                 end_page_writeback(page);
2317         }
2318         if (unlock_start <= page_end)
2319                 unlock_extent(tree, unlock_start, page_end, GFP_NOFS);
2320         unlock_page(page);
2321         return 0;
2322 }
2323
2324 /**
2325  * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
2326  * @mapping: address space structure to write
2327  * @wbc: subtract the number of written pages from *@wbc->nr_to_write
2328  * @writepage: function called for each page
2329  * @data: data passed to writepage function
2330  *
2331  * If a page is already under I/O, write_cache_pages() skips it, even
2332  * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
2333  * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
2334  * and msync() need to guarantee that all the data which was dirty at the time
2335  * the call was made get new I/O started against them.  If wbc->sync_mode is
2336  * WB_SYNC_ALL then we were called for data integrity and we must wait for
2337  * existing IO to complete.
2338  */
2339 int extent_write_cache_pages(struct extent_io_tree *tree,
2340                              struct address_space *mapping,
2341                              struct writeback_control *wbc,
2342                              writepage_t writepage, void *data)
2343 {
2344         struct backing_dev_info *bdi = mapping->backing_dev_info;
2345         int ret = 0;
2346         int done = 0;
2347         struct pagevec pvec;
2348         int nr_pages;
2349         pgoff_t index;
2350         pgoff_t end;            /* Inclusive */
2351         int scanned = 0;
2352         int range_whole = 0;
2353
2354         if (wbc->nonblocking && bdi_write_congested(bdi)) {
2355                 wbc->encountered_congestion = 1;
2356                 return 0;
2357         }
2358
2359         pagevec_init(&pvec, 0);
2360         if (wbc->range_cyclic) {
2361                 index = mapping->writeback_index; /* Start from prev offset */
2362                 end = -1;
2363         } else {
2364                 index = wbc->range_start >> PAGE_CACHE_SHIFT;
2365                 end = wbc->range_end >> PAGE_CACHE_SHIFT;
2366                 if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2367                         range_whole = 1;
2368                 scanned = 1;
2369         }
2370 retry:
2371         while (!done && (index <= end) &&
2372                (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index,
2373                                               PAGECACHE_TAG_DIRTY,
2374                                               min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
2375                 unsigned i;
2376
2377                 scanned = 1;
2378                 for (i = 0; i < nr_pages; i++) {
2379                         struct page *page = pvec.pages[i];
2380
2381                         /*
2382                          * At this point we hold neither mapping->tree_lock nor
2383                          * lock on the page itself: the page may be truncated or
2384                          * invalidated (changing page->mapping to NULL), or even
2385                          * swizzled back from swapper_space to tmpfs file
2386                          * mapping
2387                          */
2388                         if (tree->ops && tree->ops->write_cache_pages_lock_hook)
2389                                 tree->ops->write_cache_pages_lock_hook(page);
2390                         else
2391                                 lock_page(page);
2392
2393                         if (unlikely(page->mapping != mapping)) {
2394                                 unlock_page(page);
2395                                 continue;
2396                         }
2397
2398                         if (!wbc->range_cyclic && page->index > end) {
2399                                 done = 1;
2400                                 unlock_page(page);
2401                                 continue;
2402                         }
2403
2404                         if (wbc->sync_mode != WB_SYNC_NONE)
2405                                 wait_on_page_writeback(page);
2406
2407                         if (PageWriteback(page) ||
2408                             !clear_page_dirty_for_io(page)) {
2409                                 unlock_page(page);
2410                                 continue;
2411                         }
2412
2413                         ret = (*writepage)(page, wbc, data);
2414
2415                         if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
2416                                 unlock_page(page);
2417                                 ret = 0;
2418                         }
2419                         if (ret || (--(wbc->nr_to_write) <= 0))
2420                                 done = 1;
2421                         if (wbc->nonblocking && bdi_write_congested(bdi)) {
2422                                 wbc->encountered_congestion = 1;
2423                                 done = 1;
2424                         }
2425                 }
2426                 pagevec_release(&pvec);
2427                 cond_resched();
2428         }
2429         if (!scanned && !done) {
2430                 /*
2431                  * We hit the last page and there is more work to be done: wrap
2432                  * back to the start of the file
2433                  */
2434                 scanned = 1;
2435                 index = 0;
2436                 goto retry;
2437         }
2438         if (wbc->range_cyclic || (range_whole && wbc->nr_to_write > 0))
2439                 mapping->writeback_index = index;
2440
2441         if (wbc->range_cont)
2442                 wbc->range_start = index << PAGE_CACHE_SHIFT;
2443         return ret;
2444 }
2445 EXPORT_SYMBOL(extent_write_cache_pages);
2446
2447 int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
2448                           get_extent_t *get_extent,
2449                           struct writeback_control *wbc)
2450 {
2451         int ret;
2452         struct address_space *mapping = page->mapping;
2453         struct extent_page_data epd = {
2454                 .bio = NULL,
2455                 .tree = tree,
2456                 .get_extent = get_extent,
2457         };
2458         struct writeback_control wbc_writepages = {
2459                 .bdi            = wbc->bdi,
2460                 .sync_mode      = WB_SYNC_NONE,
2461                 .older_than_this = NULL,
2462                 .nr_to_write    = 64,
2463                 .range_start    = page_offset(page) + PAGE_CACHE_SIZE,
2464                 .range_end      = (loff_t)-1,
2465         };
2466
2467
2468         ret = __extent_writepage(page, wbc, &epd);
2469
2470         extent_write_cache_pages(tree, mapping, &wbc_writepages,
2471                                  __extent_writepage, &epd);
2472         if (epd.bio) {
2473                 submit_one_bio(WRITE, epd.bio, 0, 0);
2474         }
2475         return ret;
2476 }
2477 EXPORT_SYMBOL(extent_write_full_page);
2478
2479
2480 int extent_writepages(struct extent_io_tree *tree,
2481                       struct address_space *mapping,
2482                       get_extent_t *get_extent,
2483                       struct writeback_control *wbc)
2484 {
2485         int ret = 0;
2486         struct extent_page_data epd = {
2487                 .bio = NULL,
2488                 .tree = tree,
2489                 .get_extent = get_extent,
2490         };
2491
2492         ret = extent_write_cache_pages(tree, mapping, wbc,
2493                                        __extent_writepage, &epd);
2494         if (epd.bio) {
2495                 submit_one_bio(WRITE, epd.bio, 0, 0);
2496         }
2497         return ret;
2498 }
2499 EXPORT_SYMBOL(extent_writepages);
2500
2501 int extent_readpages(struct extent_io_tree *tree,
2502                      struct address_space *mapping,
2503                      struct list_head *pages, unsigned nr_pages,
2504                      get_extent_t get_extent)
2505 {
2506         struct bio *bio = NULL;
2507         unsigned page_idx;
2508         struct pagevec pvec;
2509         unsigned long bio_flags = 0;
2510
2511         pagevec_init(&pvec, 0);
2512         for (page_idx = 0; page_idx < nr_pages; page_idx++) {
2513                 struct page *page = list_entry(pages->prev, struct page, lru);
2514
2515                 prefetchw(&page->flags);
2516                 list_del(&page->lru);
2517                 /*
2518                  * what we want to do here is call add_to_page_cache_lru,
2519                  * but that isn't exported, so we reproduce it here
2520                  */
2521                 if (!add_to_page_cache(page, mapping,
2522                                         page->index, GFP_KERNEL)) {
2523
2524                         /* open coding of lru_cache_add, also not exported */
2525                         page_cache_get(page);
2526                         if (!pagevec_add(&pvec, page))
2527                                 __pagevec_lru_add(&pvec);
2528                         __extent_read_full_page(tree, page, get_extent,
2529                                                 &bio, 0, &bio_flags);
2530                 }
2531                 page_cache_release(page);
2532         }
2533         if (pagevec_count(&pvec))
2534                 __pagevec_lru_add(&pvec);
2535         BUG_ON(!list_empty(pages));
2536         if (bio)
2537                 submit_one_bio(READ, bio, 0, bio_flags);
2538         return 0;
2539 }
2540 EXPORT_SYMBOL(extent_readpages);
2541
2542 /*
2543  * basic invalidatepage code, this waits on any locked or writeback
2544  * ranges corresponding to the page, and then deletes any extent state
2545  * records from the tree
2546  */
2547 int extent_invalidatepage(struct extent_io_tree *tree,
2548                           struct page *page, unsigned long offset)
2549 {
2550         u64 start = ((u64)page->index << PAGE_CACHE_SHIFT);
2551         u64 end = start + PAGE_CACHE_SIZE - 1;
2552         size_t blocksize = page->mapping->host->i_sb->s_blocksize;
2553
2554         start += (offset + blocksize -1) & ~(blocksize - 1);
2555         if (start > end)
2556                 return 0;
2557
2558         lock_extent(tree, start, end, GFP_NOFS);
2559         wait_on_extent_writeback(tree, start, end);
2560         clear_extent_bit(tree, start, end,
2561                          EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC,
2562                          1, 1, GFP_NOFS);
2563         return 0;
2564 }
2565 EXPORT_SYMBOL(extent_invalidatepage);
2566
2567 /*
2568  * simple commit_write call, set_range_dirty is used to mark both
2569  * the pages and the extent records as dirty
2570  */
2571 int extent_commit_write(struct extent_io_tree *tree,
2572                         struct inode *inode, struct page *page,
2573                         unsigned from, unsigned to)
2574 {
2575         loff_t pos = ((loff_t)page->index << PAGE_CACHE_SHIFT) + to;
2576
2577         set_page_extent_mapped(page);
2578         set_page_dirty(page);
2579
2580         if (pos > inode->i_size) {
2581                 i_size_write(inode, pos);
2582                 mark_inode_dirty(inode);
2583         }
2584         return 0;
2585 }
2586 EXPORT_SYMBOL(extent_commit_write);
2587
2588 int extent_prepare_write(struct extent_io_tree *tree,
2589                          struct inode *inode, struct page *page,
2590                          unsigned from, unsigned to, get_extent_t *get_extent)
2591 {
2592         u64 page_start = (u64)page->index << PAGE_CACHE_SHIFT;
2593         u64 page_end = page_start + PAGE_CACHE_SIZE - 1;
2594         u64 block_start;
2595         u64 orig_block_start;
2596         u64 block_end;
2597         u64 cur_end;
2598         struct extent_map *em;
2599         unsigned blocksize = 1 << inode->i_blkbits;
2600         size_t page_offset = 0;
2601         size_t block_off_start;
2602         size_t block_off_end;
2603         int err = 0;
2604         int iocount = 0;
2605         int ret = 0;
2606         int isnew;
2607
2608         set_page_extent_mapped(page);
2609
2610         block_start = (page_start + from) & ~((u64)blocksize - 1);
2611         block_end = (page_start + to - 1) | (blocksize - 1);
2612         orig_block_start = block_start;
2613
2614         lock_extent(tree, page_start, page_end, GFP_NOFS);
2615         while(block_start <= block_end) {
2616                 em = get_extent(inode, page, page_offset, block_start,
2617                                 block_end - block_start + 1, 1);
2618                 if (IS_ERR(em) || !em) {
2619                         goto err;
2620                 }
2621                 cur_end = min(block_end, extent_map_end(em) - 1);
2622                 block_off_start = block_start & (PAGE_CACHE_SIZE - 1);
2623                 block_off_end = block_off_start + blocksize;
2624                 isnew = clear_extent_new(tree, block_start, cur_end, GFP_NOFS);
2625
2626                 if (!PageUptodate(page) && isnew &&
2627                     (block_off_end > to || block_off_start < from)) {
2628                         void *kaddr;
2629
2630                         kaddr = kmap_atomic(page, KM_USER0);
2631                         if (block_off_end > to)
2632                                 memset(kaddr + to, 0, block_off_end - to);
2633                         if (block_off_start < from)
2634                                 memset(kaddr + block_off_start, 0,
2635                                        from - block_off_start);
2636                         flush_dcache_page(page);
2637                         kunmap_atomic(kaddr, KM_USER0);
2638                 }
2639                 if ((em->block_start != EXTENT_MAP_HOLE &&
2640                      em->block_start != EXTENT_MAP_INLINE) &&
2641                     !isnew && !PageUptodate(page) &&
2642                     (block_off_end > to || block_off_start < from) &&
2643                     !test_range_bit(tree, block_start, cur_end,
2644                                     EXTENT_UPTODATE, 1)) {
2645                         u64 sector;
2646                         u64 extent_offset = block_start - em->start;
2647                         size_t iosize;
2648                         sector = (em->block_start + extent_offset) >> 9;
2649                         iosize = (cur_end - block_start + blocksize) &
2650                                 ~((u64)blocksize - 1);
2651                         /*
2652                          * we've already got the extent locked, but we
2653                          * need to split the state such that our end_bio
2654                          * handler can clear the lock.
2655                          */
2656                         set_extent_bit(tree, block_start,
2657                                        block_start + iosize - 1,
2658                                        EXTENT_LOCKED, 0, NULL, GFP_NOFS);
2659                         ret = submit_extent_page(READ, tree, page,
2660                                          sector, iosize, page_offset, em->bdev,
2661                                          NULL, 1,
2662                                          end_bio_extent_preparewrite, 0,
2663                                          0, 0);
2664                         iocount++;
2665                         block_start = block_start + iosize;
2666                 } else {
2667                         set_extent_uptodate(tree, block_start, cur_end,
2668                                             GFP_NOFS);
2669                         unlock_extent(tree, block_start, cur_end, GFP_NOFS);
2670                         block_start = cur_end + 1;
2671                 }
2672                 page_offset = block_start & (PAGE_CACHE_SIZE - 1);
2673                 free_extent_map(em);
2674         }
2675         if (iocount) {
2676                 wait_extent_bit(tree, orig_block_start,
2677                                 block_end, EXTENT_LOCKED);
2678         }
2679         check_page_uptodate(tree, page);
2680 err:
2681         /* FIXME, zero out newly allocated blocks on error */
2682         return err;
2683 }
2684 EXPORT_SYMBOL(extent_prepare_write);
2685
2686 /*
2687  * a helper for releasepage, this tests for areas of the page that
2688  * are locked or under IO and drops the related state bits if it is safe
2689  * to drop the page.
2690  */
2691 int try_release_extent_state(struct extent_map_tree *map,
2692                              struct extent_io_tree *tree, struct page *page,
2693                              gfp_t mask)
2694 {
2695         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2696         u64 end = start + PAGE_CACHE_SIZE - 1;
2697         int ret = 1;
2698
2699         if (test_range_bit(tree, start, end,
2700                            EXTENT_IOBITS | EXTENT_ORDERED, 0))
2701                 ret = 0;
2702         else {
2703                 if ((mask & GFP_NOFS) == GFP_NOFS)
2704                         mask = GFP_NOFS;
2705                 clear_extent_bit(tree, start, end, EXTENT_UPTODATE,
2706                                  1, 1, mask);
2707         }
2708         return ret;
2709 }
2710 EXPORT_SYMBOL(try_release_extent_state);
2711
2712 /*
2713  * a helper for releasepage.  As long as there are no locked extents
2714  * in the range corresponding to the page, both state records and extent
2715  * map records are removed
2716  */
2717 int try_release_extent_mapping(struct extent_map_tree *map,
2718                                struct extent_io_tree *tree, struct page *page,
2719                                gfp_t mask)
2720 {
2721         struct extent_map *em;
2722         u64 start = (u64)page->index << PAGE_CACHE_SHIFT;
2723         u64 end = start + PAGE_CACHE_SIZE - 1;
2724
2725         if ((mask & __GFP_WAIT) &&
2726             page->mapping->host->i_size > 16 * 1024 * 1024) {
2727                 u64 len;
2728                 while (start <= end) {
2729                         len = end - start + 1;
2730                         spin_lock(&map->lock);
2731                         em = lookup_extent_mapping(map, start, len);
2732                         if (!em || IS_ERR(em)) {
2733                                 spin_unlock(&map->lock);
2734                                 break;
2735                         }
2736                         if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
2737                             em->start != start) {
2738                                 spin_unlock(&map->lock);
2739                                 free_extent_map(em);
2740                                 break;
2741                         }
2742                         if (!test_range_bit(tree, em->start,
2743                                             extent_map_end(em) - 1,
2744                                             EXTENT_LOCKED | EXTENT_WRITEBACK |
2745                                             EXTENT_ORDERED,
2746                                             0)) {
2747                                 remove_extent_mapping(map, em);
2748                                 /* once for the rb tree */
2749                                 free_extent_map(em);
2750                         }
2751                         start = extent_map_end(em);
2752                         spin_unlock(&map->lock);
2753
2754                         /* once for us */
2755                         free_extent_map(em);
2756                 }
2757         }
2758         return try_release_extent_state(map, tree, page, mask);
2759 }
2760 EXPORT_SYMBOL(try_release_extent_mapping);
2761
2762 sector_t extent_bmap(struct address_space *mapping, sector_t iblock,
2763                 get_extent_t *get_extent)
2764 {
2765         struct inode *inode = mapping->host;
2766         u64 start = iblock << inode->i_blkbits;
2767         sector_t sector = 0;
2768         struct extent_map *em;
2769
2770         em = get_extent(inode, NULL, 0, start, (1 << inode->i_blkbits), 0);
2771         if (!em || IS_ERR(em))
2772                 return 0;
2773
2774         if (em->block_start == EXTENT_MAP_INLINE ||
2775             em->block_start == EXTENT_MAP_HOLE)
2776                 goto out;
2777
2778         sector = (em->block_start + start - em->start) >> inode->i_blkbits;
2779 out:
2780         free_extent_map(em);
2781         return sector;
2782 }
2783
2784 static inline struct page *extent_buffer_page(struct extent_buffer *eb,
2785                                               unsigned long i)
2786 {
2787         struct page *p;
2788         struct address_space *mapping;
2789
2790         if (i == 0)
2791                 return eb->first_page;
2792         i += eb->start >> PAGE_CACHE_SHIFT;
2793         mapping = eb->first_page->mapping;
2794         if (!mapping)
2795                 return NULL;
2796
2797         /*
2798          * extent_buffer_page is only called after pinning the page
2799          * by increasing the reference count.  So we know the page must
2800          * be in the radix tree.
2801          */
2802         rcu_read_lock();
2803         p = radix_tree_lookup(&mapping->page_tree, i);
2804         rcu_read_unlock();
2805
2806         return p;
2807 }
2808
2809 static inline unsigned long num_extent_pages(u64 start, u64 len)
2810 {
2811         return ((start + len + PAGE_CACHE_SIZE - 1) >> PAGE_CACHE_SHIFT) -
2812                 (start >> PAGE_CACHE_SHIFT);
2813 }
2814
2815 static struct extent_buffer *__alloc_extent_buffer(struct extent_io_tree *tree,
2816                                                    u64 start,
2817                                                    unsigned long len,
2818                                                    gfp_t mask)
2819 {
2820         struct extent_buffer *eb = NULL;
2821 #ifdef LEAK_DEBUG
2822         unsigned long flags;
2823 #endif
2824
2825         eb = kmem_cache_zalloc(extent_buffer_cache, mask);
2826         eb->start = start;
2827         eb->len = len;
2828         mutex_init(&eb->mutex);
2829 #ifdef LEAK_DEBUG
2830         spin_lock_irqsave(&leak_lock, flags);
2831         list_add(&eb->leak_list, &buffers);
2832         spin_unlock_irqrestore(&leak_lock, flags);
2833 #endif
2834         atomic_set(&eb->refs, 1);
2835
2836         return eb;
2837 }
2838
2839 static void __free_extent_buffer(struct extent_buffer *eb)
2840 {
2841 #ifdef LEAK_DEBUG
2842         unsigned long flags;
2843         spin_lock_irqsave(&leak_lock, flags);
2844         list_del(&eb->leak_list);
2845         spin_unlock_irqrestore(&leak_lock, flags);
2846 #endif
2847         kmem_cache_free(extent_buffer_cache, eb);
2848 }
2849
2850 struct extent_buffer *alloc_extent_buffer(struct extent_io_tree *tree,
2851                                           u64 start, unsigned long len,
2852                                           struct page *page0,
2853                                           gfp_t mask)
2854 {
2855         unsigned long num_pages = num_extent_pages(start, len);
2856         unsigned long i;
2857         unsigned long index = start >> PAGE_CACHE_SHIFT;
2858         struct extent_buffer *eb;
2859         struct extent_buffer *exists = NULL;
2860         struct page *p;
2861         struct address_space *mapping = tree->mapping;
2862         int uptodate = 1;
2863
2864         spin_lock(&tree->buffer_lock);
2865         eb = buffer_search(tree, start);
2866         if (eb) {
2867                 atomic_inc(&eb->refs);
2868                 spin_unlock(&tree->buffer_lock);
2869                 mark_page_accessed(eb->first_page);
2870                 return eb;
2871         }
2872         spin_unlock(&tree->buffer_lock);
2873
2874         eb = __alloc_extent_buffer(tree, start, len, mask);
2875         if (!eb)
2876                 return NULL;
2877
2878         if (page0) {
2879                 eb->first_page = page0;
2880                 i = 1;
2881                 index++;
2882                 page_cache_get(page0);
2883                 mark_page_accessed(page0);
2884                 set_page_extent_mapped(page0);
2885                 set_page_extent_head(page0, len);
2886                 uptodate = PageUptodate(page0);
2887         } else {
2888                 i = 0;
2889         }
2890         for (; i < num_pages; i++, index++) {
2891                 p = find_or_create_page(mapping, index, mask | __GFP_HIGHMEM);
2892                 if (!p) {
2893                         WARN_ON(1);
2894                         goto free_eb;
2895                 }
2896                 set_page_extent_mapped(p);
2897                 mark_page_accessed(p);
2898                 if (i == 0) {
2899                         eb->first_page = p;
2900                         set_page_extent_head(p, len);
2901                 } else {
2902                         set_page_private(p, EXTENT_PAGE_PRIVATE);
2903                 }
2904                 if (!PageUptodate(p))
2905                         uptodate = 0;
2906                 unlock_page(p);
2907         }
2908         if (uptodate)
2909                 eb->flags |= EXTENT_UPTODATE;
2910         eb->flags |= EXTENT_BUFFER_FILLED;
2911
2912         spin_lock(&tree->buffer_lock);
2913         exists = buffer_tree_insert(tree, start, &eb->rb_node);
2914         if (exists) {
2915                 /* add one reference for the caller */
2916                 atomic_inc(&exists->refs);
2917                 spin_unlock(&tree->buffer_lock);
2918                 goto free_eb;
2919         }
2920         spin_unlock(&tree->buffer_lock);
2921
2922         /* add one reference for the tree */
2923         atomic_inc(&eb->refs);
2924         return eb;
2925
2926 free_eb:
2927         if (!atomic_dec_and_test(&eb->refs))
2928                 return exists;
2929         for (index = 1; index < i; index++)
2930                 page_cache_release(extent_buffer_page(eb, index));
2931         page_cache_release(extent_buffer_page(eb, 0));
2932         __free_extent_buffer(eb);
2933         return exists;
2934 }
2935 EXPORT_SYMBOL(alloc_extent_buffer);
2936
2937 struct extent_buffer *find_extent_buffer(struct extent_io_tree *tree,
2938                                          u64 start, unsigned long len,
2939                                           gfp_t mask)
2940 {
2941         struct extent_buffer *eb;
2942
2943         spin_lock(&tree->buffer_lock);
2944         eb = buffer_search(tree, start);
2945         if (eb)
2946                 atomic_inc(&eb->refs);
2947         spin_unlock(&tree->buffer_lock);
2948
2949         if (eb)
2950                 mark_page_accessed(eb->first_page);
2951
2952         return eb;
2953 }
2954 EXPORT_SYMBOL(find_extent_buffer);
2955
2956 void free_extent_buffer(struct extent_buffer *eb)
2957 {
2958         if (!eb)
2959                 return;
2960
2961         if (!atomic_dec_and_test(&eb->refs))
2962                 return;
2963
2964         WARN_ON(1);
2965 }
2966 EXPORT_SYMBOL(free_extent_buffer);
2967
2968 int clear_extent_buffer_dirty(struct extent_io_tree *tree,
2969                               struct extent_buffer *eb)
2970 {
2971         int set;
2972         unsigned long i;
2973         unsigned long num_pages;
2974         struct page *page;
2975
2976         u64 start = eb->start;
2977         u64 end = start + eb->len - 1;
2978
2979         set = clear_extent_dirty(tree, start, end, GFP_NOFS);
2980         num_pages = num_extent_pages(eb->start, eb->len);
2981
2982         for (i = 0; i < num_pages; i++) {
2983                 page = extent_buffer_page(eb, i);
2984                 lock_page(page);
2985                 if (i == 0)
2986                         set_page_extent_head(page, eb->len);
2987                 else
2988                         set_page_private(page, EXTENT_PAGE_PRIVATE);
2989
2990                 /*
2991                  * if we're on the last page or the first page and the
2992                  * block isn't aligned on a page boundary, do extra checks
2993                  * to make sure we don't clean page that is partially dirty
2994                  */
2995                 if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
2996                     ((i == num_pages - 1) &&
2997                      ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
2998                         start = (u64)page->index << PAGE_CACHE_SHIFT;
2999                         end  = start + PAGE_CACHE_SIZE - 1;
3000                         if (test_range_bit(tree, start, end,
3001                                            EXTENT_DIRTY, 0)) {
3002                                 unlock_page(page);
3003                                 continue;
3004                         }
3005                 }
3006                 clear_page_dirty_for_io(page);
3007                 spin_lock_irq(&page->mapping->tree_lock);
3008                 if (!PageDirty(page)) {
3009                         radix_tree_tag_clear(&page->mapping->page_tree,
3010                                                 page_index(page),
3011                                                 PAGECACHE_TAG_DIRTY);
3012                 }
3013                 spin_unlock_irq(&page->mapping->tree_lock);
3014                 unlock_page(page);
3015         }
3016         return 0;
3017 }
3018 EXPORT_SYMBOL(clear_extent_buffer_dirty);
3019
3020 int wait_on_extent_buffer_writeback(struct extent_io_tree *tree,
3021                                     struct extent_buffer *eb)
3022 {
3023         return wait_on_extent_writeback(tree, eb->start,
3024                                         eb->start + eb->len - 1);
3025 }
3026 EXPORT_SYMBOL(wait_on_extent_buffer_writeback);
3027
3028 int set_extent_buffer_dirty(struct extent_io_tree *tree,
3029                              struct extent_buffer *eb)
3030 {
3031         unsigned long i;
3032         unsigned long num_pages;
3033
3034         num_pages = num_extent_pages(eb->start, eb->len);
3035         for (i = 0; i < num_pages; i++) {
3036                 struct page *page = extent_buffer_page(eb, i);
3037                 /* writepage may need to do something special for the
3038                  * first page, we have to make sure page->private is
3039                  * properly set.  releasepage may drop page->private
3040                  * on us if the page isn't already dirty.
3041                  */
3042                 lock_page(page);
3043                 if (i == 0) {
3044                         set_page_extent_head(page, eb->len);
3045                 } else if (PagePrivate(page) &&
3046                            page->private != EXTENT_PAGE_PRIVATE) {
3047                         set_page_extent_mapped(page);
3048                 }
3049                 __set_page_dirty_nobuffers(extent_buffer_page(eb, i));
3050                 set_extent_dirty(tree, page_offset(page),
3051                                  page_offset(page) + PAGE_CACHE_SIZE -1,
3052                                  GFP_NOFS);
3053                 unlock_page(page);
3054         }
3055         return 0;
3056 }
3057 EXPORT_SYMBOL(set_extent_buffer_dirty);
3058
3059 int clear_extent_buffer_uptodate(struct extent_io_tree *tree,
3060                                 struct extent_buffer *eb)
3061 {
3062         unsigned long i;
3063         struct page *page;
3064         unsigned long num_pages;
3065
3066         num_pages = num_extent_pages(eb->start, eb->len);
3067         eb->flags &= ~EXTENT_UPTODATE;
3068
3069         clear_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,
3070                               GFP_NOFS);
3071         for (i = 0; i < num_pages; i++) {
3072                 page = extent_buffer_page(eb, i);
3073                 if (page)
3074                         ClearPageUptodate(page);
3075         }
3076         return 0;
3077 }
3078
3079 int set_extent_buffer_uptodate(struct extent_io_tree *tree,
3080                                 struct extent_buffer *eb)
3081 {
3082         unsigned long i;
3083         struct page *page;
3084         unsigned long num_pages;
3085
3086         num_pages = num_extent_pages(eb->start, eb->len);
3087
3088         set_extent_uptodate(tree, eb->start, eb->start + eb->len - 1,
3089                             GFP_NOFS);
3090         for (i = 0; i < num_pages; i++) {
3091                 page = extent_buffer_page(eb, i);
3092                 if ((i == 0 && (eb->start & (PAGE_CACHE_SIZE - 1))) ||
3093                     ((i == num_pages - 1) &&
3094                      ((eb->start + eb->len) & (PAGE_CACHE_SIZE - 1)))) {
3095                         check_page_uptodate(tree, page);
3096                         continue;
3097                 }
3098                 SetPageUptodate(page);
3099         }
3100         return 0;
3101 }
3102 EXPORT_SYMBOL(set_extent_buffer_uptodate);
3103
3104 int extent_range_uptodate(struct extent_io_tree *tree,
3105                           u64 start, u64 end)
3106 {
3107         struct page *page;
3108         int ret;
3109         int pg_uptodate = 1;
3110         int uptodate;
3111         unsigned long index;
3112
3113         ret = test_range_bit(tree, start, end, EXTENT_UPTODATE, 1);
3114         if (ret)
3115                 return 1;
3116         while(start <= end) {
3117                 index = start >> PAGE_CACHE_SHIFT;
3118                 page = find_get_page(tree->mapping, index);
3119                 uptodate = PageUptodate(page);
3120                 page_cache_release(page);
3121                 if (!uptodate) {
3122                         pg_uptodate = 0;
3123                         break;
3124                 }
3125                 start += PAGE_CACHE_SIZE;
3126         }
3127         return pg_uptodate;
3128 }
3129
3130 int extent_buffer_uptodate(struct extent_io_tree *tree,
3131                            struct extent_buffer *eb)
3132 {
3133         int ret = 0;
3134         unsigned long num_pages;
3135         unsigned long i;
3136         struct page *page;
3137         int pg_uptodate = 1;
3138
3139         if (eb->flags & EXTENT_UPTODATE)
3140                 return 1;
3141
3142         ret = test_range_bit(tree, eb->start, eb->start + eb->len - 1,
3143                            EXTENT_UPTODATE, 1);
3144         if (ret)
3145                 return ret;
3146
3147         num_pages = num_extent_pages(eb->start, eb->len);
3148         for (i = 0; i < num_pages; i++) {
3149                 page = extent_buffer_page(eb, i);
3150                 if (!PageUptodate(page)) {
3151                         pg_uptodate = 0;
3152                         break;
3153                 }
3154         }
3155         return pg_uptodate;
3156 }
3157 EXPORT_SYMBOL(extent_buffer_uptodate);
3158
3159 int read_extent_buffer_pages(struct extent_io_tree *tree,
3160                              struct extent_buffer *eb,
3161                              u64 start, int wait,
3162                              get_extent_t *get_extent, int mirror_num)
3163 {
3164         unsigned long i;
3165         unsigned long start_i;
3166         struct page *page;
3167         int err;
3168         int ret = 0;
3169         int locked_pages = 0;
3170         int all_uptodate = 1;
3171         int inc_all_pages = 0;
3172         unsigned long num_pages;
3173         struct bio *bio = NULL;
3174         unsigned long bio_flags = 0;
3175
3176         if (eb->flags & EXTENT_UPTODATE)
3177                 return 0;
3178
3179         if (test_range_bit(tree, eb->start, eb->start + eb->len - 1,
3180                            EXTENT_UPTODATE, 1)) {
3181                 return 0;
3182         }
3183
3184         if (start) {
3185                 WARN_ON(start < eb->start);
3186                 start_i = (start >> PAGE_CACHE_SHIFT) -
3187                         (eb->start >> PAGE_CACHE_SHIFT);
3188         } else {
3189                 start_i = 0;
3190         }
3191
3192         num_pages = num_extent_pages(eb->start, eb->len);
3193         for (i = start_i; i < num_pages; i++) {
3194                 page = extent_buffer_page(eb, i);
3195                 if (!wait) {
3196                         if (!trylock_page(page))
3197                                 goto unlock_exit;
3198                 } else {
3199                         lock_page(page);
3200                 }
3201                 locked_pages++;
3202                 if (!PageUptodate(page)) {
3203                         all_uptodate = 0;
3204                 }
3205         }
3206         if (all_uptodate) {
3207                 if (start_i == 0)
3208                         eb->flags |= EXTENT_UPTODATE;
3209                 if (ret) {
3210                         printk("all up to date but ret is %d\n", ret);
3211                 }
3212                 goto unlock_exit;
3213         }
3214
3215         for (i = start_i; i < num_pages; i++) {
3216                 page = extent_buffer_page(eb, i);
3217                 if (inc_all_pages)
3218                         page_cache_get(page);
3219                 if (!PageUptodate(page)) {
3220                         if (start_i == 0)
3221                                 inc_all_pages = 1;
3222                         ClearPageError(page);
3223                         err = __extent_read_full_page(tree, page,
3224                                                       get_extent, &bio,
3225                                                       mirror_num, &bio_flags);
3226                         if (err) {
3227                                 ret = err;
3228                                 printk("err %d from __extent_read_full_page\n", ret);
3229                         }
3230                 } else {
3231                         unlock_page(page);
3232                 }
3233         }
3234
3235         if (bio)
3236                 submit_one_bio(READ, bio, mirror_num, bio_flags);
3237
3238         if (ret || !wait) {
3239                 if (ret)
3240                         printk("ret %d wait %d returning\n", ret, wait);
3241                 return ret;
3242         }
3243         for (i = start_i; i < num_pages; i++) {
3244                 page = extent_buffer_page(eb, i);
3245                 wait_on_page_locked(page);
3246                 if (!PageUptodate(page)) {
3247                         printk("page not uptodate after wait_on_page_locked\n");
3248                         ret = -EIO;
3249                 }
3250         }
3251         if (!ret)
3252                 eb->flags |= EXTENT_UPTODATE;
3253         return ret;
3254
3255 unlock_exit:
3256         i = start_i;
3257         while(locked_pages > 0) {
3258                 page = extent_buffer_page(eb, i);
3259                 i++;
3260                 unlock_page(page);
3261                 locked_pages--;
3262         }
3263         return ret;
3264 }
3265 EXPORT_SYMBOL(read_extent_buffer_pages);
3266
3267 void read_extent_buffer(struct extent_buffer *eb, void *dstv,
3268                         unsigned long start,
3269                         unsigned long len)
3270 {
3271         size_t cur;
3272         size_t offset;
3273         struct page *page;
3274         char *kaddr;
3275         char *dst = (char *)dstv;
3276         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3277         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3278
3279         WARN_ON(start > eb->len);
3280         WARN_ON(start + len > eb->start + eb->len);
3281
3282         offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3283
3284         while(len > 0) {
3285                 page = extent_buffer_page(eb, i);
3286
3287                 cur = min(len, (PAGE_CACHE_SIZE - offset));
3288                 kaddr = kmap_atomic(page, KM_USER1);
3289                 memcpy(dst, kaddr + offset, cur);
3290                 kunmap_atomic(kaddr, KM_USER1);
3291
3292                 dst += cur;
3293                 len -= cur;
3294                 offset = 0;
3295                 i++;
3296         }
3297 }
3298 EXPORT_SYMBOL(read_extent_buffer);
3299
3300 int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
3301                                unsigned long min_len, char **token, char **map,
3302                                unsigned long *map_start,
3303                                unsigned long *map_len, int km)
3304 {
3305         size_t offset = start & (PAGE_CACHE_SIZE - 1);
3306         char *kaddr;
3307         struct page *p;
3308         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3309         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3310         unsigned long end_i = (start_offset + start + min_len - 1) >>
3311                 PAGE_CACHE_SHIFT;
3312
3313         if (i != end_i)
3314                 return -EINVAL;
3315
3316         if (i == 0) {
3317                 offset = start_offset;
3318                 *map_start = 0;
3319         } else {
3320                 offset = 0;
3321                 *map_start = ((u64)i << PAGE_CACHE_SHIFT) - start_offset;
3322         }
3323         if (start + min_len > eb->len) {
3324 printk("bad mapping eb start %Lu len %lu, wanted %lu %lu\n", eb->start, eb->len, start, min_len);
3325                 WARN_ON(1);
3326         }
3327
3328         p = extent_buffer_page(eb, i);
3329         kaddr = kmap_atomic(p, km);
3330         *token = kaddr;
3331         *map = kaddr + offset;
3332         *map_len = PAGE_CACHE_SIZE - offset;
3333         return 0;
3334 }
3335 EXPORT_SYMBOL(map_private_extent_buffer);
3336
3337 int map_extent_buffer(struct extent_buffer *eb, unsigned long start,
3338                       unsigned long min_len,
3339                       char **token, char **map,
3340                       unsigned long *map_start,
3341                       unsigned long *map_len, int km)
3342 {
3343         int err;
3344         int save = 0;
3345         if (eb->map_token) {
3346                 unmap_extent_buffer(eb, eb->map_token, km);
3347                 eb->map_token = NULL;
3348                 save = 1;
3349         }
3350         err = map_private_extent_buffer(eb, start, min_len, token, map,
3351                                        map_start, map_len, km);
3352         if (!err && save) {
3353                 eb->map_token = *token;
3354                 eb->kaddr = *map;
3355                 eb->map_start = *map_start;
3356                 eb->map_len = *map_len;
3357         }
3358         return err;
3359 }
3360 EXPORT_SYMBOL(map_extent_buffer);
3361
3362 void unmap_extent_buffer(struct extent_buffer *eb, char *token, int km)
3363 {
3364         kunmap_atomic(token, km);
3365 }
3366 EXPORT_SYMBOL(unmap_extent_buffer);
3367
3368 int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
3369                           unsigned long start,
3370                           unsigned long len)
3371 {
3372         size_t cur;
3373         size_t offset;
3374         struct page *page;
3375         char *kaddr;
3376         char *ptr = (char *)ptrv;
3377         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3378         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3379         int ret = 0;
3380
3381         WARN_ON(start > eb->len);
3382         WARN_ON(start + len > eb->start + eb->len);
3383
3384         offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3385
3386         while(len > 0) {
3387                 page = extent_buffer_page(eb, i);
3388
3389                 cur = min(len, (PAGE_CACHE_SIZE - offset));
3390
3391                 kaddr = kmap_atomic(page, KM_USER0);
3392                 ret = memcmp(ptr, kaddr + offset, cur);
3393                 kunmap_atomic(kaddr, KM_USER0);
3394                 if (ret)
3395                         break;
3396
3397                 ptr += cur;
3398                 len -= cur;
3399                 offset = 0;
3400                 i++;
3401         }
3402         return ret;
3403 }
3404 EXPORT_SYMBOL(memcmp_extent_buffer);
3405
3406 void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
3407                          unsigned long start, unsigned long len)
3408 {
3409         size_t cur;
3410         size_t offset;
3411         struct page *page;
3412         char *kaddr;
3413         char *src = (char *)srcv;
3414         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3415         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3416
3417         WARN_ON(start > eb->len);
3418         WARN_ON(start + len > eb->start + eb->len);
3419
3420         offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3421
3422         while(len > 0) {
3423                 page = extent_buffer_page(eb, i);
3424                 WARN_ON(!PageUptodate(page));
3425
3426                 cur = min(len, PAGE_CACHE_SIZE - offset);
3427                 kaddr = kmap_atomic(page, KM_USER1);
3428                 memcpy(kaddr + offset, src, cur);
3429                 kunmap_atomic(kaddr, KM_USER1);
3430
3431                 src += cur;
3432                 len -= cur;
3433                 offset = 0;
3434                 i++;
3435         }
3436 }
3437 EXPORT_SYMBOL(write_extent_buffer);
3438
3439 void memset_extent_buffer(struct extent_buffer *eb, char c,
3440                           unsigned long start, unsigned long len)
3441 {
3442         size_t cur;
3443         size_t offset;
3444         struct page *page;
3445         char *kaddr;
3446         size_t start_offset = eb->start & ((u64)PAGE_CACHE_SIZE - 1);
3447         unsigned long i = (start_offset + start) >> PAGE_CACHE_SHIFT;
3448
3449         WARN_ON(start > eb->len);
3450         WARN_ON(start + len > eb->start + eb->len);
3451
3452         offset = (start_offset + start) & ((unsigned long)PAGE_CACHE_SIZE - 1);
3453
3454         while(len > 0) {
3455                 page = extent_buffer_page(eb, i);
3456                 WARN_ON(!PageUptodate(page));
3457
3458                 cur = min(len, PAGE_CACHE_SIZE - offset);
3459                 kaddr = kmap_atomic(page, KM_USER0);
3460                 memset(kaddr + offset, c, cur);
3461                 kunmap_atomic(kaddr, KM_USER0);
3462
3463                 len -= cur;
3464                 offset = 0;
3465                 i++;
3466         }
3467 }
3468 EXPORT_SYMBOL(memset_extent_buffer);
3469
3470 void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
3471                         unsigned long dst_offset, unsigned long src_offset,
3472                         unsigned long len)
3473 {
3474         u64 dst_len = dst->len;
3475         size_t cur;
3476         size_t offset;
3477         struct page *page;
3478         char *kaddr;
3479         size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
3480         unsigned long i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
3481
3482         WARN_ON(src->len != dst_len);
3483
3484         offset = (start_offset + dst_offset) &
3485                 ((unsigned long)PAGE_CACHE_SIZE - 1);
3486
3487         while(len > 0) {
3488                 page = extent_buffer_page(dst, i);
3489                 WARN_ON(!PageUptodate(page));
3490
3491                 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE - offset));
3492
3493                 kaddr = kmap_atomic(page, KM_USER0);
3494                 read_extent_buffer(src, kaddr + offset, src_offset, cur);
3495                 kunmap_atomic(kaddr, KM_USER0);
3496
3497                 src_offset += cur;
3498                 len -= cur;
3499                 offset = 0;
3500                 i++;
3501         }
3502 }
3503 EXPORT_SYMBOL(copy_extent_buffer);
3504
3505 static void move_pages(struct page *dst_page, struct page *src_page,
3506                        unsigned long dst_off, unsigned long src_off,
3507                        unsigned long len)
3508 {
3509         char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
3510         if (dst_page == src_page) {
3511                 memmove(dst_kaddr + dst_off, dst_kaddr + src_off, len);
3512         } else {
3513                 char *src_kaddr = kmap_atomic(src_page, KM_USER1);
3514                 char *p = dst_kaddr + dst_off + len;
3515                 char *s = src_kaddr + src_off + len;
3516
3517                 while (len--)
3518                         *--p = *--s;
3519
3520                 kunmap_atomic(src_kaddr, KM_USER1);
3521         }
3522         kunmap_atomic(dst_kaddr, KM_USER0);
3523 }
3524
3525 static void copy_pages(struct page *dst_page, struct page *src_page,
3526                        unsigned long dst_off, unsigned long src_off,
3527                        unsigned long len)
3528 {
3529         char *dst_kaddr = kmap_atomic(dst_page, KM_USER0);
3530         char *src_kaddr;
3531
3532         if (dst_page != src_page)
3533                 src_kaddr = kmap_atomic(src_page, KM_USER1);
3534         else
3535                 src_kaddr = dst_kaddr;
3536
3537         memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
3538         kunmap_atomic(dst_kaddr, KM_USER0);
3539         if (dst_page != src_page)
3540                 kunmap_atomic(src_kaddr, KM_USER1);
3541 }
3542
3543 void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
3544                            unsigned long src_offset, unsigned long len)
3545 {
3546         size_t cur;
3547         size_t dst_off_in_page;
3548         size_t src_off_in_page;
3549         size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
3550         unsigned long dst_i;
3551         unsigned long src_i;
3552
3553         if (src_offset + len > dst->len) {
3554                 printk("memmove bogus src_offset %lu move len %lu len %lu\n",
3555                        src_offset, len, dst->len);
3556                 BUG_ON(1);
3557         }
3558         if (dst_offset + len > dst->len) {
3559                 printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
3560                        dst_offset, len, dst->len);
3561                 BUG_ON(1);
3562         }
3563
3564         while(len > 0) {
3565                 dst_off_in_page = (start_offset + dst_offset) &
3566                         ((unsigned long)PAGE_CACHE_SIZE - 1);
3567                 src_off_in_page = (start_offset + src_offset) &
3568                         ((unsigned long)PAGE_CACHE_SIZE - 1);
3569
3570                 dst_i = (start_offset + dst_offset) >> PAGE_CACHE_SHIFT;
3571                 src_i = (start_offset + src_offset) >> PAGE_CACHE_SHIFT;
3572
3573                 cur = min(len, (unsigned long)(PAGE_CACHE_SIZE -
3574                                                src_off_in_page));
3575                 cur = min_t(unsigned long, cur,
3576                         (unsigned long)(PAGE_CACHE_SIZE - dst_off_in_page));
3577
3578                 copy_pages(extent_buffer_page(dst, dst_i),
3579                            extent_buffer_page(dst, src_i),
3580                            dst_off_in_page, src_off_in_page, cur);
3581
3582                 src_offset += cur;
3583                 dst_offset += cur;
3584                 len -= cur;
3585         }
3586 }
3587 EXPORT_SYMBOL(memcpy_extent_buffer);
3588
3589 void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
3590                            unsigned long src_offset, unsigned long len)
3591 {
3592         size_t cur;
3593         size_t dst_off_in_page;
3594         size_t src_off_in_page;
3595         unsigned long dst_end = dst_offset + len - 1;
3596         unsigned long src_end = src_offset + len - 1;
3597         size_t start_offset = dst->start & ((u64)PAGE_CACHE_SIZE - 1);
3598         unsigned long dst_i;
3599         unsigned long src_i;
3600
3601         if (src_offset + len > dst->len) {
3602                 printk("memmove bogus src_offset %lu move len %lu len %lu\n",
3603                        src_offset, len, dst->len);
3604                 BUG_ON(1);
3605         }
3606         if (dst_offset + len > dst->len) {
3607                 printk("memmove bogus dst_offset %lu move len %lu len %lu\n",
3608                        dst_offset, len, dst->len);
3609                 BUG_ON(1);
3610         }
3611         if (dst_offset < src_offset) {
3612                 memcpy_extent_buffer(dst, dst_offset, src_offset, len);
3613                 return;
3614         }
3615         while(len > 0) {
3616                 dst_i = (start_offset + dst_end) >> PAGE_CACHE_SHIFT;
3617                 src_i = (start_offset + src_end) >> PAGE_CACHE_SHIFT;
3618
3619                 dst_off_in_page = (start_offset + dst_end) &
3620                         ((unsigned long)PAGE_CACHE_SIZE - 1);
3621                 src_off_in_page = (start_offset + src_end) &
3622                         ((unsigned long)PAGE_CACHE_SIZE - 1);
3623
3624                 cur = min_t(unsigned long, len, src_off_in_page + 1);
3625                 cur = min(cur, dst_off_in_page + 1);
3626                 move_pages(extent_buffer_page(dst, dst_i),
3627                            extent_buffer_page(dst, src_i),
3628                            dst_off_in_page - cur + 1,
3629                            src_off_in_page - cur + 1, cur);
3630
3631                 dst_end -= cur;
3632                 src_end -= cur;
3633                 len -= cur;
3634         }
3635 }
3636 EXPORT_SYMBOL(memmove_extent_buffer);
3637
3638 int try_release_extent_buffer(struct extent_io_tree *tree, struct page *page)
3639 {
3640         u64 start = page_offset(page);
3641         struct extent_buffer *eb;
3642         int ret = 1;
3643         unsigned long i;
3644         unsigned long num_pages;
3645
3646         spin_lock(&tree->buffer_lock);
3647         eb = buffer_search(tree, start);
3648         if (!eb)
3649                 goto out;
3650
3651         if (atomic_read(&eb->refs) > 1) {
3652                 ret = 0;
3653                 goto out;
3654         }
3655         /* at this point we can safely release the extent buffer */
3656         num_pages = num_extent_pages(eb->start, eb->len);
3657         for (i = 0; i < num_pages; i++)
3658                 page_cache_release(extent_buffer_page(eb, i));
3659         rb_erase(&eb->rb_node, &tree->buffer);
3660         __free_extent_buffer(eb);
3661 out:
3662         spin_unlock(&tree->buffer_lock);
3663         return ret;
3664 }
3665 EXPORT_SYMBOL(try_release_extent_buffer);