1#include <linux/bitops.h>
   2#include <linux/slab.h>
   3#include <linux/bio.h>
   4#include <linux/mm.h>
   5#include <linux/pagemap.h>
   6#include <linux/page-flags.h>
 
   7#include <linux/spinlock.h>
   8#include <linux/blkdev.h>
   9#include <linux/swap.h>
  10#include <linux/writeback.h>
  11#include <linux/pagevec.h>
  12#include <linux/prefetch.h>
  13#include <linux/cleancache.h>
  14#include "extent_io.h"
 
  15#include "extent_map.h"
  16#include "ctree.h"
  17#include "btrfs_inode.h"
  18#include "volumes.h"
  19#include "check-integrity.h"
  20#include "locking.h"
  21#include "rcu-string.h"
  22#include "backref.h"
 
 
 
 
 
 
 
 
 
 
 
 
  23
  24static struct kmem_cache *extent_state_cache;
  25static struct kmem_cache *extent_buffer_cache;
  26static struct bio_set *btrfs_bioset;
  27
  28static inline bool extent_state_in_tree(const struct extent_state *state)
  29{
  30	return !RB_EMPTY_NODE(&state->rb_node);
  31}
  32
  33#ifdef CONFIG_BTRFS_DEBUG
  34static LIST_HEAD(buffers);
  35static LIST_HEAD(states);
  36
  37static DEFINE_SPINLOCK(leak_lock);
  38
  39static inline
  40void btrfs_leak_debug_add(struct list_head *new, struct list_head *head)
  41{
 
  42	unsigned long flags;
  43
  44	spin_lock_irqsave(&leak_lock, flags);
  45	list_add(new, head);
  46	spin_unlock_irqrestore(&leak_lock, flags);
  47}
  48
  49static inline
  50void btrfs_leak_debug_del(struct list_head *entry)
  51{
 
  52	unsigned long flags;
  53
  54	spin_lock_irqsave(&leak_lock, flags);
  55	list_del(entry);
  56	spin_unlock_irqrestore(&leak_lock, flags);
  57}
  58
  59static inline
  60void btrfs_leak_debug_check(void)
  61{
  62	struct extent_state *state;
  63	struct extent_buffer *eb;
 
  64
  65	while (!list_empty(&states)) {
  66		state = list_entry(states.next, struct extent_state, leak_list);
  67		pr_err("BTRFS: state leak: start %llu end %llu state %u in tree %d refs %d\n",
  68		       state->start, state->end, state->state,
  69		       extent_state_in_tree(state),
  70		       atomic_read(&state->refs));
  71		list_del(&state->leak_list);
  72		kmem_cache_free(extent_state_cache, state);
  73	}
  74
  75	while (!list_empty(&buffers)) {
  76		eb = list_entry(buffers.next, struct extent_buffer, leak_list);
  77		printk(KERN_ERR "BTRFS: buffer leak start %llu len %lu "
  78		       "refs %d\n",
  79		       eb->start, eb->len, atomic_read(&eb->refs));
 
 
 
 
  80		list_del(&eb->leak_list);
 
  81		kmem_cache_free(extent_buffer_cache, eb);
  82	}
  83}
  84
  85#define btrfs_debug_check_extent_io_range(tree, start, end)		\
  86	__btrfs_debug_check_extent_io_range(__func__, (tree), (start), (end))
  87static inline void __btrfs_debug_check_extent_io_range(const char *caller,
  88		struct extent_io_tree *tree, u64 start, u64 end)
  89{
  90	struct inode *inode;
  91	u64 isize;
  92
  93	if (!tree->mapping)
  94		return;
  95
  96	inode = tree->mapping->host;
  97	isize = i_size_read(inode);
  98	if (end >= PAGE_SIZE && (end % 2) == 0 && end != isize - 1) {
  99		btrfs_debug_rl(BTRFS_I(inode)->root->fs_info,
 100		    "%s: ino %llu isize %llu odd range [%llu,%llu]",
 101				caller, btrfs_ino(inode), isize, start, end);
 102	}
 103}
 104#else
 105#define btrfs_leak_debug_add(new, head)	do {} while (0)
 106#define btrfs_leak_debug_del(entry)	do {} while (0)
 107#define btrfs_leak_debug_check()	do {} while (0)
 108#define btrfs_debug_check_extent_io_range(c, s, e)	do {} while (0)
 109#endif
 110
 111#define BUFFER_LRU_MAX 64
 112
 113struct tree_entry {
 114	u64 start;
 115	u64 end;
 116	struct rb_node rb_node;
 117};
 118
 119struct extent_page_data {
 120	struct bio *bio;
 121	struct extent_io_tree *tree;
 122	get_extent_t *get_extent;
 123	unsigned long bio_flags;
 124
 125	/* tells writepage not to lock the state bits for this range
 126	 * it still does the unlocking
 127	 */
 128	unsigned int extent_locked:1;
 129
 130	/* tells the submit_bio code to use a WRITE_SYNC */
 131	unsigned int sync_io:1;
 132};
 133
 134static void add_extent_changeset(struct extent_state *state, unsigned bits,
 135				 struct extent_changeset *changeset,
 136				 int set)
 137{
 138	int ret;
 139
 140	if (!changeset)
 141		return;
 142	if (set && (state->state & bits) == bits)
 143		return;
 144	if (!set && (state->state & bits) == 0)
 145		return;
 146	changeset->bytes_changed += state->end - state->start + 1;
 147	ret = ulist_add(changeset->range_changed, state->start, state->end,
 148			GFP_ATOMIC);
 149	/* ENOMEM */
 150	BUG_ON(ret < 0);
 151}
 152
 153static noinline void flush_write_bio(void *data);
 154static inline struct btrfs_fs_info *
 155tree_fs_info(struct extent_io_tree *tree)
 156{
 157	if (!tree->mapping)
 158		return NULL;
 159	return btrfs_sb(tree->mapping->host->i_sb);
 160}
 161
 162int __init extent_io_init(void)
 163{
 164	extent_state_cache = kmem_cache_create("btrfs_extent_state",
 165			sizeof(struct extent_state), 0,
 166			SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
 167	if (!extent_state_cache)
 168		return -ENOMEM;
 169
 170	extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
 171			sizeof(struct extent_buffer), 0,
 172			SLAB_RECLAIM_ACCOUNT | SLAB_MEM_SPREAD, NULL);
 173	if (!extent_buffer_cache)
 174		goto free_state_cache;
 175
 176	btrfs_bioset = bioset_create(BIO_POOL_SIZE,
 177				     offsetof(struct btrfs_io_bio, bio));
 178	if (!btrfs_bioset)
 179		goto free_buffer_cache;
 180
 181	if (bioset_integrity_create(btrfs_bioset, BIO_POOL_SIZE))
 182		goto free_bioset;
 183
 184	return 0;
 185
 186free_bioset:
 187	bioset_free(btrfs_bioset);
 188	btrfs_bioset = NULL;
 189
 190free_buffer_cache:
 191	kmem_cache_destroy(extent_buffer_cache);
 192	extent_buffer_cache = NULL;
 193
 194free_state_cache:
 195	kmem_cache_destroy(extent_state_cache);
 196	extent_state_cache = NULL;
 197	return -ENOMEM;
 198}
 199
 200void extent_io_exit(void)
 201{
 202	btrfs_leak_debug_check();
 203
 204	/*
 205	 * Make sure all delayed rcu free are flushed before we
 206	 * destroy caches.
 207	 */
 208	rcu_barrier();
 209	kmem_cache_destroy(extent_state_cache);
 210	kmem_cache_destroy(extent_buffer_cache);
 211	if (btrfs_bioset)
 212		bioset_free(btrfs_bioset);
 213}
 214
 215void extent_io_tree_init(struct extent_io_tree *tree,
 216			 struct address_space *mapping)
 217{
 218	tree->state = RB_ROOT;
 219	tree->ops = NULL;
 220	tree->dirty_bytes = 0;
 221	spin_lock_init(&tree->lock);
 222	tree->mapping = mapping;
 223}
 224
 225static struct extent_state *alloc_extent_state(gfp_t mask)
 226{
 227	struct extent_state *state;
 228
 229	state = kmem_cache_alloc(extent_state_cache, mask);
 230	if (!state)
 231		return state;
 232	state->state = 0;
 233	state->failrec = NULL;
 234	RB_CLEAR_NODE(&state->rb_node);
 235	btrfs_leak_debug_add(&state->leak_list, &states);
 236	atomic_set(&state->refs, 1);
 237	init_waitqueue_head(&state->wq);
 238	trace_alloc_extent_state(state, mask, _RET_IP_);
 239	return state;
 240}
 241
 242void free_extent_state(struct extent_state *state)
 243{
 244	if (!state)
 245		return;
 246	if (atomic_dec_and_test(&state->refs)) {
 247		WARN_ON(extent_state_in_tree(state));
 248		btrfs_leak_debug_del(&state->leak_list);
 249		trace_free_extent_state(state, _RET_IP_);
 250		kmem_cache_free(extent_state_cache, state);
 251	}
 252}
 253
 254static struct rb_node *tree_insert(struct rb_root *root,
 255				   struct rb_node *search_start,
 256				   u64 offset,
 257				   struct rb_node *node,
 258				   struct rb_node ***p_in,
 259				   struct rb_node **parent_in)
 260{
 261	struct rb_node **p;
 262	struct rb_node *parent = NULL;
 263	struct tree_entry *entry;
 264
 265	if (p_in && parent_in) {
 266		p = *p_in;
 267		parent = *parent_in;
 268		goto do_insert;
 269	}
 270
 271	p = search_start ? &search_start : &root->rb_node;
 272	while (*p) {
 273		parent = *p;
 274		entry = rb_entry(parent, struct tree_entry, rb_node);
 275
 276		if (offset < entry->start)
 277			p = &(*p)->rb_left;
 278		else if (offset > entry->end)
 279			p = &(*p)->rb_right;
 280		else
 281			return parent;
 282	}
 283
 284do_insert:
 285	rb_link_node(node, parent, p);
 286	rb_insert_color(node, root);
 287	return NULL;
 288}
 289
 290static struct rb_node *__etree_search(struct extent_io_tree *tree, u64 offset,
 291				      struct rb_node **prev_ret,
 292				      struct rb_node **next_ret,
 293				      struct rb_node ***p_ret,
 294				      struct rb_node **parent_ret)
 295{
 296	struct rb_root *root = &tree->state;
 297	struct rb_node **n = &root->rb_node;
 298	struct rb_node *prev = NULL;
 299	struct rb_node *orig_prev = NULL;
 300	struct tree_entry *entry;
 301	struct tree_entry *prev_entry = NULL;
 302
 303	while (*n) {
 304		prev = *n;
 305		entry = rb_entry(prev, struct tree_entry, rb_node);
 306		prev_entry = entry;
 307
 308		if (offset < entry->start)
 309			n = &(*n)->rb_left;
 310		else if (offset > entry->end)
 311			n = &(*n)->rb_right;
 312		else
 313			return *n;
 314	}
 315
 316	if (p_ret)
 317		*p_ret = n;
 318	if (parent_ret)
 319		*parent_ret = prev;
 320
 321	if (prev_ret) {
 322		orig_prev = prev;
 323		while (prev && offset > prev_entry->end) {
 324			prev = rb_next(prev);
 325			prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 326		}
 327		*prev_ret = prev;
 328		prev = orig_prev;
 329	}
 330
 331	if (next_ret) {
 332		prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 333		while (prev && offset < prev_entry->start) {
 334			prev = rb_prev(prev);
 335			prev_entry = rb_entry(prev, struct tree_entry, rb_node);
 336		}
 337		*next_ret = prev;
 338	}
 339	return NULL;
 340}
 341
 342static inline struct rb_node *
 343tree_search_for_insert(struct extent_io_tree *tree,
 344		       u64 offset,
 345		       struct rb_node ***p_ret,
 346		       struct rb_node **parent_ret)
 347{
 348	struct rb_node *prev = NULL;
 349	struct rb_node *ret;
 350
 351	ret = __etree_search(tree, offset, &prev, NULL, p_ret, parent_ret);
 352	if (!ret)
 353		return prev;
 354	return ret;
 355}
 356
 357static inline struct rb_node *tree_search(struct extent_io_tree *tree,
 358					  u64 offset)
 359{
 360	return tree_search_for_insert(tree, offset, NULL, NULL);
 361}
 362
 363static void merge_cb(struct extent_io_tree *tree, struct extent_state *new,
 364		     struct extent_state *other)
 365{
 366	if (tree->ops && tree->ops->merge_extent_hook)
 367		tree->ops->merge_extent_hook(tree->mapping->host, new,
 368					     other);
 369}
 370
 371/*
 372 * utility function to look for merge candidates inside a given range.
 373 * Any extents with matching state are merged together into a single
 374 * extent in the tree.  Extents with EXTENT_IO in their state field
 375 * are not merged because the end_io handlers need to be able to do
 376 * operations on them without sleeping (or doing allocations/splits).
 377 *
 378 * This should be called with the tree lock held.
 379 */
 380static void merge_state(struct extent_io_tree *tree,
 381		        struct extent_state *state)
 382{
 383	struct extent_state *other;
 384	struct rb_node *other_node;
 385
 386	if (state->state & (EXTENT_IOBITS | EXTENT_BOUNDARY))
 387		return;
 388
 389	other_node = rb_prev(&state->rb_node);
 390	if (other_node) {
 391		other = rb_entry(other_node, struct extent_state, rb_node);
 392		if (other->end == state->start - 1 &&
 393		    other->state == state->state) {
 394			merge_cb(tree, state, other);
 395			state->start = other->start;
 396			rb_erase(&other->rb_node, &tree->state);
 397			RB_CLEAR_NODE(&other->rb_node);
 398			free_extent_state(other);
 399		}
 400	}
 401	other_node = rb_next(&state->rb_node);
 402	if (other_node) {
 403		other = rb_entry(other_node, struct extent_state, rb_node);
 404		if (other->start == state->end + 1 &&
 405		    other->state == state->state) {
 406			merge_cb(tree, state, other);
 407			state->end = other->end;
 408			rb_erase(&other->rb_node, &tree->state);
 409			RB_CLEAR_NODE(&other->rb_node);
 410			free_extent_state(other);
 411		}
 412	}
 413}
 414
 415static void set_state_cb(struct extent_io_tree *tree,
 416			 struct extent_state *state, unsigned *bits)
 417{
 418	if (tree->ops && tree->ops->set_bit_hook)
 419		tree->ops->set_bit_hook(tree->mapping->host, state, bits);
 420}
 421
 422static void clear_state_cb(struct extent_io_tree *tree,
 423			   struct extent_state *state, unsigned *bits)
 424{
 425	if (tree->ops && tree->ops->clear_bit_hook)
 426		tree->ops->clear_bit_hook(tree->mapping->host, state, bits);
 427}
 428
 429static void set_state_bits(struct extent_io_tree *tree,
 430			   struct extent_state *state, unsigned *bits,
 431			   struct extent_changeset *changeset);
 432
 433/*
 434 * insert an extent_state struct into the tree.  'bits' are set on the
 435 * struct before it is inserted.
 436 *
 437 * This may return -EEXIST if the extent is already there, in which case the
 438 * state struct is freed.
 439 *
 440 * The tree lock is not taken internally.  This is a utility function and
 441 * probably isn't what you want to call (see set/clear_extent_bit).
 442 */
 443static int insert_state(struct extent_io_tree *tree,
 444			struct extent_state *state, u64 start, u64 end,
 445			struct rb_node ***p,
 446			struct rb_node **parent,
 447			unsigned *bits, struct extent_changeset *changeset)
 448{
 449	struct rb_node *node;
 450
 451	if (end < start)
 452		WARN(1, KERN_ERR "BTRFS: end < start %llu %llu\n",
 453		       end, start);
 454	state->start = start;
 455	state->end = end;
 456
 457	set_state_bits(tree, state, bits, changeset);
 458
 459	node = tree_insert(&tree->state, NULL, end, &state->rb_node, p, parent);
 460	if (node) {
 461		struct extent_state *found;
 462		found = rb_entry(node, struct extent_state, rb_node);
 463		printk(KERN_ERR "BTRFS: found node %llu %llu on insert of "
 464		       "%llu %llu\n",
 465		       found->start, found->end, start, end);
 466		return -EEXIST;
 467	}
 468	merge_state(tree, state);
 469	return 0;
 470}
 471
 472static void split_cb(struct extent_io_tree *tree, struct extent_state *orig,
 473		     u64 split)
 474{
 475	if (tree->ops && tree->ops->split_extent_hook)
 476		tree->ops->split_extent_hook(tree->mapping->host, orig, split);
 477}
 478
 479/*
 480 * split a given extent state struct in two, inserting the preallocated
 481 * struct 'prealloc' as the newly created second half.  'split' indicates an
 482 * offset inside 'orig' where it should be split.
 483 *
 484 * Before calling,
 485 * the tree has 'orig' at [orig->start, orig->end].  After calling, there
 486 * are two extent state structs in the tree:
 487 * prealloc: [orig->start, split - 1]
 488 * orig: [ split, orig->end ]
 489 *
 490 * The tree locks are not taken by this function. They need to be held
 491 * by the caller.
 492 */
 493static int split_state(struct extent_io_tree *tree, struct extent_state *orig,
 494		       struct extent_state *prealloc, u64 split)
 495{
 496	struct rb_node *node;
 497
 498	split_cb(tree, orig, split);
 499
 500	prealloc->start = orig->start;
 501	prealloc->end = split - 1;
 502	prealloc->state = orig->state;
 503	orig->start = split;
 504
 505	node = tree_insert(&tree->state, &orig->rb_node, prealloc->end,
 506			   &prealloc->rb_node, NULL, NULL);
 507	if (node) {
 508		free_extent_state(prealloc);
 509		return -EEXIST;
 510	}
 511	return 0;
 512}
 513
 514static struct extent_state *next_state(struct extent_state *state)
 515{
 516	struct rb_node *next = rb_next(&state->rb_node);
 517	if (next)
 518		return rb_entry(next, struct extent_state, rb_node);
 519	else
 520		return NULL;
 521}
 522
 523/*
 524 * utility function to clear some bits in an extent state struct.
 525 * it will optionally wake up any one waiting on this state (wake == 1).
 526 *
 527 * If no bits are set on the state struct after clearing things, the
 528 * struct is freed and removed from the tree
 529 */
 530static struct extent_state *clear_state_bit(struct extent_io_tree *tree,
 531					    struct extent_state *state,
 532					    unsigned *bits, int wake,
 533					    struct extent_changeset *changeset)
 534{
 535	struct extent_state *next;
 536	unsigned bits_to_clear = *bits & ~EXTENT_CTLBITS;
 537
 538	if ((bits_to_clear & EXTENT_DIRTY) && (state->state & EXTENT_DIRTY)) {
 539		u64 range = state->end - state->start + 1;
 540		WARN_ON(range > tree->dirty_bytes);
 541		tree->dirty_bytes -= range;
 542	}
 543	clear_state_cb(tree, state, bits);
 544	add_extent_changeset(state, bits_to_clear, changeset, 0);
 545	state->state &= ~bits_to_clear;
 546	if (wake)
 547		wake_up(&state->wq);
 548	if (state->state == 0) {
 549		next = next_state(state);
 550		if (extent_state_in_tree(state)) {
 551			rb_erase(&state->rb_node, &tree->state);
 552			RB_CLEAR_NODE(&state->rb_node);
 553			free_extent_state(state);
 554		} else {
 555			WARN_ON(1);
 556		}
 557	} else {
 558		merge_state(tree, state);
 559		next = next_state(state);
 560	}
 561	return next;
 562}
 563
 564static struct extent_state *
 565alloc_extent_state_atomic(struct extent_state *prealloc)
 566{
 567	if (!prealloc)
 568		prealloc = alloc_extent_state(GFP_ATOMIC);
 569
 570	return prealloc;
 571}
 572
 573static void extent_io_tree_panic(struct extent_io_tree *tree, int err)
 574{
 575	btrfs_panic(tree_fs_info(tree), err, "Locking error: "
 576		    "Extent tree was modified by another "
 577		    "thread while locked.");
 578}
 579
 580/*
 581 * clear some bits on a range in the tree.  This may require splitting
 582 * or inserting elements in the tree, so the gfp mask is used to
 583 * indicate which allocations or sleeping are allowed.
 584 *
 585 * pass 'wake' == 1 to kick any sleepers, and 'delete' == 1 to remove
 586 * the given range from the tree regardless of state (ie for truncate).
 587 *
 588 * the range [start, end] is inclusive.
 589 *
 590 * This takes the tree lock, and returns 0 on success and < 0 on error.
 591 */
 592static int __clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
 593			      unsigned bits, int wake, int delete,
 594			      struct extent_state **cached_state,
 595			      gfp_t mask, struct extent_changeset *changeset)
 596{
 597	struct extent_state *state;
 598	struct extent_state *cached;
 599	struct extent_state *prealloc = NULL;
 600	struct rb_node *node;
 601	u64 last_end;
 602	int err;
 603	int clear = 0;
 604
 605	btrfs_debug_check_extent_io_range(tree, start, end);
 606
 607	if (bits & EXTENT_DELALLOC)
 608		bits |= EXTENT_NORESERVE;
 609
 610	if (delete)
 611		bits |= ~EXTENT_CTLBITS;
 612	bits |= EXTENT_FIRST_DELALLOC;
 613
 614	if (bits & (EXTENT_IOBITS | EXTENT_BOUNDARY))
 615		clear = 1;
 616again:
 617	if (!prealloc && gfpflags_allow_blocking(mask)) {
 618		/*
 619		 * Don't care for allocation failure here because we might end
 620		 * up not needing the pre-allocated extent state at all, which
 621		 * is the case if we only have in the tree extent states that
 622		 * cover our input range and don't cover too any other range.
 623		 * If we end up needing a new extent state we allocate it later.
 624		 */
 625		prealloc = alloc_extent_state(mask);
 626	}
 627
 628	spin_lock(&tree->lock);
 629	if (cached_state) {
 630		cached = *cached_state;
 631
 632		if (clear) {
 633			*cached_state = NULL;
 634			cached_state = NULL;
 635		}
 636
 637		if (cached && extent_state_in_tree(cached) &&
 638		    cached->start <= start && cached->end > start) {
 639			if (clear)
 640				atomic_dec(&cached->refs);
 641			state = cached;
 642			goto hit_next;
 643		}
 644		if (clear)
 645			free_extent_state(cached);
 646	}
 647	/*
 648	 * this search will find the extents that end after
 649	 * our range starts
 650	 */
 651	node = tree_search(tree, start);
 652	if (!node)
 653		goto out;
 654	state = rb_entry(node, struct extent_state, rb_node);
 655hit_next:
 656	if (state->start > end)
 657		goto out;
 658	WARN_ON(state->end < start);
 659	last_end = state->end;
 660
 661	/* the state doesn't have the wanted bits, go ahead */
 662	if (!(state->state & bits)) {
 663		state = next_state(state);
 664		goto next;
 665	}
 666
 667	/*
 668	 *     | ---- desired range ---- |
 669	 *  | state | or
 670	 *  | ------------- state -------------- |
 671	 *
 672	 * We need to split the extent we found, and may flip
 673	 * bits on second half.
 674	 *
 675	 * If the extent we found extends past our range, we
 676	 * just split and search again.  It'll get split again
 677	 * the next time though.
 678	 *
 679	 * If the extent we found is inside our range, we clear
 680	 * the desired bit on it.
 681	 */
 682
 683	if (state->start < start) {
 684		prealloc = alloc_extent_state_atomic(prealloc);
 685		BUG_ON(!prealloc);
 686		err = split_state(tree, state, prealloc, start);
 687		if (err)
 688			extent_io_tree_panic(tree, err);
 689
 690		prealloc = NULL;
 691		if (err)
 692			goto out;
 693		if (state->end <= end) {
 694			state = clear_state_bit(tree, state, &bits, wake,
 695						changeset);
 696			goto next;
 697		}
 698		goto search_again;
 699	}
 700	/*
 701	 * | ---- desired range ---- |
 702	 *                        | state |
 703	 * We need to split the extent, and clear the bit
 704	 * on the first half
 705	 */
 706	if (state->start <= end && state->end > end) {
 707		prealloc = alloc_extent_state_atomic(prealloc);
 708		BUG_ON(!prealloc);
 709		err = split_state(tree, state, prealloc, end + 1);
 710		if (err)
 711			extent_io_tree_panic(tree, err);
 712
 713		if (wake)
 714			wake_up(&state->wq);
 715
 716		clear_state_bit(tree, prealloc, &bits, wake, changeset);
 717
 718		prealloc = NULL;
 719		goto out;
 720	}
 721
 722	state = clear_state_bit(tree, state, &bits, wake, changeset);
 723next:
 724	if (last_end == (u64)-1)
 725		goto out;
 726	start = last_end + 1;
 727	if (start <= end && state && !need_resched())
 728		goto hit_next;
 729	goto search_again;
 730
 731out:
 732	spin_unlock(&tree->lock);
 733	if (prealloc)
 734		free_extent_state(prealloc);
 735
 736	return 0;
 737
 738search_again:
 739	if (start > end)
 740		goto out;
 741	spin_unlock(&tree->lock);
 742	if (gfpflags_allow_blocking(mask))
 743		cond_resched();
 744	goto again;
 745}
 746
 747static void wait_on_state(struct extent_io_tree *tree,
 748			  struct extent_state *state)
 749		__releases(tree->lock)
 750		__acquires(tree->lock)
 751{
 752	DEFINE_WAIT(wait);
 753	prepare_to_wait(&state->wq, &wait, TASK_UNINTERRUPTIBLE);
 754	spin_unlock(&tree->lock);
 755	schedule();
 756	spin_lock(&tree->lock);
 757	finish_wait(&state->wq, &wait);
 758}
 759
 760/*
 761 * waits for one or more bits to clear on a range in the state tree.
 762 * The range [start, end] is inclusive.
 763 * The tree lock is taken by this function
 764 */
 765static void wait_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
 766			    unsigned long bits)
 767{
 768	struct extent_state *state;
 769	struct rb_node *node;
 770
 771	btrfs_debug_check_extent_io_range(tree, start, end);
 772
 773	spin_lock(&tree->lock);
 774again:
 775	while (1) {
 776		/*
 777		 * this search will find all the extents that end after
 778		 * our range starts
 779		 */
 780		node = tree_search(tree, start);
 781process_node:
 782		if (!node)
 783			break;
 784
 785		state = rb_entry(node, struct extent_state, rb_node);
 786
 787		if (state->start > end)
 788			goto out;
 789
 790		if (state->state & bits) {
 791			start = state->start;
 792			atomic_inc(&state->refs);
 793			wait_on_state(tree, state);
 794			free_extent_state(state);
 795			goto again;
 796		}
 797		start = state->end + 1;
 798
 799		if (start > end)
 800			break;
 801
 802		if (!cond_resched_lock(&tree->lock)) {
 803			node = rb_next(node);
 804			goto process_node;
 805		}
 806	}
 807out:
 808	spin_unlock(&tree->lock);
 809}
 810
 811static void set_state_bits(struct extent_io_tree *tree,
 812			   struct extent_state *state,
 813			   unsigned *bits, struct extent_changeset *changeset)
 814{
 815	unsigned bits_to_set = *bits & ~EXTENT_CTLBITS;
 816
 817	set_state_cb(tree, state, bits);
 818	if ((bits_to_set & EXTENT_DIRTY) && !(state->state & EXTENT_DIRTY)) {
 819		u64 range = state->end - state->start + 1;
 820		tree->dirty_bytes += range;
 821	}
 822	add_extent_changeset(state, bits_to_set, changeset, 1);
 823	state->state |= bits_to_set;
 824}
 825
 826static void cache_state_if_flags(struct extent_state *state,
 827				 struct extent_state **cached_ptr,
 828				 unsigned flags)
 829{
 830	if (cached_ptr && !(*cached_ptr)) {
 831		if (!flags || (state->state & flags)) {
 832			*cached_ptr = state;
 833			atomic_inc(&state->refs);
 834		}
 835	}
 836}
 837
 838static void cache_state(struct extent_state *state,
 839			struct extent_state **cached_ptr)
 840{
 841	return cache_state_if_flags(state, cached_ptr,
 842				    EXTENT_IOBITS | EXTENT_BOUNDARY);
 843}
 844
 845/*
 846 * set some bits on a range in the tree.  This may require allocations or
 847 * sleeping, so the gfp mask is used to indicate what is allowed.
 848 *
 849 * If any of the exclusive bits are set, this will fail with -EEXIST if some
 850 * part of the range already has the desired bits set.  The start of the
 851 * existing range is returned in failed_start in this case.
 852 *
 853 * [start, end] is inclusive This takes the tree lock.
 854 */
 855
 856static int __must_check
 857__set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
 858		 unsigned bits, unsigned exclusive_bits,
 859		 u64 *failed_start, struct extent_state **cached_state,
 860		 gfp_t mask, struct extent_changeset *changeset)
 861{
 862	struct extent_state *state;
 863	struct extent_state *prealloc = NULL;
 864	struct rb_node *node;
 865	struct rb_node **p;
 866	struct rb_node *parent;
 867	int err = 0;
 868	u64 last_start;
 869	u64 last_end;
 870
 871	btrfs_debug_check_extent_io_range(tree, start, end);
 872
 873	bits |= EXTENT_FIRST_DELALLOC;
 874again:
 875	if (!prealloc && gfpflags_allow_blocking(mask)) {
 876		prealloc = alloc_extent_state(mask);
 877		BUG_ON(!prealloc);
 878	}
 879
 880	spin_lock(&tree->lock);
 881	if (cached_state && *cached_state) {
 882		state = *cached_state;
 883		if (state->start <= start && state->end > start &&
 884		    extent_state_in_tree(state)) {
 885			node = &state->rb_node;
 886			goto hit_next;
 887		}
 888	}
 889	/*
 890	 * this search will find all the extents that end after
 891	 * our range starts.
 892	 */
 893	node = tree_search_for_insert(tree, start, &p, &parent);
 894	if (!node) {
 895		prealloc = alloc_extent_state_atomic(prealloc);
 896		BUG_ON(!prealloc);
 897		err = insert_state(tree, prealloc, start, end,
 898				   &p, &parent, &bits, changeset);
 899		if (err)
 900			extent_io_tree_panic(tree, err);
 901
 902		cache_state(prealloc, cached_state);
 903		prealloc = NULL;
 904		goto out;
 905	}
 906	state = rb_entry(node, struct extent_state, rb_node);
 907hit_next:
 908	last_start = state->start;
 909	last_end = state->end;
 910
 911	/*
 912	 * | ---- desired range ---- |
 913	 * | state |
 914	 *
 915	 * Just lock what we found and keep going
 916	 */
 917	if (state->start == start && state->end <= end) {
 918		if (state->state & exclusive_bits) {
 919			*failed_start = state->start;
 920			err = -EEXIST;
 921			goto out;
 922		}
 923
 924		set_state_bits(tree, state, &bits, changeset);
 925		cache_state(state, cached_state);
 926		merge_state(tree, state);
 927		if (last_end == (u64)-1)
 928			goto out;
 929		start = last_end + 1;
 930		state = next_state(state);
 931		if (start < end && state && state->start == start &&
 932		    !need_resched())
 933			goto hit_next;
 934		goto search_again;
 935	}
 936
 937	/*
 938	 *     | ---- desired range ---- |
 939	 * | state |
 940	 *   or
 941	 * | ------------- state -------------- |
 942	 *
 943	 * We need to split the extent we found, and may flip bits on
 944	 * second half.
 945	 *
 946	 * If the extent we found extends past our
 947	 * range, we just split and search again.  It'll get split
 948	 * again the next time though.
 949	 *
 950	 * If the extent we found is inside our range, we set the
 951	 * desired bit on it.
 952	 */
 953	if (state->start < start) {
 954		if (state->state & exclusive_bits) {
 955			*failed_start = start;
 956			err = -EEXIST;
 957			goto out;
 958		}
 959
 960		prealloc = alloc_extent_state_atomic(prealloc);
 961		BUG_ON(!prealloc);
 962		err = split_state(tree, state, prealloc, start);
 963		if (err)
 964			extent_io_tree_panic(tree, err);
 965
 966		prealloc = NULL;
 967		if (err)
 968			goto out;
 969		if (state->end <= end) {
 970			set_state_bits(tree, state, &bits, changeset);
 971			cache_state(state, cached_state);
 972			merge_state(tree, state);
 973			if (last_end == (u64)-1)
 974				goto out;
 975			start = last_end + 1;
 976			state = next_state(state);
 977			if (start < end && state && state->start == start &&
 978			    !need_resched())
 979				goto hit_next;
 980		}
 981		goto search_again;
 982	}
 983	/*
 984	 * | ---- desired range ---- |
 985	 *     | state | or               | state |
 986	 *
 987	 * There's a hole, we need to insert something in it and
 988	 * ignore the extent we found.
 989	 */
 990	if (state->start > start) {
 991		u64 this_end;
 992		if (end < last_start)
 993			this_end = end;
 994		else
 995			this_end = last_start - 1;
 996
 997		prealloc = alloc_extent_state_atomic(prealloc);
 998		BUG_ON(!prealloc);
 999
1000		/*
1001		 * Avoid to free 'prealloc' if it can be merged with
1002		 * the later extent.
1003		 */
1004		err = insert_state(tree, prealloc, start, this_end,
1005				   NULL, NULL, &bits, changeset);
1006		if (err)
1007			extent_io_tree_panic(tree, err);
1008
1009		cache_state(prealloc, cached_state);
1010		prealloc = NULL;
1011		start = this_end + 1;
1012		goto search_again;
1013	}
1014	/*
1015	 * | ---- desired range ---- |
1016	 *                        | state |
1017	 * We need to split the extent, and set the bit
1018	 * on the first half
1019	 */
1020	if (state->start <= end && state->end > end) {
1021		if (state->state & exclusive_bits) {
1022			*failed_start = start;
1023			err = -EEXIST;
1024			goto out;
1025		}
1026
1027		prealloc = alloc_extent_state_atomic(prealloc);
1028		BUG_ON(!prealloc);
1029		err = split_state(tree, state, prealloc, end + 1);
1030		if (err)
1031			extent_io_tree_panic(tree, err);
1032
1033		set_state_bits(tree, prealloc, &bits, changeset);
1034		cache_state(prealloc, cached_state);
1035		merge_state(tree, prealloc);
1036		prealloc = NULL;
1037		goto out;
1038	}
1039
1040	goto search_again;
1041
1042out:
1043	spin_unlock(&tree->lock);
1044	if (prealloc)
1045		free_extent_state(prealloc);
1046
1047	return err;
1048
1049search_again:
1050	if (start > end)
1051		goto out;
1052	spin_unlock(&tree->lock);
1053	if (gfpflags_allow_blocking(mask))
1054		cond_resched();
1055	goto again;
1056}
1057
1058int set_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1059		   unsigned bits, u64 * failed_start,
1060		   struct extent_state **cached_state, gfp_t mask)
1061{
1062	return __set_extent_bit(tree, start, end, bits, 0, failed_start,
1063				cached_state, mask, NULL);
1064}
1065
1066
1067/**
1068 * convert_extent_bit - convert all bits in a given range from one bit to
1069 * 			another
1070 * @tree:	the io tree to search
1071 * @start:	the start offset in bytes
1072 * @end:	the end offset in bytes (inclusive)
1073 * @bits:	the bits to set in this range
1074 * @clear_bits:	the bits to clear in this range
1075 * @cached_state:	state that we're going to cache
1076 * @mask:	the allocation mask
1077 *
1078 * This will go through and set bits for the given range.  If any states exist
1079 * already in this range they are set with the given bit and cleared of the
1080 * clear_bits.  This is only meant to be used by things that are mergeable, ie
1081 * converting from say DELALLOC to DIRTY.  This is not meant to be used with
1082 * boundary bits like LOCK.
1083 */
1084int convert_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1085		       unsigned bits, unsigned clear_bits,
1086		       struct extent_state **cached_state, gfp_t mask)
1087{
1088	struct extent_state *state;
1089	struct extent_state *prealloc = NULL;
1090	struct rb_node *node;
1091	struct rb_node **p;
1092	struct rb_node *parent;
1093	int err = 0;
1094	u64 last_start;
1095	u64 last_end;
1096	bool first_iteration = true;
1097
1098	btrfs_debug_check_extent_io_range(tree, start, end);
1099
1100again:
1101	if (!prealloc && gfpflags_allow_blocking(mask)) {
1102		/*
1103		 * Best effort, don't worry if extent state allocation fails
1104		 * here for the first iteration. We might have a cached state
1105		 * that matches exactly the target range, in which case no
1106		 * extent state allocations are needed. We'll only know this
1107		 * after locking the tree.
1108		 */
1109		prealloc = alloc_extent_state(mask);
1110		if (!prealloc && !first_iteration)
1111			return -ENOMEM;
1112	}
1113
1114	spin_lock(&tree->lock);
1115	if (cached_state && *cached_state) {
1116		state = *cached_state;
1117		if (state->start <= start && state->end > start &&
1118		    extent_state_in_tree(state)) {
1119			node = &state->rb_node;
1120			goto hit_next;
1121		}
1122	}
1123
1124	/*
1125	 * this search will find all the extents that end after
1126	 * our range starts.
1127	 */
1128	node = tree_search_for_insert(tree, start, &p, &parent);
1129	if (!node) {
1130		prealloc = alloc_extent_state_atomic(prealloc);
1131		if (!prealloc) {
1132			err = -ENOMEM;
1133			goto out;
1134		}
1135		err = insert_state(tree, prealloc, start, end,
1136				   &p, &parent, &bits, NULL);
1137		if (err)
1138			extent_io_tree_panic(tree, err);
1139		cache_state(prealloc, cached_state);
1140		prealloc = NULL;
1141		goto out;
1142	}
1143	state = rb_entry(node, struct extent_state, rb_node);
1144hit_next:
1145	last_start = state->start;
1146	last_end = state->end;
1147
1148	/*
1149	 * | ---- desired range ---- |
1150	 * | state |
1151	 *
1152	 * Just lock what we found and keep going
1153	 */
1154	if (state->start == start && state->end <= end) {
1155		set_state_bits(tree, state, &bits, NULL);
1156		cache_state(state, cached_state);
1157		state = clear_state_bit(tree, state, &clear_bits, 0, NULL);
1158		if (last_end == (u64)-1)
1159			goto out;
1160		start = last_end + 1;
1161		if (start < end && state && state->start == start &&
1162		    !need_resched())
1163			goto hit_next;
1164		goto search_again;
1165	}
1166
1167	/*
1168	 *     | ---- desired range ---- |
1169	 * | state |
1170	 *   or
1171	 * | ------------- state -------------- |
1172	 *
1173	 * We need to split the extent we found, and may flip bits on
1174	 * second half.
1175	 *
1176	 * If the extent we found extends past our
1177	 * range, we just split and search again.  It'll get split
1178	 * again the next time though.
1179	 *
1180	 * If the extent we found is inside our range, we set the
1181	 * desired bit on it.
1182	 */
1183	if (state->start < start) {
1184		prealloc = alloc_extent_state_atomic(prealloc);
1185		if (!prealloc) {
1186			err = -ENOMEM;
1187			goto out;
1188		}
1189		err = split_state(tree, state, prealloc, start);
1190		if (err)
1191			extent_io_tree_panic(tree, err);
1192		prealloc = NULL;
1193		if (err)
1194			goto out;
1195		if (state->end <= end) {
1196			set_state_bits(tree, state, &bits, NULL);
1197			cache_state(state, cached_state);
1198			state = clear_state_bit(tree, state, &clear_bits, 0,
1199						NULL);
1200			if (last_end == (u64)-1)
1201				goto out;
1202			start = last_end + 1;
1203			if (start < end && state && state->start == start &&
1204			    !need_resched())
1205				goto hit_next;
1206		}
1207		goto search_again;
1208	}
1209	/*
1210	 * | ---- desired range ---- |
1211	 *     | state | or               | state |
1212	 *
1213	 * There's a hole, we need to insert something in it and
1214	 * ignore the extent we found.
1215	 */
1216	if (state->start > start) {
1217		u64 this_end;
1218		if (end < last_start)
1219			this_end = end;
1220		else
1221			this_end = last_start - 1;
1222
1223		prealloc = alloc_extent_state_atomic(prealloc);
1224		if (!prealloc) {
1225			err = -ENOMEM;
1226			goto out;
1227		}
1228
1229		/*
1230		 * Avoid to free 'prealloc' if it can be merged with
1231		 * the later extent.
1232		 */
1233		err = insert_state(tree, prealloc, start, this_end,
1234				   NULL, NULL, &bits, NULL);
1235		if (err)
1236			extent_io_tree_panic(tree, err);
1237		cache_state(prealloc, cached_state);
1238		prealloc = NULL;
1239		start = this_end + 1;
1240		goto search_again;
1241	}
1242	/*
1243	 * | ---- desired range ---- |
1244	 *                        | state |
1245	 * We need to split the extent, and set the bit
1246	 * on the first half
1247	 */
1248	if (state->start <= end && state->end > end) {
1249		prealloc = alloc_extent_state_atomic(prealloc);
1250		if (!prealloc) {
1251			err = -ENOMEM;
1252			goto out;
1253		}
1254
1255		err = split_state(tree, state, prealloc, end + 1);
1256		if (err)
1257			extent_io_tree_panic(tree, err);
1258
1259		set_state_bits(tree, prealloc, &bits, NULL);
1260		cache_state(prealloc, cached_state);
1261		clear_state_bit(tree, prealloc, &clear_bits, 0, NULL);
1262		prealloc = NULL;
1263		goto out;
1264	}
1265
1266	goto search_again;
1267
1268out:
1269	spin_unlock(&tree->lock);
1270	if (prealloc)
1271		free_extent_state(prealloc);
1272
1273	return err;
1274
1275search_again:
1276	if (start > end)
1277		goto out;
1278	spin_unlock(&tree->lock);
1279	if (gfpflags_allow_blocking(mask))
1280		cond_resched();
1281	first_iteration = false;
1282	goto again;
1283}
1284
1285/* wrappers around set/clear extent bit */
1286int set_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1287			   unsigned bits, gfp_t mask,
1288			   struct extent_changeset *changeset)
1289{
1290	/*
1291	 * We don't support EXTENT_LOCKED yet, as current changeset will
1292	 * record any bits changed, so for EXTENT_LOCKED case, it will
1293	 * either fail with -EEXIST or changeset will record the whole
1294	 * range.
1295	 */
1296	BUG_ON(bits & EXTENT_LOCKED);
1297
1298	return __set_extent_bit(tree, start, end, bits, 0, NULL, NULL, mask,
1299				changeset);
1300}
1301
1302int clear_extent_bit(struct extent_io_tree *tree, u64 start, u64 end,
1303		     unsigned bits, int wake, int delete,
1304		     struct extent_state **cached, gfp_t mask)
1305{
1306	return __clear_extent_bit(tree, start, end, bits, wake, delete,
1307				  cached, mask, NULL);
1308}
1309
1310int clear_record_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1311			     unsigned bits, gfp_t mask,
1312			     struct extent_changeset *changeset)
1313{
1314	/*
1315	 * Don't support EXTENT_LOCKED case, same reason as
1316	 * set_record_extent_bits().
1317	 */
1318	BUG_ON(bits & EXTENT_LOCKED);
1319
1320	return __clear_extent_bit(tree, start, end, bits, 0, 0, NULL, mask,
1321				  changeset);
1322}
1323
1324/*
1325 * either insert or lock state struct between start and end use mask to tell
1326 * us if waiting is desired.
1327 */
1328int lock_extent_bits(struct extent_io_tree *tree, u64 start, u64 end,
1329		     struct extent_state **cached_state)
1330{
1331	int err;
1332	u64 failed_start;
1333
1334	while (1) {
1335		err = __set_extent_bit(tree, start, end, EXTENT_LOCKED,
1336				       EXTENT_LOCKED, &failed_start,
1337				       cached_state, GFP_NOFS, NULL);
1338		if (err == -EEXIST) {
1339			wait_extent_bit(tree, failed_start, end, EXTENT_LOCKED);
1340			start = failed_start;
1341		} else
1342			break;
1343		WARN_ON(start > end);
1344	}
1345	return err;
1346}
1347
1348int try_lock_extent(struct extent_io_tree *tree, u64 start, u64 end)
1349{
1350	int err;
1351	u64 failed_start;
1352
1353	err = __set_extent_bit(tree, start, end, EXTENT_LOCKED, EXTENT_LOCKED,
1354			       &failed_start, NULL, GFP_NOFS, NULL);
1355	if (err == -EEXIST) {
1356		if (failed_start > start)
1357			clear_extent_bit(tree, start, failed_start - 1,
1358					 EXTENT_LOCKED, 1, 0, NULL, GFP_NOFS);
1359		return 0;
1360	}
1361	return 1;
1362}
1363
1364void extent_range_clear_dirty_for_io(struct inode *inode, u64 start, u64 end)
1365{
1366	unsigned long index = start >> PAGE_SHIFT;
1367	unsigned long end_index = end >> PAGE_SHIFT;
1368	struct page *page;
1369
1370	while (index <= end_index) {
1371		page = find_get_page(inode->i_mapping, index);
1372		BUG_ON(!page); /* Pages should be in the extent_io_tree */
1373		clear_page_dirty_for_io(page);
1374		put_page(page);
1375		index++;
1376	}
1377}
1378
1379void extent_range_redirty_for_io(struct inode *inode, u64 start, u64 end)
1380{
1381	unsigned long index = start >> PAGE_SHIFT;
1382	unsigned long end_index = end >> PAGE_SHIFT;
1383	struct page *page;
1384
1385	while (index <= end_index) {
1386		page = find_get_page(inode->i_mapping, index);
1387		BUG_ON(!page); /* Pages should be in the extent_io_tree */
1388		__set_page_dirty_nobuffers(page);
1389		account_page_redirty(page);
1390		put_page(page);
1391		index++;
1392	}
1393}
1394
1395/*
1396 * helper function to set both pages and extents in the tree writeback
1397 */
1398static void set_range_writeback(struct extent_io_tree *tree, u64 start, u64 end)
1399{
1400	unsigned long index = start >> PAGE_SHIFT;
1401	unsigned long end_index = end >> PAGE_SHIFT;
1402	struct page *page;
 
 
 
 
 
 
 
 
 
1403
 
1404	while (index <= end_index) {
1405		page = find_get_page(tree->mapping, index);
1406		BUG_ON(!page); /* Pages should be in the extent_io_tree */
1407		set_page_writeback(page);
1408		put_page(page);
1409		index++;
1410	}
1411}
1412
1413/* find the first state struct with 'bits' set after 'start', and
1414 * return it.  tree->lock must be held.  NULL will returned if
1415 * nothing was found after 'start'
1416 */
1417static struct extent_state *
1418find_first_extent_bit_state(struct extent_io_tree *tree,
1419			    u64 start, unsigned bits)
1420{
1421	struct rb_node *node;
1422	struct extent_state *state;
1423
1424	/*
1425	 * this search will find all the extents that end after
1426	 * our range starts.
1427	 */
1428	node = tree_search(tree, start);
1429	if (!node)
1430		goto out;
1431
1432	while (1) {
1433		state = rb_entry(node, struct extent_state, rb_node);
1434		if (state->end >= start && (state->state & bits))
1435			return state;
1436
1437		node = rb_next(node);
1438		if (!node)
1439			break;
1440	}
1441out:
1442	return NULL;
1443}
1444
1445/*
1446 * find the first offset in the io tree with 'bits' set. zero is
1447 * returned if we find something, and *start_ret and *end_ret are
1448 * set to reflect the state struct that was found.
1449 *
1450 * If nothing was found, 1 is returned. If found something, return 0.
1451 */
1452int find_first_extent_bit(struct extent_io_tree *tree, u64 start,
1453			  u64 *start_ret, u64 *end_ret, unsigned bits,
1454			  struct extent_state **cached_state)
1455{
1456	struct extent_state *state;
1457	struct rb_node *n;
1458	int ret = 1;
1459
1460	spin_lock(&tree->lock);
1461	if (cached_state && *cached_state) {
1462		state = *cached_state;
1463		if (state->end == start - 1 && extent_state_in_tree(state)) {
1464			n = rb_next(&state->rb_node);
1465			while (n) {
1466				state = rb_entry(n, struct extent_state,
1467						 rb_node);
1468				if (state->state & bits)
1469					goto got_it;
1470				n = rb_next(n);
1471			}
1472			free_extent_state(*cached_state);
1473			*cached_state = NULL;
1474			goto out;
1475		}
1476		free_extent_state(*cached_state);
1477		*cached_state = NULL;
1478	}
1479
1480	state = find_first_extent_bit_state(tree, start, bits);
1481got_it:
1482	if (state) {
1483		cache_state_if_flags(state, cached_state, 0);
1484		*start_ret = state->start;
1485		*end_ret = state->end;
1486		ret = 0;
1487	}
1488out:
1489	spin_unlock(&tree->lock);
1490	return ret;
1491}
1492
1493/*
1494 * find a contiguous range of bytes in the file marked as delalloc, not
1495 * more than 'max_bytes'.  start and end are used to return the range,
1496 *
1497 * 1 is returned if we find something, 0 if nothing was in the tree
1498 */
1499static noinline u64 find_delalloc_range(struct extent_io_tree *tree,
1500					u64 *start, u64 *end, u64 max_bytes,
1501					struct extent_state **cached_state)
1502{
1503	struct rb_node *node;
1504	struct extent_state *state;
1505	u64 cur_start = *start;
1506	u64 found = 0;
1507	u64 total_bytes = 0;
1508
1509	spin_lock(&tree->lock);
1510
1511	/*
1512	 * this search will find all the extents that end after
1513	 * our range starts.
1514	 */
1515	node = tree_search(tree, cur_start);
1516	if (!node) {
1517		if (!found)
1518			*end = (u64)-1;
1519		goto out;
1520	}
1521
1522	while (1) {
1523		state = rb_entry(node, struct extent_state, rb_node);
1524		if (found && (state->start != cur_start ||
1525			      (state->state & EXTENT_BOUNDARY))) {
1526			goto out;
1527		}
1528		if (!(state->state & EXTENT_DELALLOC)) {
1529			if (!found)
1530				*end = state->end;
1531			goto out;
1532		}
1533		if (!found) {
1534			*start = state->start;
1535			*cached_state = state;
1536			atomic_inc(&state->refs);
1537		}
1538		found++;
1539		*end = state->end;
1540		cur_start = state->end + 1;
1541		node = rb_next(node);
1542		total_bytes += state->end - state->start + 1;
1543		if (total_bytes >= max_bytes)
1544			break;
1545		if (!node)
1546			break;
1547	}
1548out:
1549	spin_unlock(&tree->lock);
1550	return found;
1551}
1552
1553static noinline void __unlock_for_delalloc(struct inode *inode,
1554					   struct page *locked_page,
1555					   u64 start, u64 end)
1556{
1557	int ret;
1558	struct page *pages[16];
1559	unsigned long index = start >> PAGE_SHIFT;
1560	unsigned long end_index = end >> PAGE_SHIFT;
1561	unsigned long nr_pages = end_index - index + 1;
1562	int i;
1563
 
1564	if (index == locked_page->index && end_index == index)
1565		return;
1566
1567	while (nr_pages > 0) {
1568		ret = find_get_pages_contig(inode->i_mapping, index,
1569				     min_t(unsigned long, nr_pages,
1570				     ARRAY_SIZE(pages)), pages);
1571		for (i = 0; i < ret; i++) {
1572			if (pages[i] != locked_page)
1573				unlock_page(pages[i]);
1574			put_page(pages[i]);
1575		}
1576		nr_pages -= ret;
1577		index += ret;
1578		cond_resched();
1579	}
1580}
1581
1582static noinline int lock_delalloc_pages(struct inode *inode,
1583					struct page *locked_page,
1584					u64 delalloc_start,
1585					u64 delalloc_end)
1586{
1587	unsigned long index = delalloc_start >> PAGE_SHIFT;
1588	unsigned long start_index = index;
1589	unsigned long end_index = delalloc_end >> PAGE_SHIFT;
1590	unsigned long pages_locked = 0;
1591	struct page *pages[16];
1592	unsigned long nrpages;
1593	int ret;
1594	int i;
1595
1596	/* the caller is responsible for locking the start index */
1597	if (index == locked_page->index && index == end_index)
1598		return 0;
1599
1600	/* skip the page at the start index */
1601	nrpages = end_index - index + 1;
1602	while (nrpages > 0) {
1603		ret = find_get_pages_contig(inode->i_mapping, index,
1604				     min_t(unsigned long,
1605				     nrpages, ARRAY_SIZE(pages)), pages);
1606		if (ret == 0) {
1607			ret = -EAGAIN;
1608			goto done;
1609		}
1610		/* now we have an array of pages, lock them all */
1611		for (i = 0; i < ret; i++) {
1612			/*
1613			 * the caller is taking responsibility for
1614			 * locked_page
1615			 */
1616			if (pages[i] != locked_page) {
1617				lock_page(pages[i]);
1618				if (!PageDirty(pages[i]) ||
1619				    pages[i]->mapping != inode->i_mapping) {
1620					ret = -EAGAIN;
1621					unlock_page(pages[i]);
1622					put_page(pages[i]);
1623					goto done;
1624				}
1625			}
1626			put_page(pages[i]);
1627			pages_locked++;
1628		}
1629		nrpages -= ret;
1630		index += ret;
1631		cond_resched();
1632	}
1633	ret = 0;
1634done:
1635	if (ret && pages_locked) {
1636		__unlock_for_delalloc(inode, locked_page,
1637			      delalloc_start,
1638			      ((u64)(start_index + pages_locked - 1)) <<
1639			      PAGE_SHIFT);
1640	}
1641	return ret;
1642}
1643
1644/*
1645 * find a contiguous range of bytes in the file marked as delalloc, not
1646 * more than 'max_bytes'.  start and end are used to return the range,
 
 
 
 
 
1647 *
1648 * 1 is returned if we find something, 0 if nothing was in the tree
 
 
 
 
1649 */
1650STATIC u64 find_lock_delalloc_range(struct inode *inode,
1651				    struct extent_io_tree *tree,
1652				    struct page *locked_page, u64 *start,
1653				    u64 *end, u64 max_bytes)
1654{
 
 
 
 
 
 
1655	u64 delalloc_start;
1656	u64 delalloc_end;
1657	u64 found;
1658	struct extent_state *cached_state = NULL;
1659	int ret;
1660	int loops = 0;
1661
 
 
 
 
 
 
1662again:
1663	/* step one, find a bunch of delalloc bytes starting at start */
1664	delalloc_start = *start;
1665	delalloc_end = 0;
1666	found = find_delalloc_range(tree, &delalloc_start, &delalloc_end,
1667				    max_bytes, &cached_state);
1668	if (!found || delalloc_end <= *start) {
1669		*start = delalloc_start;
1670		*end = delalloc_end;
 
 
1671		free_extent_state(cached_state);
1672		return 0;
1673	}
1674
1675	/*
1676	 * start comes from the offset of locked_page.  We have to lock
1677	 * pages in order, so we can't process delalloc bytes before
1678	 * locked_page
1679	 */
1680	if (delalloc_start < *start)
1681		delalloc_start = *start;
1682
1683	/*
1684	 * make sure to limit the number of pages we try to lock down
1685	 */
1686	if (delalloc_end + 1 - delalloc_start > max_bytes)
1687		delalloc_end = delalloc_start + max_bytes - 1;
1688
1689	/* step two, lock all the pages after the page that has start */
1690	ret = lock_delalloc_pages(inode, locked_page,
1691				  delalloc_start, delalloc_end);
 
1692	if (ret == -EAGAIN) {
1693		/* some of the pages are gone, lets avoid looping by
1694		 * shortening the size of the delalloc range we're searching
1695		 */
1696		free_extent_state(cached_state);
1697		cached_state = NULL;
1698		if (!loops) {
1699			max_bytes = PAGE_SIZE;
1700			loops = 1;
1701			goto again;
1702		} else {
1703			found = 0;
1704			goto out_failed;
1705		}
1706	}
1707	BUG_ON(ret); /* Only valid values are 0 and -EAGAIN */
1708
1709	/* step three, lock the state bits for the whole range */
1710	lock_extent_bits(tree, delalloc_start, delalloc_end, &cached_state);
1711
1712	/* then test to make sure it is all still delalloc */
1713	ret = test_range_bit(tree, delalloc_start, delalloc_end,
1714			     EXTENT_DELALLOC, 1, cached_state);
1715	if (!ret) {
1716		unlock_extent_cached(tree, delalloc_start, delalloc_end,
1717				     &cached_state, GFP_NOFS);
1718		__unlock_for_delalloc(inode, locked_page,
1719			      delalloc_start, delalloc_end);
1720		cond_resched();
1721		goto again;
1722	}
1723	free_extent_state(cached_state);
1724	*start = delalloc_start;
1725	*end = delalloc_end;
1726out_failed:
1727	return found;
1728}
1729
1730void extent_clear_unlock_delalloc(struct inode *inode, u64 start, u64 end,
1731				 struct page *locked_page,
1732				 unsigned clear_bits,
1733				 unsigned long page_ops)
1734{
1735	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
1736	int ret;
1737	struct page *pages[16];
1738	unsigned long index = start >> PAGE_SHIFT;
1739	unsigned long end_index = end >> PAGE_SHIFT;
1740	unsigned long nr_pages = end_index - index + 1;
1741	int i;
1742
1743	clear_extent_bit(tree, start, end, clear_bits, 1, 0, NULL, GFP_NOFS);
1744	if (page_ops == 0)
1745		return;
1746
1747	if ((page_ops & PAGE_SET_ERROR) && nr_pages > 0)
1748		mapping_set_error(inode->i_mapping, -EIO);
1749
1750	while (nr_pages > 0) {
1751		ret = find_get_pages_contig(inode->i_mapping, index,
1752				     min_t(unsigned long,
1753				     nr_pages, ARRAY_SIZE(pages)), pages);
1754		for (i = 0; i < ret; i++) {
1755
1756			if (page_ops & PAGE_SET_PRIVATE2)
1757				SetPagePrivate2(pages[i]);
1758
1759			if (pages[i] == locked_page) {
1760				put_page(pages[i]);
1761				continue;
1762			}
1763			if (page_ops & PAGE_CLEAR_DIRTY)
1764				clear_page_dirty_for_io(pages[i]);
1765			if (page_ops & PAGE_SET_WRITEBACK)
1766				set_page_writeback(pages[i]);
1767			if (page_ops & PAGE_SET_ERROR)
1768				SetPageError(pages[i]);
1769			if (page_ops & PAGE_END_WRITEBACK)
1770				end_page_writeback(pages[i]);
1771			if (page_ops & PAGE_UNLOCK)
1772				unlock_page(pages[i]);
1773			put_page(pages[i]);
1774		}
1775		nr_pages -= ret;
1776		index += ret;
1777		cond_resched();
1778	}
1779}
1780
1781/*
1782 * count the number of bytes in the tree that have a given bit(s)
1783 * set.  This can be fairly slow, except for EXTENT_DIRTY which is
1784 * cached.  The total number found is returned.
1785 */
1786u64 count_range_bits(struct extent_io_tree *tree,
1787		     u64 *start, u64 search_end, u64 max_bytes,
1788		     unsigned bits, int contig)
1789{
1790	struct rb_node *node;
1791	struct extent_state *state;
1792	u64 cur_start = *start;
1793	u64 total_bytes = 0;
1794	u64 last = 0;
1795	int found = 0;
1796
1797	if (WARN_ON(search_end <= cur_start))
1798		return 0;
1799
1800	spin_lock(&tree->lock);
1801	if (cur_start == 0 && bits == EXTENT_DIRTY) {
1802		total_bytes = tree->dirty_bytes;
1803		goto out;
1804	}
1805	/*
1806	 * this search will find all the extents that end after
1807	 * our range starts.
1808	 */
1809	node = tree_search(tree, cur_start);
1810	if (!node)
1811		goto out;
1812
1813	while (1) {
1814		state = rb_entry(node, struct extent_state, rb_node);
1815		if (state->start > search_end)
1816			break;
1817		if (contig && found && state->start > last + 1)
1818			break;
1819		if (state->end >= cur_start && (state->state & bits) == bits) {
1820			total_bytes += min(search_end, state->end) + 1 -
1821				       max(cur_start, state->start);
1822			if (total_bytes >= max_bytes)
1823				break;
1824			if (!found) {
1825				*start = max(cur_start, state->start);
1826				found = 1;
1827			}
1828			last = state->end;
1829		} else if (contig && found) {
1830			break;
1831		}
1832		node = rb_next(node);
1833		if (!node)
1834			break;
1835	}
1836out:
1837	spin_unlock(&tree->lock);
1838	return total_bytes;
1839}
1840
1841/*
1842 * set the private field for a given byte offset in the tree.  If there isn't
1843 * an extent_state there already, this does nothing.
1844 */
1845static noinline int set_state_failrec(struct extent_io_tree *tree, u64 start,
1846		struct io_failure_record *failrec)
1847{
1848	struct rb_node *node;
1849	struct extent_state *state;
1850	int ret = 0;
1851
1852	spin_lock(&tree->lock);
1853	/*
1854	 * this search will find all the extents that end after
1855	 * our range starts.
1856	 */
1857	node = tree_search(tree, start);
1858	if (!node) {
1859		ret = -ENOENT;
1860		goto out;
1861	}
1862	state = rb_entry(node, struct extent_state, rb_node);
1863	if (state->start != start) {
1864		ret = -ENOENT;
1865		goto out;
1866	}
1867	state->failrec = failrec;
1868out:
1869	spin_unlock(&tree->lock);
1870	return ret;
1871}
1872
1873static noinline int get_state_failrec(struct extent_io_tree *tree, u64 start,
1874		struct io_failure_record **failrec)
1875{
1876	struct rb_node *node;
1877	struct extent_state *state;
1878	int ret = 0;
1879
1880	spin_lock(&tree->lock);
1881	/*
1882	 * this search will find all the extents that end after
1883	 * our range starts.
1884	 */
1885	node = tree_search(tree, start);
1886	if (!node) {
1887		ret = -ENOENT;
1888		goto out;
1889	}
1890	state = rb_entry(node, struct extent_state, rb_node);
1891	if (state->start != start) {
1892		ret = -ENOENT;
1893		goto out;
1894	}
1895	*failrec = state->failrec;
1896out:
1897	spin_unlock(&tree->lock);
1898	return ret;
1899}
1900
1901/*
1902 * searches a range in the state tree for a given mask.
1903 * If 'filled' == 1, this returns 1 only if every extent in the tree
1904 * has the bits set.  Otherwise, 1 is returned if any bit in the
1905 * range is found set.
 
 
 
 
1906 */
1907int test_range_bit(struct extent_io_tree *tree, u64 start, u64 end,
1908		   unsigned bits, int filled, struct extent_state *cached)
1909{
1910	struct extent_state *state = NULL;
1911	struct rb_node *node;
1912	int bitset = 0;
1913
1914	spin_lock(&tree->lock);
1915	if (cached && extent_state_in_tree(cached) && cached->start <= start &&
1916	    cached->end > start)
1917		node = &cached->rb_node;
1918	else
1919		node = tree_search(tree, start);
1920	while (node && start <= end) {
1921		state = rb_entry(node, struct extent_state, rb_node);
1922
1923		if (filled && state->start > start) {
1924			bitset = 0;
1925			break;
1926		}
1927
1928		if (state->start > end)
1929			break;
1930
1931		if (state->state & bits) {
1932			bitset = 1;
1933			if (!filled)
1934				break;
1935		} else if (filled) {
1936			bitset = 0;
1937			break;
1938		}
1939
1940		if (state->end == (u64)-1)
1941			break;
1942
1943		start = state->end + 1;
1944		if (start > end)
1945			break;
1946		node = rb_next(node);
1947		if (!node) {
1948			if (filled)
1949				bitset = 0;
1950			break;
1951		}
1952	}
1953	spin_unlock(&tree->lock);
1954	return bitset;
1955}
1956
1957/*
1958 * helper function to set a given page up to date if all the
1959 * extents in the tree for that page are up to date
 
 
1960 */
1961static void check_page_uptodate(struct extent_io_tree *tree, struct page *page)
1962{
1963	u64 start = page_offset(page);
1964	u64 end = start + PAGE_SIZE - 1;
1965	if (test_range_bit(tree, start, end, EXTENT_UPTODATE, 1, NULL))
1966		SetPageUptodate(page);
1967}
1968
1969int free_io_failure(struct inode *inode, struct io_failure_record *rec)
1970{
1971	int ret;
1972	int err = 0;
1973	struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
1974
1975	set_state_failrec(failure_tree, rec->start, NULL);
1976	ret = clear_extent_bits(failure_tree, rec->start,
1977				rec->start + rec->len - 1,
1978				EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
1979	if (ret)
1980		err = ret;
1981
1982	ret = clear_extent_bits(&BTRFS_I(inode)->io_tree, rec->start,
1983				rec->start + rec->len - 1,
1984				EXTENT_DAMAGED, GFP_NOFS);
1985	if (ret && !err)
1986		err = ret;
1987
1988	kfree(rec);
1989	return err;
1990}
1991
1992/*
1993 * this bypasses the standard btrfs submit functions deliberately, as
1994 * the standard behavior is to write all copies in a raid setup. here we only
1995 * want to write the one bad copy. so we do the mapping for ourselves and issue
1996 * submit_bio directly.
1997 * to avoid any synchronization issues, wait for the data after writing, which
1998 * actually prevents the read that triggered the error from finishing.
1999 * currently, there can be no more than two copies of every data bit. thus,
2000 * exactly one rewrite is required.
 
2001 */
2002int repair_io_failure(struct inode *inode, u64 start, u64 length, u64 logical,
2003		      struct page *page, unsigned int pg_offset, int mirror_num)
 
2004{
2005	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2006	struct bio *bio;
2007	struct btrfs_device *dev;
2008	u64 map_length = 0;
2009	u64 sector;
2010	struct btrfs_bio *bbio = NULL;
2011	struct btrfs_mapping_tree *map_tree = &fs_info->mapping_tree;
2012	int ret;
2013
2014	ASSERT(!(fs_info->sb->s_flags & MS_RDONLY));
2015	BUG_ON(!mirror_num);
2016
2017	/* we can't repair anything in raid56 yet */
2018	if (btrfs_is_parity_mirror(map_tree, logical, length, mirror_num))
2019		return 0;
2020
2021	bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
2022	if (!bio)
2023		return -EIO;
2024	bio->bi_iter.bi_size = 0;
2025	map_length = length;
2026
2027	ret = btrfs_map_block(fs_info, WRITE, logical,
2028			      &map_length, &bbio, mirror_num);
2029	if (ret) {
2030		bio_put(bio);
2031		return -EIO;
2032	}
2033	BUG_ON(mirror_num != bbio->mirror_num);
2034	sector = bbio->stripes[mirror_num-1].physical >> 9;
2035	bio->bi_iter.bi_sector = sector;
2036	dev = bbio->stripes[mirror_num-1].dev;
2037	btrfs_put_bbio(bbio);
2038	if (!dev || !dev->bdev || !dev->writeable) {
2039		bio_put(bio);
2040		return -EIO;
 
2041	}
2042	bio->bi_bdev = dev->bdev;
2043	bio_add_page(bio, page, length, pg_offset);
2044
2045	if (btrfsic_submit_bio_wait(WRITE_SYNC, bio)) {
2046		/* try to remap that extent elsewhere? */
2047		bio_put(bio);
2048		btrfs_dev_stat_inc_and_print(dev, BTRFS_DEV_STAT_WRITE_ERRS);
2049		return -EIO;
2050	}
2051
2052	btrfs_info_rl_in_rcu(fs_info,
2053		"read error corrected: ino %llu off %llu (dev %s sector %llu)",
2054				  btrfs_ino(inode), start,
2055				  rcu_str_deref(dev->name), sector);
2056	bio_put(bio);
2057	return 0;
2058}
2059
2060int repair_eb_io_failure(struct btrfs_root *root, struct extent_buffer *eb,
2061			 int mirror_num)
2062{
2063	u64 start = eb->start;
2064	unsigned long i, num_pages = num_extent_pages(eb->start, eb->len);
2065	int ret = 0;
2066
2067	if (root->fs_info->sb->s_flags & MS_RDONLY)
2068		return -EROFS;
2069
2070	for (i = 0; i < num_pages; i++) {
2071		struct page *p = eb->pages[i];
2072
2073		ret = repair_io_failure(root->fs_info->btree_inode, start,
2074					PAGE_SIZE, start, p,
2075					start - page_offset(p), mirror_num);
2076		if (ret)
2077			break;
2078		start += PAGE_SIZE;
2079	}
2080
2081	return ret;
 
2082}
2083
2084/*
2085 * each time an IO finishes, we do a fast check in the IO failure tree
2086 * to see if we need to process or clean up an io_failure_record
 
 
 
 
 
 
 
 
2087 */
2088int clean_io_failure(struct inode *inode, u64 start, struct page *page,
2089		     unsigned int pg_offset)
2090{
2091	u64 private;
2092	struct io_failure_record *failrec;
2093	struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2094	struct extent_state *state;
2095	int num_copies;
2096	int ret;
2097
2098	private = 0;
2099	ret = count_range_bits(&BTRFS_I(inode)->io_failure_tree, &private,
2100				(u64)-1, 1, EXTENT_DIRTY, 0);
2101	if (!ret)
2102		return 0;
 
 
 
 
 
 
 
 
 
2103
2104	ret = get_state_failrec(&BTRFS_I(inode)->io_failure_tree, start,
2105			&failrec);
2106	if (ret)
2107		return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2108
2109	BUG_ON(!failrec->this_mirror);
 
 
2110
2111	if (failrec->in_validation) {
2112		/* there was no real error, just free the record */
2113		pr_debug("clean_io_failure: freeing dummy error at %llu\n",
2114			 failrec->start);
2115		goto out;
2116	}
2117	if (fs_info->sb->s_flags & MS_RDONLY)
2118		goto out;
 
 
 
 
 
 
2119
2120	spin_lock(&BTRFS_I(inode)->io_tree.lock);
2121	state = find_first_extent_bit_state(&BTRFS_I(inode)->io_tree,
2122					    failrec->start,
2123					    EXTENT_LOCKED);
2124	spin_unlock(&BTRFS_I(inode)->io_tree.lock);
2125
2126	if (state && state->start <= failrec->start &&
2127	    state->end >= failrec->start + failrec->len - 1) {
2128		num_copies = btrfs_num_copies(fs_info, failrec->logical,
2129					      failrec->len);
2130		if (num_copies > 1)  {
2131			repair_io_failure(inode, start, failrec->len,
2132					  failrec->logical, page,
2133					  pg_offset, failrec->failed_mirror);
2134		}
2135	}
2136
2137out:
2138	free_io_failure(inode, failrec);
2139
2140	return 0;
2141}
2142
2143/*
2144 * Can be called when
2145 * - hold extent lock
2146 * - under ordered extent
2147 * - the inode is freeing
2148 */
2149void btrfs_free_io_failure_record(struct inode *inode, u64 start, u64 end)
2150{
2151	struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2152	struct io_failure_record *failrec;
2153	struct extent_state *state, *next;
2154
2155	if (RB_EMPTY_ROOT(&failure_tree->state))
2156		return;
2157
2158	spin_lock(&failure_tree->lock);
2159	state = find_first_extent_bit_state(failure_tree, start, EXTENT_DIRTY);
2160	while (state) {
2161		if (state->start > end)
2162			break;
2163
2164		ASSERT(state->end <= end);
2165
2166		next = next_state(state);
2167
2168		failrec = state->failrec;
2169		free_extent_state(state);
2170		kfree(failrec);
2171
2172		state = next;
2173	}
2174	spin_unlock(&failure_tree->lock);
 
 
2175}
2176
2177int btrfs_get_io_failure_record(struct inode *inode, u64 start, u64 end,
2178		struct io_failure_record **failrec_ret)
2179{
2180	struct io_failure_record *failrec;
2181	struct extent_map *em;
2182	struct extent_io_tree *failure_tree = &BTRFS_I(inode)->io_failure_tree;
2183	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2184	struct extent_map_tree *em_tree = &BTRFS_I(inode)->extent_tree;
2185	int ret;
2186	u64 logical;
2187
2188	ret = get_state_failrec(failure_tree, start, &failrec);
2189	if (ret) {
2190		failrec = kzalloc(sizeof(*failrec), GFP_NOFS);
2191		if (!failrec)
 
 
 
 
 
 
 
 
 
 
 
 
 
2192			return -ENOMEM;
2193
2194		failrec->start = start;
2195		failrec->len = end - start + 1;
2196		failrec->this_mirror = 0;
2197		failrec->bio_flags = 0;
2198		failrec->in_validation = 0;
2199
2200		read_lock(&em_tree->lock);
2201		em = lookup_extent_mapping(em_tree, start, failrec->len);
2202		if (!em) {
2203			read_unlock(&em_tree->lock);
2204			kfree(failrec);
2205			return -EIO;
2206		}
2207
2208		if (em->start > start || em->start + em->len <= start) {
2209			free_extent_map(em);
2210			em = NULL;
2211		}
2212		read_unlock(&em_tree->lock);
2213		if (!em) {
2214			kfree(failrec);
2215			return -EIO;
2216		}
2217
2218		logical = start - em->start;
2219		logical = em->block_start + logical;
2220		if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2221			logical = em->block_start;
2222			failrec->bio_flags = EXTENT_BIO_COMPRESSED;
2223			extent_set_compress_type(&failrec->bio_flags,
2224						 em->compress_type);
2225		}
2226
2227		pr_debug("Get IO Failure Record: (new) logical=%llu, start=%llu, len=%llu\n",
2228			 logical, start, failrec->len);
2229
2230		failrec->logical = logical;
2231		free_extent_map(em);
2232
2233		/* set the bits in the private failure tree */
2234		ret = set_extent_bits(failure_tree, start, end,
2235					EXTENT_LOCKED | EXTENT_DIRTY, GFP_NOFS);
2236		if (ret >= 0)
2237			ret = set_state_failrec(failure_tree, start, failrec);
2238		/* set the bits in the inode's tree */
2239		if (ret >= 0)
2240			ret = set_extent_bits(tree, start, end, EXTENT_DAMAGED,
2241						GFP_NOFS);
2242		if (ret < 0) {
2243			kfree(failrec);
2244			return ret;
2245		}
2246	} else {
2247		pr_debug("Get IO Failure Record: (found) logical=%llu, start=%llu, len=%llu, validation=%d\n",
2248			 failrec->logical, failrec->start, failrec->len,
2249			 failrec->in_validation);
2250		/*
2251		 * when data can be on disk more than twice, add to failrec here
2252		 * (e.g. with a list for failed_mirror) to make
2253		 * clean_io_failure() clean all those errors at once.
2254		 */
2255	}
2256
2257	*failrec_ret = failrec;
2258
2259	return 0;
2260}
2261
2262int btrfs_check_repairable(struct inode *inode, struct bio *failed_bio,
2263			   struct io_failure_record *failrec, int failed_mirror)
2264{
2265	int num_copies;
2266
2267	num_copies = btrfs_num_copies(BTRFS_I(inode)->root->fs_info,
2268				      failrec->logical, failrec->len);
2269	if (num_copies == 1) {
2270		/*
2271		 * we only have a single copy of the data, so don't bother with
2272		 * all the retry and error correction code that follows. no
2273		 * matter what the error is, it is very likely to persist.
2274		 */
2275		pr_debug("Check Repairable: cannot repair, num_copies=%d, next_mirror %d, failed_mirror %d\n",
2276			 num_copies, failrec->this_mirror, failed_mirror);
2277		return 0;
2278	}
2279
2280	/*
2281	 * there are two premises:
2282	 *	a) deliver good data to the caller
2283	 *	b) correct the bad sectors on disk
2284	 */
2285	if (failed_bio->bi_vcnt > 1) {
2286		/*
2287		 * to fulfill b), we need to know the exact failing sectors, as
2288		 * we don't want to rewrite any more than the failed ones. thus,
2289		 * we need separate read requests for the failed bio
2290		 *
2291		 * if the following BUG_ON triggers, our validation request got
2292		 * merged. we need separate requests for our algorithm to work.
2293		 */
2294		BUG_ON(failrec->in_validation);
2295		failrec->in_validation = 1;
2296		failrec->this_mirror = failed_mirror;
2297	} else {
2298		/*
2299		 * we're ready to fulfill a) and b) alongside. get a good copy
2300		 * of the failed sector and if we succeed, we have setup
2301		 * everything for repair_io_failure to do the rest for us.
2302		 */
2303		if (failrec->in_validation) {
2304			BUG_ON(failrec->this_mirror != failed_mirror);
2305			failrec->in_validation = 0;
2306			failrec->this_mirror = 0;
2307		}
2308		failrec->failed_mirror = failed_mirror;
2309		failrec->this_mirror++;
2310		if (failrec->this_mirror == failed_mirror)
2311			failrec->this_mirror++;
2312	}
2313
2314	if (failrec->this_mirror > num_copies) {
2315		pr_debug("Check Repairable: (fail) num_copies=%d, next_mirror %d, failed_mirror %d\n",
2316			 num_copies, failrec->this_mirror, failed_mirror);
2317		return 0;
2318	}
2319
2320	return 1;
2321}
2322
2323
2324struct bio *btrfs_create_repair_bio(struct inode *inode, struct bio *failed_bio,
2325				    struct io_failure_record *failrec,
2326				    struct page *page, int pg_offset, int icsum,
2327				    bio_end_io_t *endio_func, void *data)
2328{
2329	struct bio *bio;
2330	struct btrfs_io_bio *btrfs_failed_bio;
2331	struct btrfs_io_bio *btrfs_bio;
2332
2333	bio = btrfs_io_bio_alloc(GFP_NOFS, 1);
2334	if (!bio)
2335		return NULL;
2336
2337	bio->bi_end_io = endio_func;
2338	bio->bi_iter.bi_sector = failrec->logical >> 9;
2339	bio->bi_bdev = BTRFS_I(inode)->root->fs_info->fs_devices->latest_bdev;
2340	bio->bi_iter.bi_size = 0;
2341	bio->bi_private = data;
2342
2343	btrfs_failed_bio = btrfs_io_bio(failed_bio);
2344	if (btrfs_failed_bio->csum) {
2345		struct btrfs_fs_info *fs_info = BTRFS_I(inode)->root->fs_info;
2346		u16 csum_size = btrfs_super_csum_size(fs_info->super_copy);
2347
2348		btrfs_bio = btrfs_io_bio(bio);
2349		btrfs_bio->csum = btrfs_bio->csum_inline;
2350		icsum *= csum_size;
2351		memcpy(btrfs_bio->csum, btrfs_failed_bio->csum + icsum,
2352		       csum_size);
2353	}
2354
2355	bio_add_page(bio, page, failrec->len, pg_offset);
2356
2357	return bio;
2358}
2359
2360/*
2361 * this is a generic handler for readpage errors (default
2362 * readpage_io_failed_hook). if other copies exist, read those and write back
2363 * good data to the failed position. does not investigate in remapping the
2364 * failed extent elsewhere, hoping the device will be smart enough to do this as
2365 * needed
2366 */
2367
2368static int bio_readpage_error(struct bio *failed_bio, u64 phy_offset,
2369			      struct page *page, u64 start, u64 end,
2370			      int failed_mirror)
2371{
2372	struct io_failure_record *failrec;
2373	struct inode *inode = page->mapping->host;
2374	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
2375	struct bio *bio;
2376	int read_mode;
2377	int ret;
2378
2379	BUG_ON(failed_bio->bi_rw & REQ_WRITE);
2380
2381	ret = btrfs_get_io_failure_record(inode, start, end, &failrec);
2382	if (ret)
2383		return ret;
2384
2385	ret = btrfs_check_repairable(inode, failed_bio, failrec, failed_mirror);
2386	if (!ret) {
2387		free_io_failure(inode, failrec);
2388		return -EIO;
2389	}
2390
2391	if (failed_bio->bi_vcnt > 1)
2392		read_mode = READ_SYNC | REQ_FAILFAST_DEV;
2393	else
2394		read_mode = READ_SYNC;
2395
2396	phy_offset >>= inode->i_sb->s_blocksize_bits;
2397	bio = btrfs_create_repair_bio(inode, failed_bio, failrec, page,
2398				      start - page_offset(page),
2399				      (int)phy_offset, failed_bio->bi_end_io,
2400				      NULL);
2401	if (!bio) {
2402		free_io_failure(inode, failrec);
2403		return -EIO;
2404	}
2405
2406	pr_debug("Repair Read Error: submitting new read[%#x] to this_mirror=%d, in_validation=%d\n",
2407		 read_mode, failrec->this_mirror, failrec->in_validation);
2408
2409	ret = tree->ops->submit_bio_hook(inode, read_mode, bio,
2410					 failrec->this_mirror,
2411					 failrec->bio_flags, 0);
2412	if (ret) {
2413		free_io_failure(inode, failrec);
2414		bio_put(bio);
2415	}
2416
2417	return ret;
2418}
2419
2420/* lots and lots of room for performance fixes in the end_bio funcs */
2421
2422void end_extent_writepage(struct page *page, int err, u64 start, u64 end)
2423{
2424	int uptodate = (err == 0);
2425	struct extent_io_tree *tree;
2426	int ret = 0;
2427
2428	tree = &BTRFS_I(page->mapping->host)->io_tree;
2429
2430	if (tree->ops && tree->ops->writepage_end_io_hook) {
2431		ret = tree->ops->writepage_end_io_hook(page, start,
2432					       end, NULL, uptodate);
2433		if (ret)
2434			uptodate = 0;
2435	}
2436
2437	if (!uptodate) {
2438		ClearPageUptodate(page);
2439		SetPageError(page);
2440		ret = ret < 0 ? ret : -EIO;
2441		mapping_set_error(page->mapping, ret);
2442	}
2443}
2444
2445/*
2446 * after a writepage IO is done, we need to:
2447 * clear the uptodate bits on error
2448 * clear the writeback bits in the extent tree for this IO
2449 * end_page_writeback if the page has no more pending IO
2450 *
2451 * Scheduling is not allowed, so the extent state tree is expected
2452 * to have one and only one object corresponding to this IO.
2453 */
2454static void end_bio_extent_writepage(struct bio *bio)
2455{
2456	struct bio_vec *bvec;
2457	u64 start;
2458	u64 end;
2459	int i;
2460
2461	bio_for_each_segment_all(bvec, bio, i) {
2462		struct page *page = bvec->bv_page;
2463
2464		/* We always issue full-page reads, but if some block
2465		 * in a page fails to read, blk_update_request() will
2466		 * advance bv_offset and adjust bv_len to compensate.
2467		 * Print a warning for nonzero offsets, and an error
2468		 * if they don't add up to a full page.  */
2469		if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2470			if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2471				btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2472				   "partial page write in btrfs with offset %u and length %u",
2473					bvec->bv_offset, bvec->bv_len);
2474			else
2475				btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2476				   "incomplete page write in btrfs with offset %u and "
2477				   "length %u",
2478					bvec->bv_offset, bvec->bv_len);
 
 
 
 
 
 
 
2479		}
2480
2481		start = page_offset(page);
2482		end = start + bvec->bv_offset + bvec->bv_len - 1;
2483
2484		end_extent_writepage(page, bio->bi_error, start, end);
2485		end_page_writeback(page);
 
 
 
2486	}
2487
2488	bio_put(bio);
2489}
2490
2491static void
2492endio_readpage_release_extent(struct extent_io_tree *tree, u64 start, u64 len,
2493			      int uptodate)
2494{
2495	struct extent_state *cached = NULL;
2496	u64 end = start + len - 1;
2497
2498	if (uptodate && tree->track_uptodate)
2499		set_extent_uptodate(tree, start, end, &cached, GFP_ATOMIC);
2500	unlock_extent_cached(tree, start, end, &cached, GFP_ATOMIC);
2501}
2502
2503/*
2504 * after a readpage IO is done, we need to:
2505 * clear the uptodate bits on error
2506 * set the uptodate bits if things worked
2507 * set the page up to date if all extents in the tree are uptodate
2508 * clear the lock bit in the extent tree
2509 * unlock the page if there are no other extents locked for it
2510 *
2511 * Scheduling is not allowed, so the extent state tree is expected
2512 * to have one and only one object corresponding to this IO.
2513 */
2514static void end_bio_extent_readpage(struct bio *bio)
2515{
2516	struct bio_vec *bvec;
2517	int uptodate = !bio->bi_error;
2518	struct btrfs_io_bio *io_bio = btrfs_io_bio(bio);
2519	struct extent_io_tree *tree;
2520	u64 offset = 0;
2521	u64 start;
2522	u64 end;
2523	u64 len;
2524	u64 extent_start = 0;
2525	u64 extent_len = 0;
2526	int mirror;
2527	int ret;
2528	int i;
2529
2530	bio_for_each_segment_all(bvec, bio, i) {
2531		struct page *page = bvec->bv_page;
2532		struct inode *inode = page->mapping->host;
2533
2534		pr_debug("end_bio_extent_readpage: bi_sector=%llu, err=%d, "
2535			 "mirror=%u\n", (u64)bio->bi_iter.bi_sector,
2536			 bio->bi_error, io_bio->mirror_num);
2537		tree = &BTRFS_I(inode)->io_tree;
2538
2539		/* We always issue full-page reads, but if some block
2540		 * in a page fails to read, blk_update_request() will
2541		 * advance bv_offset and adjust bv_len to compensate.
2542		 * Print a warning for nonzero offsets, and an error
2543		 * if they don't add up to a full page.  */
2544		if (bvec->bv_offset || bvec->bv_len != PAGE_SIZE) {
2545			if (bvec->bv_offset + bvec->bv_len != PAGE_SIZE)
2546				btrfs_err(BTRFS_I(page->mapping->host)->root->fs_info,
2547				   "partial page read in btrfs with offset %u and length %u",
2548					bvec->bv_offset, bvec->bv_len);
2549			else
2550				btrfs_info(BTRFS_I(page->mapping->host)->root->fs_info,
2551				   "incomplete page read in btrfs with offset %u and "
2552				   "length %u",
2553					bvec->bv_offset, bvec->bv_len);
2554		}
2555
2556		start = page_offset(page);
2557		end = start + bvec->bv_offset + bvec->bv_len - 1;
2558		len = bvec->bv_len;
2559
2560		mirror = io_bio->mirror_num;
2561		if (likely(uptodate && tree->ops &&
2562			   tree->ops->readpage_end_io_hook)) {
2563			ret = tree->ops->readpage_end_io_hook(io_bio, offset,
2564							      page, start, end,
2565							      mirror);
2566			if (ret)
2567				uptodate = 0;
2568			else
2569				clean_io_failure(inode, start, page, 0);
2570		}
2571
2572		if (likely(uptodate))
2573			goto readpage_ok;
2574
2575		if (tree->ops && tree->ops->readpage_io_failed_hook) {
2576			ret = tree->ops->readpage_io_failed_hook(page, mirror);
2577			if (!ret && !bio->bi_error)
2578				uptodate = 1;
2579		} else {
2580			/*
2581			 * The generic bio_readpage_error handles errors the
2582			 * following way: If possible, new read requests are
2583			 * created and submitted and will end up in
2584			 * end_bio_extent_readpage as well (if we're lucky, not
2585			 * in the !uptodate case). In that case it returns 0 and
2586			 * we just go on with the next page in our bio. If it
2587			 * can't handle the error it will return -EIO and we
2588			 * remain responsible for that page.
2589			 */
2590			ret = bio_readpage_error(bio, offset, page, start, end,
2591						 mirror);
2592			if (ret == 0) {
2593				uptodate = !bio->bi_error;
2594				offset += len;
2595				continue;
2596			}
2597		}
2598readpage_ok:
2599		if (likely(uptodate)) {
2600			loff_t i_size = i_size_read(inode);
2601			pgoff_t end_index = i_size >> PAGE_SHIFT;
2602			unsigned off;
2603
2604			/* Zero out the end if this page straddles i_size */
2605			off = i_size & (PAGE_SIZE-1);
2606			if (page->index == end_index && off)
2607				zero_user_segment(page, off, PAGE_SIZE);
2608			SetPageUptodate(page);
2609		} else {
2610			ClearPageUptodate(page);
2611			SetPageError(page);
2612		}
2613		unlock_page(page);
2614		offset += len;
2615
2616		if (unlikely(!uptodate)) {
2617			if (extent_len) {
2618				endio_readpage_release_extent(tree,
2619							      extent_start,
2620							      extent_len, 1);
2621				extent_start = 0;
2622				extent_len = 0;
2623			}
2624			endio_readpage_release_extent(tree, start,
2625						      end - start + 1, 0);
2626		} else if (!extent_len) {
2627			extent_start = start;
2628			extent_len = end + 1 - start;
2629		} else if (extent_start + extent_len == start) {
2630			extent_len += end + 1 - start;
2631		} else {
2632			endio_readpage_release_extent(tree, extent_start,
2633						      extent_len, uptodate);
2634			extent_start = start;
2635			extent_len = end + 1 - start;
2636		}
2637	}
2638
2639	if (extent_len)
2640		endio_readpage_release_extent(tree, extent_start, extent_len,
2641					      uptodate);
2642	if (io_bio->end_io)
2643		io_bio->end_io(io_bio, bio->bi_error);
2644	bio_put(bio);
2645}
2646
2647/*
2648 * this allocates from the btrfs_bioset.  We're returning a bio right now
2649 * but you can call btrfs_io_bio for the appropriate container_of magic
2650 */
2651struct bio *
2652btrfs_bio_alloc(struct block_device *bdev, u64 first_sector, int nr_vecs,
2653		gfp_t gfp_flags)
2654{
2655	struct btrfs_io_bio *btrfs_bio;
2656	struct bio *bio;
2657
2658	bio = bio_alloc_bioset(gfp_flags, nr_vecs, btrfs_bioset);
 
 
2659
2660	if (bio == NULL && (current->flags & PF_MEMALLOC)) {
2661		while (!bio && (nr_vecs /= 2)) {
2662			bio = bio_alloc_bioset(gfp_flags,
2663					       nr_vecs, btrfs_bioset);
2664		}
2665	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2666
2667	if (bio) {
2668		bio->bi_bdev = bdev;
2669		bio->bi_iter.bi_sector = first_sector;
2670		btrfs_bio = btrfs_io_bio(bio);
2671		btrfs_bio->csum = NULL;
2672		btrfs_bio->csum_allocated = NULL;
2673		btrfs_bio->end_io = NULL;
2674	}
2675	return bio;
2676}
2677
2678struct bio *btrfs_bio_clone(struct bio *bio, gfp_t gfp_mask)
 
 
2679{
2680	struct btrfs_io_bio *btrfs_bio;
2681	struct bio *new;
2682
2683	new = bio_clone_bioset(bio, gfp_mask, btrfs_bioset);
2684	if (new) {
2685		btrfs_bio = btrfs_io_bio(new);
2686		btrfs_bio->csum = NULL;
2687		btrfs_bio->csum_allocated = NULL;
2688		btrfs_bio->end_io = NULL;
2689
2690#ifdef CONFIG_BLK_CGROUP
2691		/* FIXME, put this into bio_clone_bioset */
2692		if (bio->bi_css)
2693			bio_associate_blkcg(new, bio->bi_css);
2694#endif
 
 
 
 
 
 
 
 
 
 
2695	}
2696	return new;
2697}
2698
2699/* this also allocates from the btrfs_bioset */
2700struct bio *btrfs_io_bio_alloc(gfp_t gfp_mask, unsigned int nr_iovecs)
2701{
2702	struct btrfs_io_bio *btrfs_bio;
2703	struct bio *bio;
2704
2705	bio = bio_alloc_bioset(gfp_mask, nr_iovecs, btrfs_bioset);
2706	if (bio) {
2707		btrfs_bio = btrfs_io_bio(bio);
2708		btrfs_bio->csum = NULL;
2709		btrfs_bio->csum_allocated = NULL;
2710		btrfs_bio->end_io = NULL;
2711	}
2712	return bio;
2713}
2714
2715
2716static int __must_check submit_one_bio(int rw, struct bio *bio,
2717				       int mirror_num, unsigned long bio_flags)
2718{
2719	int ret = 0;
2720	struct bio_vec *bvec = bio->bi_io_vec + bio->bi_vcnt - 1;
2721	struct page *page = bvec->bv_page;
2722	struct extent_io_tree *tree = bio->bi_private;
2723	u64 start;
2724
2725	start = page_offset(page) + bvec->bv_offset;
2726
2727	bio->bi_private = NULL;
2728
2729	bio_get(bio);
2730
2731	if (tree->ops && tree->ops->submit_bio_hook)
2732		ret = tree->ops->submit_bio_hook(page->mapping->host, rw, bio,
2733					   mirror_num, bio_flags, start);
2734	else
2735		btrfsic_submit_bio(rw, bio);
2736
2737	bio_put(bio);
2738	return ret;
2739}
2740
2741static int merge_bio(int rw, struct extent_io_tree *tree, struct page *page,
2742		     unsigned long offset, size_t size, struct bio *bio,
2743		     unsigned long bio_flags)
2744{
2745	int ret = 0;
2746	if (tree->ops && tree->ops->merge_bio_hook)
2747		ret = tree->ops->merge_bio_hook(rw, page, offset, size, bio,
2748						bio_flags);
2749	BUG_ON(ret < 0);
2750	return ret;
2751
2752}
2753
2754static int submit_extent_page(int rw, struct extent_io_tree *tree,
2755			      struct writeback_control *wbc,
2756			      struct page *page, sector_t sector,
2757			      size_t size, unsigned long offset,
2758			      struct block_device *bdev,
2759			      struct bio **bio_ret,
2760			      unsigned long max_pages,
2761			      bio_end_io_t end_io_func,
2762			      int mirror_num,
2763			      unsigned long prev_bio_flags,
2764			      unsigned long bio_flags,
2765			      bool force_bio_submit)
2766{
2767	int ret = 0;
2768	struct bio *bio;
2769	int contig = 0;
2770	int old_compressed = prev_bio_flags & EXTENT_BIO_COMPRESSED;
2771	size_t page_size = min_t(size_t, size, PAGE_SIZE);
2772
2773	if (bio_ret && *bio_ret) {
2774		bio = *bio_ret;
2775		if (old_compressed)
2776			contig = bio->bi_iter.bi_sector == sector;
2777		else
2778			contig = bio_end_sector(bio) == sector;
2779
2780		if (prev_bio_flags != bio_flags || !contig ||
2781		    force_bio_submit ||
2782		    merge_bio(rw, tree, page, offset, page_size, bio, bio_flags) ||
2783		    bio_add_page(bio, page, page_size, offset) < page_size) {
2784			ret = submit_one_bio(rw, bio, mirror_num,
2785					     prev_bio_flags);
2786			if (ret < 0) {
2787				*bio_ret = NULL;
2788				return ret;
2789			}
2790			bio = NULL;
2791		} else {
2792			if (wbc)
2793				wbc_account_io(wbc, page, page_size);
2794			return 0;
2795		}
2796	}
2797
2798	bio = btrfs_bio_alloc(bdev, sector, BIO_MAX_PAGES,
2799			GFP_NOFS | __GFP_HIGH);
2800	if (!bio)
2801		return -ENOMEM;
2802
2803	bio_add_page(bio, page, page_size, offset);
2804	bio->bi_end_io = end_io_func;
2805	bio->bi_private = tree;
2806	if (wbc) {
2807		wbc_init_bio(wbc, bio);
2808		wbc_account_io(wbc, page, page_size);
2809	}
2810
2811	if (bio_ret)
2812		*bio_ret = bio;
2813	else
2814		ret = submit_one_bio(rw, bio, mirror_num, bio_flags);
2815
2816	return ret;
 
2817}
2818
2819static void attach_extent_buffer_page(struct extent_buffer *eb,
2820				      struct page *page)
2821{
2822	if (!PagePrivate(page)) {
2823		SetPagePrivate(page);
2824		get_page(page);
2825		set_page_private(page, (unsigned long)eb);
2826	} else {
2827		WARN_ON(page->private != (unsigned long)eb);
2828	}
2829}
2830
2831void set_page_extent_mapped(struct page *page)
2832{
2833	if (!PagePrivate(page)) {
2834		SetPagePrivate(page);
2835		get_page(page);
2836		set_page_private(page, EXTENT_PAGE_PRIVATE);
2837	}
 
 
 
2838}
2839
2840static struct extent_map *
2841__get_extent_map(struct inode *inode, struct page *page, size_t pg_offset,
2842		 u64 start, u64 len, get_extent_t *get_extent,
2843		 struct extent_map **em_cached)
2844{
2845	struct extent_map *em;
2846
2847	if (em_cached && *em_cached) {
 
 
2848		em = *em_cached;
2849		if (extent_map_in_tree(em) && start >= em->start &&
2850		    start < extent_map_end(em)) {
2851			atomic_inc(&em->refs);
2852			return em;
2853		}
2854
2855		free_extent_map(em);
2856		*em_cached = NULL;
2857	}
2858
2859	em = get_extent(inode, page, pg_offset, start, len, 0);
2860	if (em_cached && !IS_ERR_OR_NULL(em)) {
2861		BUG_ON(*em_cached);
2862		atomic_inc(&em->refs);
2863		*em_cached = em;
2864	}
2865	return em;
2866}
2867/*
2868 * basic readpage implementation.  Locked extent state structs are inserted
2869 * into the tree that are removed when the IO is done (by the end_io
2870 * handlers)
2871 * XXX JDM: This needs looking at to ensure proper page locking
 
2872 */
2873static int __do_readpage(struct extent_io_tree *tree,
2874			 struct page *page,
2875			 get_extent_t *get_extent,
2876			 struct extent_map **em_cached,
2877			 struct bio **bio, int mirror_num,
2878			 unsigned long *bio_flags, int rw,
2879			 u64 *prev_em_start)
2880{
2881	struct inode *inode = page->mapping->host;
 
2882	u64 start = page_offset(page);
2883	u64 page_end = start + PAGE_SIZE - 1;
2884	u64 end;
2885	u64 cur = start;
2886	u64 extent_offset;
2887	u64 last_byte = i_size_read(inode);
2888	u64 block_start;
2889	u64 cur_end;
2890	sector_t sector;
2891	struct extent_map *em;
2892	struct block_device *bdev;
2893	int ret;
2894	int nr = 0;
2895	size_t pg_offset = 0;
2896	size_t iosize;
2897	size_t disk_io_size;
2898	size_t blocksize = inode->i_sb->s_blocksize;
2899	unsigned long this_bio_flag = 0;
2900
2901	set_page_extent_mapped(page);
2902
2903	end = page_end;
2904	if (!PageUptodate(page)) {
2905		if (cleancache_get_page(page) == 0) {
2906			BUG_ON(blocksize != PAGE_SIZE);
2907			unlock_extent(tree, start, end);
2908			goto out;
2909		}
2910	}
2911
2912	if (page->index == last_byte >> PAGE_SHIFT) {
2913		char *userpage;
2914		size_t zero_offset = last_byte & (PAGE_SIZE - 1);
2915
2916		if (zero_offset) {
2917			iosize = PAGE_SIZE - zero_offset;
2918			userpage = kmap_atomic(page);
2919			memset(userpage + zero_offset, 0, iosize);
2920			flush_dcache_page(page);
2921			kunmap_atomic(userpage);
2922		}
2923	}
 
 
2924	while (cur <= end) {
2925		unsigned long pnr = (last_byte >> PAGE_SHIFT) + 1;
2926		bool force_bio_submit = false;
 
2927
 
2928		if (cur >= last_byte) {
2929			char *userpage;
2930			struct extent_state *cached = NULL;
2931
2932			iosize = PAGE_SIZE - pg_offset;
2933			userpage = kmap_atomic(page);
2934			memset(userpage + pg_offset, 0, iosize);
2935			flush_dcache_page(page);
2936			kunmap_atomic(userpage);
2937			set_extent_uptodate(tree, cur, cur + iosize - 1,
2938					    &cached, GFP_NOFS);
2939			unlock_extent_cached(tree, cur,
2940					     cur + iosize - 1,
2941					     &cached, GFP_NOFS);
2942			break;
2943		}
2944		em = __get_extent_map(inode, page, pg_offset, cur,
2945				      end - cur + 1, get_extent, em_cached);
2946		if (IS_ERR_OR_NULL(em)) {
2947			SetPageError(page);
2948			unlock_extent(tree, cur, end);
2949			break;
2950		}
2951		extent_offset = cur - em->start;
2952		BUG_ON(extent_map_end(em) <= cur);
2953		BUG_ON(end < cur);
2954
2955		if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags)) {
2956			this_bio_flag |= EXTENT_BIO_COMPRESSED;
2957			extent_set_compress_type(&this_bio_flag,
2958						 em->compress_type);
2959		}
2960
2961		iosize = min(extent_map_end(em) - cur, end - cur + 1);
2962		cur_end = min(extent_map_end(em) - 1, end);
2963		iosize = ALIGN(iosize, blocksize);
2964		if (this_bio_flag & EXTENT_BIO_COMPRESSED) {
2965			disk_io_size = em->block_len;
2966			sector = em->block_start >> 9;
2967		} else {
2968			sector = (em->block_start + extent_offset) >> 9;
2969			disk_io_size = iosize;
2970		}
2971		bdev = em->bdev;
2972		block_start = em->block_start;
2973		if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
2974			block_start = EXTENT_MAP_HOLE;
2975
2976		/*
2977		 * If we have a file range that points to a compressed extent
2978		 * and it's followed by a consecutive file range that points to
2979		 * to the same compressed extent (possibly with a different
2980		 * offset and/or length, so it either points to the whole extent
2981		 * or only part of it), we must make sure we do not submit a
2982		 * single bio to populate the pages for the 2 ranges because
2983		 * this makes the compressed extent read zero out the pages
2984		 * belonging to the 2nd range. Imagine the following scenario:
2985		 *
2986		 *  File layout
2987		 *  [0 - 8K]                     [8K - 24K]
2988		 *    |                               |
2989		 *    |                               |
2990		 * points to extent X,         points to extent X,
2991		 * offset 4K, length of 8K     offset 0, length 16K
2992		 *
2993		 * [extent X, compressed length = 4K uncompressed length = 16K]
2994		 *
2995		 * If the bio to read the compressed extent covers both ranges,
2996		 * it will decompress extent X into the pages belonging to the
2997		 * first range and then it will stop, zeroing out the remaining
2998		 * pages that belong to the other range that points to extent X.
2999		 * So here we make sure we submit 2 bios, one for the first
3000		 * range and another one for the third range. Both will target
3001		 * the same physical extent from disk, but we can't currently
3002		 * make the compressed bio endio callback populate the pages
3003		 * for both ranges because each compressed bio is tightly
3004		 * coupled with a single extent map, and each range can have
3005		 * an extent map with a different offset value relative to the
3006		 * uncompressed data of our extent and different lengths. This
3007		 * is a corner case so we prioritize correctness over
3008		 * non-optimal behavior (submitting 2 bios for the same extent).
3009		 */
3010		if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags) &&
3011		    prev_em_start && *prev_em_start != (u64)-1 &&
3012		    *prev_em_start != em->orig_start)
3013			force_bio_submit = true;
3014
3015		if (prev_em_start)
3016			*prev_em_start = em->orig_start;
3017
3018		free_extent_map(em);
3019		em = NULL;
3020
3021		/* we've found a hole, just zero and go on */
3022		if (block_start == EXTENT_MAP_HOLE) {
3023			char *userpage;
3024			struct extent_state *cached = NULL;
3025
3026			userpage = kmap_atomic(page);
3027			memset(userpage + pg_offset, 0, iosize);
3028			flush_dcache_page(page);
3029			kunmap_atomic(userpage);
3030
3031			set_extent_uptodate(tree, cur, cur + iosize - 1,
3032					    &cached, GFP_NOFS);
3033			unlock_extent_cached(tree, cur,
3034					     cur + iosize - 1,
3035					     &cached, GFP_NOFS);
3036			cur = cur + iosize;
3037			pg_offset += iosize;
3038			continue;
3039		}
3040		/* the get_extent function already copied into the page */
3041		if (test_range_bit(tree, cur, cur_end,
3042				   EXTENT_UPTODATE, 1, NULL)) {
3043			check_page_uptodate(tree, page);
3044			unlock_extent(tree, cur, cur + iosize - 1);
3045			cur = cur + iosize;
3046			pg_offset += iosize;
3047			continue;
3048		}
3049		/* we have an inline extent but it didn't get marked up
3050		 * to date.  Error out
3051		 */
3052		if (block_start == EXTENT_MAP_INLINE) {
3053			SetPageError(page);
3054			unlock_extent(tree, cur, cur + iosize - 1);
3055			cur = cur + iosize;
3056			pg_offset += iosize;
3057			continue;
3058		}
3059
3060		pnr -= page->index;
3061		ret = submit_extent_page(rw, tree, NULL, page,
3062					 sector, disk_io_size, pg_offset,
3063					 bdev, bio, pnr,
3064					 end_bio_extent_readpage, mirror_num,
3065					 *bio_flags,
3066					 this_bio_flag,
3067					 force_bio_submit);
3068		if (!ret) {
3069			nr++;
3070			*bio_flags = this_bio_flag;
3071		} else {
3072			SetPageError(page);
3073			unlock_extent(tree, cur, cur + iosize - 1);
3074		}
 
 
 
 
 
3075		cur = cur + iosize;
3076		pg_offset += iosize;
3077	}
3078out:
3079	if (!nr) {
3080		if (!PageError(page))
3081			SetPageUptodate(page);
3082		unlock_page(page);
3083	}
3084	return 0;
3085}
3086
3087static inline void __do_contiguous_readpages(struct extent_io_tree *tree,
3088					     struct page *pages[], int nr_pages,
3089					     u64 start, u64 end,
3090					     get_extent_t *get_extent,
3091					     struct extent_map **em_cached,
3092					     struct bio **bio, int mirror_num,
3093					     unsigned long *bio_flags, int rw,
3094					     u64 *prev_em_start)
3095{
3096	struct inode *inode;
3097	struct btrfs_ordered_extent *ordered;
3098	int index;
3099
3100	inode = pages[0]->mapping->host;
3101	while (1) {
3102		lock_extent(tree, start, end);
3103		ordered = btrfs_lookup_ordered_range(inode, start,
3104						     end - start + 1);
3105		if (!ordered)
3106			break;
3107		unlock_extent(tree, start, end);
3108		btrfs_start_ordered_extent(inode, ordered, 1);
3109		btrfs_put_ordered_extent(ordered);
3110	}
3111
3112	for (index = 0; index < nr_pages; index++) {
3113		__do_readpage(tree, pages[index], get_extent, em_cached, bio,
3114			      mirror_num, bio_flags, rw, prev_em_start);
3115		put_page(pages[index]);
3116	}
3117}
3118
3119static void __extent_readpages(struct extent_io_tree *tree,
3120			       struct page *pages[],
3121			       int nr_pages, get_extent_t *get_extent,
3122			       struct extent_map **em_cached,
3123			       struct bio **bio, int mirror_num,
3124			       unsigned long *bio_flags, int rw,
3125			       u64 *prev_em_start)
3126{
3127	u64 start = 0;
3128	u64 end = 0;
3129	u64 page_start;
3130	int index;
3131	int first_index = 0;
3132
3133	for (index = 0; index < nr_pages; index++) {
3134		page_start = page_offset(pages[index]);
3135		if (!end) {
3136			start = page_start;
3137			end = start + PAGE_SIZE - 1;
3138			first_index = index;
3139		} else if (end + 1 == page_start) {
3140			end += PAGE_SIZE;
3141		} else {
3142			__do_contiguous_readpages(tree, &pages[first_index],
3143						  index - first_index, start,
3144						  end, get_extent, em_cached,
3145						  bio, mirror_num, bio_flags,
3146						  rw, prev_em_start);
3147			start = page_start;
3148			end = start + PAGE_SIZE - 1;
3149			first_index = index;
3150		}
3151	}
3152
3153	if (end)
3154		__do_contiguous_readpages(tree, &pages[first_index],
3155					  index - first_index, start,
3156					  end, get_extent, em_cached, bio,
3157					  mirror_num, bio_flags, rw,
3158					  prev_em_start);
3159}
3160
3161static int __extent_read_full_page(struct extent_io_tree *tree,
3162				   struct page *page,
3163				   get_extent_t *get_extent,
3164				   struct bio **bio, int mirror_num,
3165				   unsigned long *bio_flags, int rw)
3166{
3167	struct inode *inode = page->mapping->host;
3168	struct btrfs_ordered_extent *ordered;
3169	u64 start = page_offset(page);
3170	u64 end = start + PAGE_SIZE - 1;
 
 
3171	int ret;
3172
3173	while (1) {
3174		lock_extent(tree, start, end);
3175		ordered = btrfs_lookup_ordered_range(inode, start,
3176						PAGE_SIZE);
3177		if (!ordered)
3178			break;
3179		unlock_extent(tree, start, end);
3180		btrfs_start_ordered_extent(inode, ordered, 1);
3181		btrfs_put_ordered_extent(ordered);
3182	}
3183
3184	ret = __do_readpage(tree, page, get_extent, NULL, bio, mirror_num,
3185			    bio_flags, rw, NULL);
 
 
 
3186	return ret;
3187}
3188
3189int extent_read_full_page(struct extent_io_tree *tree, struct page *page,
3190			    get_extent_t *get_extent, int mirror_num)
 
 
 
3191{
3192	struct bio *bio = NULL;
3193	unsigned long bio_flags = 0;
3194	int ret;
3195
3196	ret = __extent_read_full_page(tree, page, get_extent, &bio, mirror_num,
3197				      &bio_flags, READ);
3198	if (bio)
3199		ret = submit_one_bio(READ, bio, mirror_num, bio_flags);
3200	return ret;
3201}
3202
3203static noinline void update_nr_written(struct page *page,
3204				      struct writeback_control *wbc,
3205				      unsigned long nr_written)
3206{
3207	wbc->nr_to_write -= nr_written;
3208	if (wbc->range_cyclic || (wbc->nr_to_write > 0 &&
3209	    wbc->range_start == 0 && wbc->range_end == LLONG_MAX))
3210		page->mapping->writeback_index = page->index + nr_written;
3211}
3212
3213/*
3214 * helper for __extent_writepage, doing all of the delayed allocation setup.
3215 *
3216 * This returns 1 if our fill_delalloc function did all the work required
3217 * to write the page (copy into inline extent).  In this case the IO has
3218 * been started and the page is already unlocked.
3219 *
3220 * This returns 0 if all went well (page still locked)
3221 * This returns < 0 if there were errors (page still locked)
3222 */
3223static noinline_for_stack int writepage_delalloc(struct inode *inode,
3224			      struct page *page, struct writeback_control *wbc,
3225			      struct extent_page_data *epd,
3226			      u64 delalloc_start,
3227			      unsigned long *nr_written)
3228{
3229	struct extent_io_tree *tree = epd->tree;
3230	u64 page_end = delalloc_start + PAGE_SIZE - 1;
3231	u64 nr_delalloc;
3232	u64 delalloc_to_write = 0;
3233	u64 delalloc_end = 0;
3234	int ret;
3235	int page_started = 0;
3236
3237	if (epd->extent_locked || !tree->ops || !tree->ops->fill_delalloc)
3238		return 0;
3239
3240	while (delalloc_end < page_end) {
3241		nr_delalloc = find_lock_delalloc_range(inode, tree,
3242					       page,
3243					       &delalloc_start,
3244					       &delalloc_end,
3245					       BTRFS_MAX_EXTENT_SIZE);
3246		if (nr_delalloc == 0) {
3247			delalloc_start = delalloc_end + 1;
3248			continue;
3249		}
3250		ret = tree->ops->fill_delalloc(inode, page,
3251					       delalloc_start,
3252					       delalloc_end,
3253					       &page_started,
3254					       nr_written);
3255		/* File system has been set read-only */
3256		if (ret) {
3257			SetPageError(page);
3258			/* fill_delalloc should be return < 0 for error
3259			 * but just in case, we use > 0 here meaning the
3260			 * IO is started, so we don't want to return > 0
3261			 * unless things are going well.
3262			 */
3263			ret = ret < 0 ? ret : -EIO;
3264			goto done;
3265		}
3266		/*
3267		 * delalloc_end is already one less than the total length, so
3268		 * we don't subtract one from PAGE_SIZE
3269		 */
3270		delalloc_to_write += (delalloc_end - delalloc_start +
3271				      PAGE_SIZE) >> PAGE_SHIFT;
3272		delalloc_start = delalloc_end + 1;
3273	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3274	if (wbc->nr_to_write < delalloc_to_write) {
3275		int thresh = 8192;
3276
3277		if (delalloc_to_write < thresh * 2)
3278			thresh = delalloc_to_write;
3279		wbc->nr_to_write = min_t(u64, delalloc_to_write,
3280					 thresh);
3281	}
3282
3283	/* did the fill delalloc function already unlock and start
3284	 * the IO?
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3285	 */
3286	if (page_started) {
3287		/*
3288		 * we've unlocked the page, so we can't update
3289		 * the mapping's writeback index, just update
3290		 * nr_to_write.
3291		 */
3292		wbc->nr_to_write -= *nr_written;
3293		return 1;
3294	}
3295
3296	ret = 0;
 
3297
3298done:
3299	return ret;
 
 
 
 
 
 
 
 
 
3300}
3301
3302/*
3303 * helper for __extent_writepage.  This calls the writepage start hooks,
3304 * and does the loop to map the page into extents and bios.
3305 *
3306 * We return 1 if the IO is started and the page is unlocked,
3307 * 0 if all went well (page still locked)
3308 * < 0 if there were errors (page still locked)
3309 */
3310static noinline_for_stack int __extent_writepage_io(struct inode *inode,
3311				 struct page *page,
3312				 struct writeback_control *wbc,
3313				 struct extent_page_data *epd,
3314				 loff_t i_size,
3315				 unsigned long nr_written,
3316				 int write_flags, int *nr_ret)
3317{
3318	struct extent_io_tree *tree = epd->tree;
3319	u64 start = page_offset(page);
3320	u64 page_end = start + PAGE_SIZE - 1;
3321	u64 end;
3322	u64 cur = start;
3323	u64 extent_offset;
3324	u64 block_start;
3325	u64 iosize;
3326	sector_t sector;
3327	struct extent_state *cached_state = NULL;
3328	struct extent_map *em;
3329	struct block_device *bdev;
3330	size_t pg_offset = 0;
3331	size_t blocksize;
3332	int ret = 0;
3333	int nr = 0;
3334	bool compressed;
3335
3336	if (tree->ops && tree->ops->writepage_start_hook) {
3337		ret = tree->ops->writepage_start_hook(page, start,
3338						      page_end);
3339		if (ret) {
3340			/* Fixup worker will requeue */
3341			if (ret == -EBUSY)
3342				wbc->pages_skipped++;
3343			else
3344				redirty_page_for_writepage(wbc, page);
3345
3346			update_nr_written(page, wbc, nr_written);
3347			unlock_page(page);
3348			ret = 1;
3349			goto done_unlocked;
3350		}
3351	}
3352
3353	/*
3354	 * we don't want to touch the inode after unlocking the page,
3355	 * so we update the mapping writeback index now
3356	 */
3357	update_nr_written(page, wbc, nr_written + 1);
3358
3359	end = page_end;
3360	if (i_size <= start) {
3361		if (tree->ops && tree->ops->writepage_end_io_hook)
3362			tree->ops->writepage_end_io_hook(page, start,
3363							 page_end, NULL, 1);
3364		goto done;
3365	}
3366
3367	blocksize = inode->i_sb->s_blocksize;
3368
3369	while (cur <= end) {
 
 
3370		u64 em_end;
 
 
 
 
3371		if (cur >= i_size) {
3372			if (tree->ops && tree->ops->writepage_end_io_hook)
3373				tree->ops->writepage_end_io_hook(page, cur,
3374							 page_end, NULL, 1);
 
 
 
 
 
 
 
 
3375			break;
3376		}
3377		em = epd->get_extent(inode, page, pg_offset, cur,
3378				     end - cur + 1, 1);
3379		if (IS_ERR_OR_NULL(em)) {
3380			SetPageError(page);
 
 
 
 
 
 
3381			ret = PTR_ERR_OR_ZERO(em);
3382			break;
3383		}
3384
3385		extent_offset = cur - em->start;
3386		em_end = extent_map_end(em);
3387		BUG_ON(em_end <= cur);
3388		BUG_ON(end < cur);
3389		iosize = min(em_end - cur, end - cur + 1);
3390		iosize = ALIGN(iosize, blocksize);
3391		sector = (em->block_start + extent_offset) >> 9;
3392		bdev = em->bdev;
3393		block_start = em->block_start;
3394		compressed = test_bit(EXTENT_FLAG_COMPRESSED, &em->flags);
3395		free_extent_map(em);
3396		em = NULL;
 
 
3397
3398		/*
3399		 * compressed and inline extents are written through other
3400		 * paths in the FS
3401		 */
3402		if (compressed || block_start == EXTENT_MAP_HOLE ||
3403		    block_start == EXTENT_MAP_INLINE) {
3404			/*
3405			 * end_io notification does not happen here for
3406			 * compressed extents
3407			 */
3408			if (!compressed && tree->ops &&
3409			    tree->ops->writepage_end_io_hook)
3410				tree->ops->writepage_end_io_hook(page, cur,
3411							 cur + iosize - 1,
3412							 NULL, 1);
3413			else if (compressed) {
3414				/* we don't want to end_page_writeback on
3415				 * a compressed extent.  this happens
3416				 * elsewhere
3417				 */
3418				nr++;
3419			}
3420
3421			cur += iosize;
3422			pg_offset += iosize;
3423			continue;
 
 
3424		}
3425
3426		if (tree->ops && tree->ops->writepage_io_hook) {
3427			ret = tree->ops->writepage_io_hook(page, cur,
3428						cur + iosize - 1);
3429		} else {
3430			ret = 0;
3431		}
3432		if (ret) {
3433			SetPageError(page);
3434		} else {
3435			unsigned long max_nr = (i_size >> PAGE_SHIFT) + 1;
3436
3437			set_range_writeback(tree, cur, cur + iosize - 1);
3438			if (!PageWriteback(page)) {
3439				btrfs_err(BTRFS_I(inode)->root->fs_info,
3440					   "page %lu not writeback, cur %llu end %llu",
3441				       page->index, cur, end);
3442			}
3443
3444			ret = submit_extent_page(write_flags, tree, wbc, page,
3445						 sector, iosize, pg_offset,
3446						 bdev, &epd->bio, max_nr,
3447						 end_bio_extent_writepage,
3448						 0, 0, 0, false);
3449			if (ret)
3450				SetPageError(page);
3451		}
3452		cur = cur + iosize;
3453		pg_offset += iosize;
3454		nr++;
3455	}
3456done:
3457	*nr_ret = nr;
3458
3459done_unlocked:
 
 
3460
3461	/* drop our reference on any cached states */
3462	free_extent_state(cached_state);
 
 
 
 
3463	return ret;
3464}
3465
3466/*
3467 * the writepage semantics are similar to regular writepage.  extent
3468 * records are inserted to lock ranges in the tree, and as dirty areas
3469 * are found, they are marked writeback.  Then the lock bits are removed
3470 * and the end_io handler clears the writeback ranges
 
 
 
3471 */
3472static int __extent_writepage(struct page *page, struct writeback_control *wbc,
3473			      void *data)
3474{
 
3475	struct inode *inode = page->mapping->host;
3476	struct extent_page_data *epd = data;
3477	u64 start = page_offset(page);
3478	u64 page_end = start + PAGE_SIZE - 1;
3479	int ret;
3480	int nr = 0;
3481	size_t pg_offset = 0;
3482	loff_t i_size = i_size_read(inode);
3483	unsigned long end_index = i_size >> PAGE_SHIFT;
3484	int write_flags;
3485	unsigned long nr_written = 0;
3486
3487	if (wbc->sync_mode == WB_SYNC_ALL)
3488		write_flags = WRITE_SYNC;
3489	else
3490		write_flags = WRITE;
3491
3492	trace___extent_writepage(page, inode, wbc);
3493
3494	WARN_ON(!PageLocked(page));
3495
3496	ClearPageError(page);
3497
3498	pg_offset = i_size & (PAGE_SIZE - 1);
3499	if (page->index > end_index ||
3500	   (page->index == end_index && !pg_offset)) {
3501		page->mapping->a_ops->invalidatepage(page, 0, PAGE_SIZE);
3502		unlock_page(page);
3503		return 0;
3504	}
3505
3506	if (page->index == end_index) {
3507		char *userpage;
3508
3509		userpage = kmap_atomic(page);
3510		memset(userpage + pg_offset, 0,
3511		       PAGE_SIZE - pg_offset);
3512		kunmap_atomic(userpage);
3513		flush_dcache_page(page);
3514	}
3515
3516	pg_offset = 0;
3517
3518	set_page_extent_mapped(page);
 
 
3519
3520	ret = writepage_delalloc(inode, page, wbc, epd, start, &nr_written);
3521	if (ret == 1)
3522		goto done_unlocked;
3523	if (ret)
3524		goto done;
3525
3526	ret = __extent_writepage_io(inode, page, wbc, epd,
3527				    i_size, nr_written, write_flags, &nr);
3528	if (ret == 1)
3529		goto done_unlocked;
 
 
3530
3531done:
3532	if (nr == 0) {
3533		/* make sure the mapping tag for page dirty gets cleared */
3534		set_page_writeback(page);
3535		end_page_writeback(page);
3536	}
3537	if (PageError(page)) {
3538		ret = ret < 0 ? ret : -EIO;
3539		end_extent_writepage(page, ret, start, page_end);
 
3540	}
3541	unlock_page(page);
 
3542	return ret;
3543
3544done_unlocked:
3545	return 0;
3546}
3547
3548void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
3549{
3550	wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
3551		       TASK_UNINTERRUPTIBLE);
3552}
3553
3554static noinline_for_stack int
3555lock_extent_buffer_for_io(struct extent_buffer *eb,
3556			  struct btrfs_fs_info *fs_info,
3557			  struct extent_page_data *epd)
 
 
 
 
 
3558{
3559	unsigned long i, num_pages;
3560	int flush = 0;
3561	int ret = 0;
3562
3563	if (!btrfs_try_tree_write_lock(eb)) {
3564		flush = 1;
3565		flush_write_bio(epd);
3566		btrfs_tree_lock(eb);
3567	}
3568
3569	if (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
3570		btrfs_tree_unlock(eb);
3571		if (!epd->sync_io)
3572			return 0;
3573		if (!flush) {
3574			flush_write_bio(epd);
3575			flush = 1;
3576		}
3577		while (1) {
3578			wait_on_extent_buffer_writeback(eb);
3579			btrfs_tree_lock(eb);
3580			if (!test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags))
3581				break;
3582			btrfs_tree_unlock(eb);
3583		}
3584	}
3585
3586	/*
3587	 * We need to do this to prevent races in people who check if the eb is
3588	 * under IO since we can end up having no IO bits set for a short period
3589	 * of time.
3590	 */
3591	spin_lock(&eb->refs_lock);
3592	if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3593		set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3594		spin_unlock(&eb->refs_lock);
3595		btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
3596		__percpu_counter_add(&fs_info->dirty_metadata_bytes,
3597				     -eb->len,
3598				     fs_info->dirty_metadata_batch);
3599		ret = 1;
3600	} else {
3601		spin_unlock(&eb->refs_lock);
3602	}
3603
3604	btrfs_tree_unlock(eb);
3605
3606	if (!ret)
3607		return ret;
3608
3609	num_pages = num_extent_pages(eb->start, eb->len);
3610	for (i = 0; i < num_pages; i++) {
3611		struct page *p = eb->pages[i];
3612
3613		if (!trylock_page(p)) {
3614			if (!flush) {
3615				flush_write_bio(epd);
3616				flush = 1;
3617			}
3618			lock_page(p);
3619		}
3620	}
3621
3622	return ret;
3623}
3624
3625static void end_extent_buffer_writeback(struct extent_buffer *eb)
3626{
3627	clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
3628	smp_mb__after_atomic();
3629	wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
3630}
3631
3632static void set_btree_ioerr(struct page *page)
3633{
3634	struct extent_buffer *eb = (struct extent_buffer *)page->private;
3635	struct btrfs_inode *btree_ino = BTRFS_I(eb->fs_info->btree_inode);
3636
3637	SetPageError(page);
3638	if (test_and_set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
3639		return;
 
 
 
 
 
 
 
 
 
 
3640
3641	/*
3642	 * If writeback for a btree extent that doesn't belong to a log tree
3643	 * failed, increment the counter transaction->eb_write_errors.
3644	 * We do this because while the transaction is running and before it's
3645	 * committing (when we call filemap_fdata[write|wait]_range against
3646	 * the btree inode), we might have
3647	 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
3648	 * returns an error or an error happens during writeback, when we're
3649	 * committing the transaction we wouldn't know about it, since the pages
3650	 * can be no longer dirty nor marked anymore for writeback (if a
3651	 * subsequent modification to the extent buffer didn't happen before the
3652	 * transaction commit), which makes filemap_fdata[write|wait]_range not
3653	 * able to find the pages tagged with SetPageError at transaction
3654	 * commit time. So if this happens we must abort the transaction,
3655	 * otherwise we commit a super block with btree roots that point to
3656	 * btree nodes/leafs whose content on disk is invalid - either garbage
3657	 * or the content of some node/leaf from a past generation that got
3658	 * cowed or deleted and is no longer valid.
3659	 *
3660	 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
3661	 * not be enough - we need to distinguish between log tree extents vs
3662	 * non-log tree extents, and the next filemap_fdatawait_range() call
3663	 * will catch and clear such errors in the mapping - and that call might
3664	 * be from a log sync and not from a transaction commit. Also, checking
3665	 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
3666	 * not done and would not be reliable - the eb might have been released
3667	 * from memory and reading it back again means that flag would not be
3668	 * set (since it's a runtime flag, not persisted on disk).
3669	 *
3670	 * Using the flags below in the btree inode also makes us achieve the
3671	 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
3672	 * writeback for all dirty pages and before filemap_fdatawait_range()
3673	 * is called, the writeback for all dirty pages had already finished
3674	 * with errors - because we were not using AS_EIO/AS_ENOSPC,
3675	 * filemap_fdatawait_range() would return success, as it could not know
3676	 * that writeback errors happened (the pages were no longer tagged for
3677	 * writeback).
3678	 */
3679	switch (eb->log_index) {
3680	case -1:
3681		set_bit(BTRFS_INODE_BTREE_ERR, &btree_ino->runtime_flags);
3682		break;
3683	case 0:
3684		set_bit(BTRFS_INODE_BTREE_LOG1_ERR, &btree_ino->runtime_flags);
3685		break;
3686	case 1:
3687		set_bit(BTRFS_INODE_BTREE_LOG2_ERR, &btree_ino->runtime_flags);
3688		break;
3689	default:
3690		BUG(); /* unexpected, logic error */
3691	}
3692}
3693
3694static void end_bio_extent_buffer_writepage(struct bio *bio)
 
 
 
 
 
3695{
3696	struct bio_vec *bvec;
3697	struct extent_buffer *eb;
3698	int i, done;
3699
3700	bio_for_each_segment_all(bvec, bio, i) {
3701		struct page *page = bvec->bv_page;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3702
3703		eb = (struct extent_buffer *)page->private;
3704		BUG_ON(!eb);
3705		done = atomic_dec_and_test(&eb->io_pages);
3706
3707		if (bio->bi_error ||
3708		    test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)) {
3709			ClearPageUptodate(page);
3710			set_btree_ioerr(page);
3711		}
3712
3713		end_page_writeback(page);
 
 
3714
3715		if (!done)
3716			continue;
3717
3718		end_extent_buffer_writeback(eb);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3719	}
 
3720
3721	bio_put(bio);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3722}
3723
3724static noinline_for_stack int write_one_eb(struct extent_buffer *eb,
3725			struct btrfs_fs_info *fs_info,
3726			struct writeback_control *wbc,
3727			struct extent_page_data *epd)
3728{
3729	struct block_device *bdev = fs_info->fs_devices->latest_bdev;
3730	struct extent_io_tree *tree = &BTRFS_I(fs_info->btree_inode)->io_tree;
3731	u64 offset = eb->start;
3732	unsigned long i, num_pages;
3733	unsigned long bio_flags = 0;
3734	int rw = (epd->sync_io ? WRITE_SYNC : WRITE) | REQ_META;
3735	int ret = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3736
3737	clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
3738	num_pages = num_extent_pages(eb->start, eb->len);
3739	atomic_set(&eb->io_pages, num_pages);
3740	if (btrfs_header_owner(eb) == BTRFS_TREE_LOG_OBJECTID)
3741		bio_flags = EXTENT_BIO_TREE_LOG;
3742
3743	for (i = 0; i < num_pages; i++) {
3744		struct page *p = eb->pages[i];
3745
3746		clear_page_dirty_for_io(p);
3747		set_page_writeback(p);
3748		ret = submit_extent_page(rw, tree, wbc, p, offset >> 9,
3749					 PAGE_SIZE, 0, bdev, &epd->bio,
3750					 -1, end_bio_extent_buffer_writepage,
3751					 0, epd->bio_flags, bio_flags, false);
3752		epd->bio_flags = bio_flags;
3753		if (ret) {
3754			set_btree_ioerr(p);
3755			end_page_writeback(p);
3756			if (atomic_sub_and_test(num_pages - i, &eb->io_pages))
3757				end_extent_buffer_writeback(eb);
3758			ret = -EIO;
3759			break;
3760		}
3761		offset += PAGE_SIZE;
3762		update_nr_written(p, wbc, 1);
3763		unlock_page(p);
3764	}
 
 
 
 
 
 
 
3765
3766	if (unlikely(ret)) {
3767		for (; i < num_pages; i++) {
3768			struct page *p = eb->pages[i];
3769			clear_page_dirty_for_io(p);
3770			unlock_page(p);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3771		}
 
3772	}
 
 
3773
3774	return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3775}
3776
3777int btree_write_cache_pages(struct address_space *mapping,
3778				   struct writeback_control *wbc)
3779{
3780	struct extent_io_tree *tree = &BTRFS_I(mapping->host)->io_tree;
3781	struct btrfs_fs_info *fs_info = BTRFS_I(mapping->host)->root->fs_info;
3782	struct extent_buffer *eb, *prev_eb = NULL;
3783	struct extent_page_data epd = {
3784		.bio = NULL,
3785		.tree = tree,
3786		.extent_locked = 0,
3787		.sync_io = wbc->sync_mode == WB_SYNC_ALL,
3788		.bio_flags = 0,
3789	};
3790	int ret = 0;
3791	int done = 0;
3792	int nr_to_write_done = 0;
3793	struct pagevec pvec;
3794	int nr_pages;
3795	pgoff_t index;
3796	pgoff_t end;		/* Inclusive */
3797	int scanned = 0;
3798	int tag;
3799
3800	pagevec_init(&pvec, 0);
3801	if (wbc->range_cyclic) {
3802		index = mapping->writeback_index; /* Start from prev offset */
3803		end = -1;
 
 
 
 
 
3804	} else {
3805		index = wbc->range_start >> PAGE_SHIFT;
3806		end = wbc->range_end >> PAGE_SHIFT;
3807		scanned = 1;
3808	}
3809	if (wbc->sync_mode == WB_SYNC_ALL)
3810		tag = PAGECACHE_TAG_TOWRITE;
3811	else
3812		tag = PAGECACHE_TAG_DIRTY;
 
3813retry:
3814	if (wbc->sync_mode == WB_SYNC_ALL)
3815		tag_pages_for_writeback(mapping, index, end);
3816	while (!done && !nr_to_write_done && (index <= end) &&
3817	       (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3818			min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3819		unsigned i;
3820
3821		scanned = 1;
3822		for (i = 0; i < nr_pages; i++) {
3823			struct page *page = pvec.pages[i];
3824
3825			if (!PagePrivate(page))
3826				continue;
3827
3828			if (!wbc->range_cyclic && page->index > end) {
3829				done = 1;
3830				break;
3831			}
3832
3833			spin_lock(&mapping->private_lock);
3834			if (!PagePrivate(page)) {
3835				spin_unlock(&mapping->private_lock);
3836				continue;
3837			}
3838
3839			eb = (struct extent_buffer *)page->private;
3840
3841			/*
3842			 * Shouldn't happen and normally this would be a BUG_ON
3843			 * but no sense in crashing the users box for something
3844			 * we can survive anyway.
3845			 */
3846			if (WARN_ON(!eb)) {
3847				spin_unlock(&mapping->private_lock);
3848				continue;
3849			}
3850
3851			if (eb == prev_eb) {
3852				spin_unlock(&mapping->private_lock);
3853				continue;
3854			}
3855
3856			ret = atomic_inc_not_zero(&eb->refs);
3857			spin_unlock(&mapping->private_lock);
3858			if (!ret)
3859				continue;
3860
3861			prev_eb = eb;
3862			ret = lock_extent_buffer_for_io(eb, fs_info, &epd);
3863			if (!ret) {
3864				free_extent_buffer(eb);
3865				continue;
3866			}
3867
3868			ret = write_one_eb(eb, fs_info, wbc, &epd);
3869			if (ret) {
3870				done = 1;
3871				free_extent_buffer(eb);
3872				break;
3873			}
3874			free_extent_buffer(eb);
3875
3876			/*
3877			 * the filesystem may choose to bump up nr_to_write.
3878			 * We have to make sure to honor the new nr_to_write
3879			 * at any time
3880			 */
3881			nr_to_write_done = wbc->nr_to_write <= 0;
3882		}
3883		pagevec_release(&pvec);
3884		cond_resched();
3885	}
3886	if (!scanned && !done) {
3887		/*
3888		 * We hit the last page and there is more work to be done: wrap
3889		 * back to the start of the file
3890		 */
3891		scanned = 1;
3892		index = 0;
3893		goto retry;
3894	}
3895	flush_write_bio(&epd);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3896	return ret;
3897}
3898
3899/**
3900 * write_cache_pages - walk the list of dirty pages of the given address space and write all of them.
3901 * @mapping: address space structure to write
3902 * @wbc: subtract the number of written pages from *@wbc->nr_to_write
3903 * @writepage: function called for each page
3904 * @data: data passed to writepage function
3905 *
3906 * If a page is already under I/O, write_cache_pages() skips it, even
3907 * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
3908 * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
3909 * and msync() need to guarantee that all the data which was dirty at the time
3910 * the call was made get new I/O started against them.  If wbc->sync_mode is
3911 * WB_SYNC_ALL then we were called for data integrity and we must wait for
3912 * existing IO to complete.
3913 */
3914static int extent_write_cache_pages(struct extent_io_tree *tree,
3915			     struct address_space *mapping,
3916			     struct writeback_control *wbc,
3917			     writepage_t writepage, void *data,
3918			     void (*flush_fn)(void *))
3919{
 
3920	struct inode *inode = mapping->host;
3921	int ret = 0;
3922	int done = 0;
3923	int err = 0;
3924	int nr_to_write_done = 0;
3925	struct pagevec pvec;
3926	int nr_pages;
3927	pgoff_t index;
3928	pgoff_t end;		/* Inclusive */
 
 
3929	int scanned = 0;
3930	int tag;
3931
3932	/*
3933	 * We have to hold onto the inode so that ordered extents can do their
3934	 * work when the IO finishes.  The alternative to this is failing to add
3935	 * an ordered extent if the igrab() fails there and that is a huge pain
3936	 * to deal with, so instead just hold onto the inode throughout the
3937	 * writepages operation.  If it fails here we are freeing up the inode
3938	 * anyway and we'd rather not waste our time writing out stuff that is
3939	 * going to be truncated anyway.
3940	 */
3941	if (!igrab(inode))
3942		return 0;
3943
3944	pagevec_init(&pvec, 0);
3945	if (wbc->range_cyclic) {
3946		index = mapping->writeback_index; /* Start from prev offset */
3947		end = -1;
 
 
 
 
 
3948	} else {
3949		index = wbc->range_start >> PAGE_SHIFT;
3950		end = wbc->range_end >> PAGE_SHIFT;
 
 
3951		scanned = 1;
3952	}
3953	if (wbc->sync_mode == WB_SYNC_ALL)
 
 
 
 
 
 
 
 
 
 
 
 
 
3954		tag = PAGECACHE_TAG_TOWRITE;
3955	else
3956		tag = PAGECACHE_TAG_DIRTY;
3957retry:
3958	if (wbc->sync_mode == WB_SYNC_ALL)
3959		tag_pages_for_writeback(mapping, index, end);
 
3960	while (!done && !nr_to_write_done && (index <= end) &&
3961	       (nr_pages = pagevec_lookup_tag(&pvec, mapping, &index, tag,
3962			min(end - index, (pgoff_t)PAGEVEC_SIZE-1) + 1))) {
3963		unsigned i;
3964
3965		scanned = 1;
3966		for (i = 0; i < nr_pages; i++) {
3967			struct page *page = pvec.pages[i];
3968
 
3969			/*
3970			 * At this point we hold neither mapping->tree_lock nor
3971			 * lock on the page itself: the page may be truncated or
3972			 * invalidated (changing page->mapping to NULL), or even
3973			 * swizzled back from swapper_space to tmpfs file
3974			 * mapping
3975			 */
3976			if (!trylock_page(page)) {
3977				flush_fn(data);
3978				lock_page(page);
3979			}
3980
3981			if (unlikely(page->mapping != mapping)) {
3982				unlock_page(page);
3983				continue;
3984			}
3985
3986			if (!wbc->range_cyclic && page->index > end) {
3987				done = 1;
3988				unlock_page(page);
3989				continue;
3990			}
3991
3992			if (wbc->sync_mode != WB_SYNC_NONE) {
3993				if (PageWriteback(page))
3994					flush_fn(data);
3995				wait_on_page_writeback(page);
3996			}
3997
3998			if (PageWriteback(page) ||
3999			    !clear_page_dirty_for_io(page)) {
4000				unlock_page(page);
4001				continue;
4002			}
4003
4004			ret = (*writepage)(page, wbc, data);
4005
4006			if (unlikely(ret == AOP_WRITEPAGE_ACTIVATE)) {
4007				unlock_page(page);
4008				ret = 0;
4009			}
4010			if (!err && ret < 0)
4011				err = ret;
4012
4013			/*
4014			 * the filesystem may choose to bump up nr_to_write.
4015			 * We have to make sure to honor the new nr_to_write
4016			 * at any time
4017			 */
4018			nr_to_write_done = wbc->nr_to_write <= 0;
 
4019		}
4020		pagevec_release(&pvec);
4021		cond_resched();
4022	}
4023	if (!scanned && !done && !err) {
4024		/*
4025		 * We hit the last page and there is more work to be done: wrap
4026		 * back to the start of the file
4027		 */
4028		scanned = 1;
4029		index = 0;
4030		goto retry;
4031	}
4032	btrfs_add_delayed_iput(inode);
4033	return err;
4034}
4035
4036static void flush_epd_write_bio(struct extent_page_data *epd)
4037{
4038	if (epd->bio) {
4039		int rw = WRITE;
4040		int ret;
4041
4042		if (epd->sync_io)
4043			rw = WRITE_SYNC;
4044
4045		ret = submit_one_bio(rw, epd->bio, 0, epd->bio_flags);
4046		BUG_ON(ret < 0); /* -ENOMEM */
4047		epd->bio = NULL;
 
 
4048	}
4049}
4050
4051static noinline void flush_write_bio(void *data)
4052{
4053	struct extent_page_data *epd = data;
4054	flush_epd_write_bio(epd);
4055}
4056
4057int extent_write_full_page(struct extent_io_tree *tree, struct page *page,
4058			  get_extent_t *get_extent,
4059			  struct writeback_control *wbc)
4060{
4061	int ret;
4062	struct extent_page_data epd = {
4063		.bio = NULL,
4064		.tree = tree,
4065		.get_extent = get_extent,
4066		.extent_locked = 0,
4067		.sync_io = wbc->sync_mode == WB_SYNC_ALL,
4068		.bio_flags = 0,
4069	};
4070
4071	ret = __extent_writepage(page, wbc, &epd);
 
4072
4073	flush_epd_write_bio(&epd);
4074	return ret;
4075}
4076
4077int extent_write_locked_range(struct extent_io_tree *tree, struct inode *inode,
4078			      u64 start, u64 end, get_extent_t *get_extent,
4079			      int mode)
 
 
 
 
 
4080{
 
4081	int ret = 0;
4082	struct address_space *mapping = inode->i_mapping;
4083	struct page *page;
4084	unsigned long nr_pages = (end - start + PAGE_SIZE) >>
4085		PAGE_SHIFT;
4086
4087	struct extent_page_data epd = {
4088		.bio = NULL,
4089		.tree = tree,
4090		.get_extent = get_extent,
4091		.extent_locked = 1,
4092		.sync_io = mode == WB_SYNC_ALL,
4093		.bio_flags = 0,
4094	};
4095	struct writeback_control wbc_writepages = {
4096		.sync_mode	= mode,
4097		.nr_to_write	= nr_pages * 2,
4098		.range_start	= start,
4099		.range_end	= end + 1,
4100	};
4101
4102	while (start <= end) {
4103		page = find_get_page(mapping, start >> PAGE_SHIFT);
4104		if (clear_page_dirty_for_io(page))
4105			ret = __extent_writepage(page, &wbc_writepages, &epd);
4106		else {
4107			if (tree->ops && tree->ops->writepage_end_io_hook)
4108				tree->ops->writepage_end_io_hook(page, start,
4109						 start + PAGE_SIZE - 1,
4110						 NULL, 1);
4111			unlock_page(page);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4112		}
 
 
 
 
4113		put_page(page);
4114		start += PAGE_SIZE;
4115	}
4116
4117	flush_epd_write_bio(&epd);
4118	return ret;
4119}
4120
4121int extent_writepages(struct extent_io_tree *tree,
4122		      struct address_space *mapping,
4123		      get_extent_t *get_extent,
4124		      struct writeback_control *wbc)
4125{
 
4126	int ret = 0;
4127	struct extent_page_data epd = {
4128		.bio = NULL,
4129		.tree = tree,
4130		.get_extent = get_extent,
4131		.extent_locked = 0,
4132		.sync_io = wbc->sync_mode == WB_SYNC_ALL,
4133		.bio_flags = 0,
4134	};
4135
4136	ret = extent_write_cache_pages(tree, mapping, wbc,
4137				       __extent_writepage, &epd,
4138				       flush_write_bio);
4139	flush_epd_write_bio(&epd);
 
 
 
 
4140	return ret;
4141}
4142
4143int extent_readpages(struct extent_io_tree *tree,
4144		     struct address_space *mapping,
4145		     struct list_head *pages, unsigned nr_pages,
4146		     get_extent_t get_extent)
4147{
4148	struct bio *bio = NULL;
4149	unsigned page_idx;
4150	unsigned long bio_flags = 0;
4151	struct page *pagepool[16];
4152	struct page *page;
4153	struct extent_map *em_cached = NULL;
4154	int nr = 0;
4155	u64 prev_em_start = (u64)-1;
 
4156
4157	for (page_idx = 0; page_idx < nr_pages; page_idx++) {
4158		page = list_entry(pages->prev, struct page, lru);
 
4159
4160		prefetchw(&page->flags);
4161		list_del(&page->lru);
4162		if (add_to_page_cache_lru(page, mapping,
4163					page->index, GFP_NOFS)) {
4164			put_page(page);
4165			continue;
4166		}
4167
4168		pagepool[nr++] = page;
4169		if (nr < ARRAY_SIZE(pagepool))
4170			continue;
4171		__extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
4172				   &bio, 0, &bio_flags, READ, &prev_em_start);
4173		nr = 0;
4174	}
4175	if (nr)
4176		__extent_readpages(tree, pagepool, nr, get_extent, &em_cached,
4177				   &bio, 0, &bio_flags, READ, &prev_em_start);
4178
4179	if (em_cached)
4180		free_extent_map(em_cached);
4181
4182	BUG_ON(!list_empty(pages));
4183	if (bio)
4184		return submit_one_bio(READ, bio, 0, bio_flags);
4185	return 0;
4186}
4187
4188/*
4189 * basic invalidatepage code, this waits on any locked or writeback
4190 * ranges corresponding to the page, and then deletes any extent state
4191 * records from the tree
4192 */
4193int extent_invalidatepage(struct extent_io_tree *tree,
4194			  struct page *page, unsigned long offset)
4195{
4196	struct extent_state *cached_state = NULL;
4197	u64 start = page_offset(page);
4198	u64 end = start + PAGE_SIZE - 1;
4199	size_t blocksize = page->mapping->host->i_sb->s_blocksize;
 
 
 
4200
4201	start += ALIGN(offset, blocksize);
4202	if (start > end)
4203		return 0;
4204
4205	lock_extent_bits(tree, start, end, &cached_state);
4206	wait_on_page_writeback(page);
4207	clear_extent_bit(tree, start, end,
4208			 EXTENT_LOCKED | EXTENT_DIRTY | EXTENT_DELALLOC |
4209			 EXTENT_DO_ACCOUNTING,
4210			 1, 1, &cached_state, GFP_NOFS);
 
 
 
4211	return 0;
4212}
4213
4214/*
4215 * a helper for releasepage, this tests for areas of the page that
4216 * are locked or under IO and drops the related state bits if it is safe
4217 * to drop the page.
4218 */
4219static int try_release_extent_state(struct extent_map_tree *map,
4220				    struct extent_io_tree *tree,
4221				    struct page *page, gfp_t mask)
4222{
4223	u64 start = page_offset(page);
4224	u64 end = start + PAGE_SIZE - 1;
4225	int ret = 1;
4226
4227	if (test_range_bit(tree, start, end,
4228			   EXTENT_IOBITS, 0, NULL))
4229		ret = 0;
4230	else {
4231		if ((mask & GFP_NOFS) == GFP_NOFS)
4232			mask = GFP_NOFS;
 
 
4233		/*
4234		 * at this point we can safely clear everything except the
4235		 * locked bit and the nodatasum bit
 
 
4236		 */
4237		ret = clear_extent_bit(tree, start, end,
4238				 ~(EXTENT_LOCKED | EXTENT_NODATASUM),
4239				 0, 0, NULL, mask);
4240
4241		/* if clear_extent_bit failed for enomem reasons,
4242		 * we can't allow the release to continue.
4243		 */
4244		if (ret < 0)
4245			ret = 0;
4246		else
4247			ret = 1;
4248	}
4249	return ret;
4250}
4251
4252/*
4253 * a helper for releasepage.  As long as there are no locked extents
4254 * in the range corresponding to the page, both state records and extent
4255 * map records are removed
4256 */
4257int try_release_extent_mapping(struct extent_map_tree *map,
4258			       struct extent_io_tree *tree, struct page *page,
4259			       gfp_t mask)
4260{
4261	struct extent_map *em;
4262	u64 start = page_offset(page);
4263	u64 end = start + PAGE_SIZE - 1;
 
 
 
4264
4265	if (gfpflags_allow_blocking(mask) &&
4266	    page->mapping->host->i_size > SZ_16M) {
4267		u64 len;
4268		while (start <= end) {
 
 
 
4269			len = end - start + 1;
4270			write_lock(&map->lock);
4271			em = lookup_extent_mapping(map, start, len);
4272			if (!em) {
4273				write_unlock(&map->lock);
4274				break;
4275			}
4276			if (test_bit(EXTENT_FLAG_PINNED, &em->flags) ||
4277			    em->start != start) {
4278				write_unlock(&map->lock);
4279				free_extent_map(em);
4280				break;
4281			}
4282			if (!test_range_bit(tree, em->start,
4283					    extent_map_end(em) - 1,
4284					    EXTENT_LOCKED | EXTENT_WRITEBACK,
4285					    0, NULL)) {
4286				remove_extent_mapping(map, em);
4287				/* once for the rb tree */
4288				free_extent_map(em);
4289			}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4290			start = extent_map_end(em);
4291			write_unlock(&map->lock);
4292
4293			/* once for us */
4294			free_extent_map(em);
 
 
4295		}
4296	}
4297	return try_release_extent_state(map, tree, page, mask);
4298}
4299
 
 
 
 
 
 
 
4300/*
4301 * helper function for fiemap, which doesn't want to see any holes.
4302 * This maps until we find something past 'last'
 
 
 
 
 
4303 */
4304static struct extent_map *get_extent_skip_holes(struct inode *inode,
4305						u64 offset,
4306						u64 last,
4307						get_extent_t *get_extent)
4308{
4309	u64 sectorsize = BTRFS_I(inode)->root->sectorsize;
4310	struct extent_map *em;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4311	u64 len;
 
 
 
4312
4313	if (offset >= last)
4314		return NULL;
 
 
 
 
4315
4316	while (1) {
4317		len = last - offset;
4318		if (len == 0)
4319			break;
4320		len = ALIGN(len, sectorsize);
4321		em = get_extent(inode, NULL, 0, offset, len, 0);
4322		if (IS_ERR_OR_NULL(em))
4323			return em;
 
 
 
4324
4325		/* if this isn't a hole return it */
4326		if (!test_bit(EXTENT_FLAG_VACANCY, &em->flags) &&
4327		    em->block_start != EXTENT_MAP_HOLE) {
4328			return em;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4329		}
 
4330
4331		/* this is a hole, advance to the next extent */
4332		offset = extent_map_end(em);
4333		free_extent_map(em);
4334		if (offset >= last)
4335			break;
 
 
 
 
 
 
 
 
 
 
4336	}
4337	return NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4338}
4339
4340int extent_fiemap(struct inode *inode, struct fiemap_extent_info *fieinfo,
4341		__u64 start, __u64 len, get_extent_t *get_extent)
4342{
4343	int ret = 0;
4344	u64 off = start;
4345	u64 max = start + len;
4346	u32 flags = 0;
4347	u32 found_type;
4348	u64 last;
4349	u64 last_for_get_extent = 0;
4350	u64 disko = 0;
4351	u64 isize = i_size_read(inode);
4352	struct btrfs_key found_key;
4353	struct extent_map *em = NULL;
4354	struct extent_state *cached_state = NULL;
4355	struct btrfs_path *path;
4356	struct btrfs_root *root = BTRFS_I(inode)->root;
4357	int end = 0;
4358	u64 em_start = 0;
4359	u64 em_len = 0;
4360	u64 em_end = 0;
4361
4362	if (len == 0)
4363		return -EINVAL;
 
4364
4365	path = btrfs_alloc_path();
4366	if (!path)
4367		return -ENOMEM;
4368	path->leave_spinning = 1;
 
 
 
4369
4370	start = round_down(start, BTRFS_I(inode)->root->sectorsize);
4371	len = round_up(max, BTRFS_I(inode)->root->sectorsize) - start;
 
4372
4373	/*
4374	 * lookup the last file extent.  We're not using i_size here
4375	 * because there might be preallocation past i_size
4376	 */
4377	ret = btrfs_lookup_file_extent(NULL, root, path, btrfs_ino(inode), -1,
4378				       0);
4379	if (ret < 0) {
4380		btrfs_free_path(path);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4381		return ret;
 
 
 
 
 
4382	}
4383	WARN_ON(!ret);
4384	path->slots[0]--;
4385	btrfs_item_key_to_cpu(path->nodes[0], &found_key, path->slots[0]);
4386	found_type = found_key.type;
4387
4388	/* No extents, but there might be delalloc bits */
4389	if (found_key.objectid != btrfs_ino(inode) ||
4390	    found_type != BTRFS_EXTENT_DATA_KEY) {
4391		/* have to trust i_size as the end */
4392		last = (u64)-1;
4393		last_for_get_extent = isize;
4394	} else {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4395		/*
4396		 * remember the start of the last extent.  There are a
4397		 * bunch of different factors that go into the length of the
4398		 * extent, so its much less complex to remember where it started
4399		 */
4400		last = found_key.offset;
4401		last_for_get_extent = last + 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4402	}
4403	btrfs_release_path(path);
4404
4405	/*
4406	 * we might have some extents allocated but more delalloc past those
4407	 * extents.  so, we trust isize unless the start of the last extent is
4408	 * beyond isize
4409	 */
4410	if (last < isize) {
4411		last = (u64)-1;
4412		last_for_get_extent = isize;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4413	}
4414
4415	lock_extent_bits(&BTRFS_I(inode)->io_tree, start, start + len - 1,
4416			 &cached_state);
4417
4418	em = get_extent_skip_holes(inode, start, last_for_get_extent,
4419				   get_extent);
4420	if (!em)
4421		goto out;
4422	if (IS_ERR(em)) {
4423		ret = PTR_ERR(em);
4424		goto out;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4425	}
4426
4427	while (!end) {
4428		u64 offset_in_extent = 0;
 
 
 
 
 
 
 
 
4429
4430		/* break if the extent we found is outside the range */
4431		if (em->start >= max || extent_map_end(em) < off)
4432			break;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4433
4434		/*
4435		 * get_extent may return an extent that starts before our
4436		 * requested range.  We have to make sure the ranges
4437		 * we return to fiemap always move forward and don't
4438		 * overlap, so adjust the offsets here
4439		 */
4440		em_start = max(em->start, off);
4441
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4442		/*
4443		 * record the offset from the start of the extent
4444		 * for adjusting the disk offset below.  Only do this if the
4445		 * extent isn't compressed since our in ram offset may be past
4446		 * what we have actually allocated on disk.
4447		 */
4448		if (!test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
4449			offset_in_extent = em_start - em->start;
4450		em_end = extent_map_end(em);
4451		em_len = em_end - em_start;
4452		disko = 0;
4453		flags = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4454
4455		/*
4456		 * bump off for our next call to get_extent
 
4457		 */
4458		off = extent_map_end(em);
4459		if (off >= max)
4460			end = 1;
4461
4462		if (em->block_start == EXTENT_MAP_LAST_BYTE) {
4463			end = 1;
4464			flags |= FIEMAP_EXTENT_LAST;
4465		} else if (em->block_start == EXTENT_MAP_INLINE) {
4466			flags |= (FIEMAP_EXTENT_DATA_INLINE |
4467				  FIEMAP_EXTENT_NOT_ALIGNED);
4468		} else if (em->block_start == EXTENT_MAP_DELALLOC) {
4469			flags |= (FIEMAP_EXTENT_DELALLOC |
4470				  FIEMAP_EXTENT_UNKNOWN);
4471		} else if (fieinfo->fi_extents_max) {
4472			u64 bytenr = em->block_start -
4473				(em->start - em->orig_start);
4474
4475			disko = em->block_start + offset_in_extent;
4476
4477			/*
4478			 * As btrfs supports shared space, this information
4479			 * can be exported to userspace tools via
4480			 * flag FIEMAP_EXTENT_SHARED.  If fi_extents_max == 0
4481			 * then we're just getting a count and we can skip the
4482			 * lookup stuff.
4483			 */
4484			ret = btrfs_check_shared(NULL, root->fs_info,
4485						 root->objectid,
4486						 btrfs_ino(inode), bytenr);
4487			if (ret < 0)
4488				goto out_free;
4489			if (ret)
4490				flags |= FIEMAP_EXTENT_SHARED;
4491			ret = 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4492		}
4493		if (test_bit(EXTENT_FLAG_COMPRESSED, &em->flags))
 
4494			flags |= FIEMAP_EXTENT_ENCODED;
4495		if (test_bit(EXTENT_FLAG_PREALLOC, &em->flags))
4496			flags |= FIEMAP_EXTENT_UNWRITTEN;
4497
4498		free_extent_map(em);
4499		em = NULL;
4500		if ((em_start >= last) || em_len == (u64)-1 ||
4501		   (last == (u64)-1 && isize <= em_end)) {
4502			flags |= FIEMAP_EXTENT_LAST;
4503			end = 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4504		}
4505
4506		/* now scan forward to see if this is really the last extent. */
4507		em = get_extent_skip_holes(inode, off, last_for_get_extent,
4508					   get_extent);
4509		if (IS_ERR(em)) {
4510			ret = PTR_ERR(em);
4511			goto out;
4512		}
4513		if (!em) {
4514			flags |= FIEMAP_EXTENT_LAST;
4515			end = 1;
 
 
 
4516		}
4517		ret = fiemap_fill_next_extent(fieinfo, em_start, disko,
4518					      em_len, flags);
4519		if (ret) {
4520			if (ret == 1)
4521				ret = 0;
4522			goto out_free;
 
4523		}
 
 
 
 
 
 
 
 
 
 
 
4524	}
4525out_free:
4526	free_extent_map(em);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4527out:
 
 
 
4528	btrfs_free_path(path);
4529	unlock_extent_cached(&BTRFS_I(inode)->io_tree, start, start + len - 1,
4530			     &cached_state, GFP_NOFS);
4531	return ret;
4532}
4533
4534static void __free_extent_buffer(struct extent_buffer *eb)
4535{
4536	btrfs_leak_debug_del(&eb->leak_list);
4537	kmem_cache_free(extent_buffer_cache, eb);
4538}
4539
4540int extent_buffer_under_io(struct extent_buffer *eb)
4541{
4542	return (atomic_read(&eb->io_pages) ||
4543		test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
4544		test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4545}
4546
4547/*
4548 * Helper for releasing extent buffer page.
4549 */
4550static void btrfs_release_extent_buffer_page(struct extent_buffer *eb)
4551{
4552	unsigned long index;
4553	struct page *page;
4554	int mapped = !test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4555
4556	BUG_ON(extent_buffer_under_io(eb));
4557
4558	index = num_extent_pages(eb->start, eb->len);
4559	if (index == 0)
4560		return;
 
 
 
 
 
 
 
 
 
 
4561
4562	do {
4563		index--;
4564		page = eb->pages[index];
4565		if (!page)
4566			continue;
 
 
 
 
 
 
 
 
4567		if (mapped)
4568			spin_lock(&page->mapping->private_lock);
 
 
 
 
4569		/*
4570		 * We do this since we'll remove the pages after we've
4571		 * removed the eb from the radix tree, so we could race
4572		 * and have this page now attached to the new eb.  So
4573		 * only clear page_private if it's still connected to
4574		 * this eb.
4575		 */
4576		if (PagePrivate(page) &&
4577		    page->private == (unsigned long)eb) {
4578			BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
4579			BUG_ON(PageDirty(page));
4580			BUG_ON(PageWriteback(page));
4581			/*
4582			 * We need to make sure we haven't be attached
4583			 * to a new eb.
4584			 */
4585			ClearPagePrivate(page);
4586			set_page_private(page, 0);
4587			/* One for the page private */
4588			put_page(page);
4589		}
4590
4591		if (mapped)
4592			spin_unlock(&page->mapping->private_lock);
 
 
4593
4594		/* One for when we alloced the page */
4595		put_page(page);
4596	} while (index != 0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4597}
4598
4599/*
4600 * Helper for releasing the extent buffer.
4601 */
4602static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
4603{
4604	btrfs_release_extent_buffer_page(eb);
 
4605	__free_extent_buffer(eb);
4606}
4607
4608static struct extent_buffer *
4609__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
4610		      unsigned long len)
4611{
4612	struct extent_buffer *eb = NULL;
4613
4614	eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
4615	eb->start = start;
4616	eb->len = len;
4617	eb->fs_info = fs_info;
4618	eb->bflags = 0;
4619	rwlock_init(&eb->lock);
4620	atomic_set(&eb->write_locks, 0);
4621	atomic_set(&eb->read_locks, 0);
4622	atomic_set(&eb->blocking_readers, 0);
4623	atomic_set(&eb->blocking_writers, 0);
4624	atomic_set(&eb->spinning_readers, 0);
4625	atomic_set(&eb->spinning_writers, 0);
4626	eb->lock_nested = 0;
4627	init_waitqueue_head(&eb->write_lock_wq);
4628	init_waitqueue_head(&eb->read_lock_wq);
4629
4630	btrfs_leak_debug_add(&eb->leak_list, &buffers);
4631
4632	spin_lock_init(&eb->refs_lock);
4633	atomic_set(&eb->refs, 1);
4634	atomic_set(&eb->io_pages, 0);
4635
4636	/*
4637	 * Sanity checks, currently the maximum is 64k covered by 16x 4k pages
4638	 */
4639	BUILD_BUG_ON(BTRFS_MAX_METADATA_BLOCKSIZE
4640		> MAX_INLINE_EXTENT_BUFFER_SIZE);
4641	BUG_ON(len > MAX_INLINE_EXTENT_BUFFER_SIZE);
4642
4643	return eb;
4644}
4645
4646struct extent_buffer *btrfs_clone_extent_buffer(struct extent_buffer *src)
4647{
4648	unsigned long i;
4649	struct page *p;
4650	struct extent_buffer *new;
4651	unsigned long num_pages = num_extent_pages(src->start, src->len);
 
4652
4653	new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
4654	if (new == NULL)
4655		return NULL;
4656
4657	for (i = 0; i < num_pages; i++) {
4658		p = alloc_page(GFP_NOFS);
4659		if (!p) {
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4660			btrfs_release_extent_buffer(new);
4661			return NULL;
4662		}
4663		attach_extent_buffer_page(new, p);
4664		WARN_ON(PageDirty(p));
4665		SetPageUptodate(p);
4666		new->pages[i] = p;
4667	}
4668
4669	copy_extent_buffer(new, src, 0, 0, src->len);
4670	set_bit(EXTENT_BUFFER_UPTODATE, &new->bflags);
4671	set_bit(EXTENT_BUFFER_DUMMY, &new->bflags);
4672
4673	return new;
4674}
4675
4676struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
4677						  u64 start, unsigned long len)
4678{
4679	struct extent_buffer *eb;
4680	unsigned long num_pages;
4681	unsigned long i;
4682
4683	num_pages = num_extent_pages(start, len);
4684
4685	eb = __alloc_extent_buffer(fs_info, start, len);
4686	if (!eb)
4687		return NULL;
4688
4689	for (i = 0; i < num_pages; i++) {
4690		eb->pages[i] = alloc_page(GFP_NOFS);
4691		if (!eb->pages[i])
 
 
 
 
 
4692			goto err;
4693	}
 
4694	set_extent_buffer_uptodate(eb);
4695	btrfs_set_header_nritems(eb, 0);
4696	set_bit(EXTENT_BUFFER_DUMMY, &eb->bflags);
4697
4698	return eb;
4699err:
4700	for (; i > 0; i--)
4701		__free_page(eb->pages[i - 1]);
 
 
 
 
4702	__free_extent_buffer(eb);
4703	return NULL;
4704}
4705
4706struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
4707						u64 start)
4708{
4709	unsigned long len;
4710
4711	if (!fs_info) {
4712		/*
4713		 * Called only from tests that don't always have a fs_info
4714		 * available, but we know that nodesize is 4096
4715		 */
4716		len = 4096;
4717	} else {
4718		len = fs_info->tree_root->nodesize;
4719	}
4720
4721	return __alloc_dummy_extent_buffer(fs_info, start, len);
4722}
4723
4724static void check_buffer_tree_ref(struct extent_buffer *eb)
4725{
4726	int refs;
4727	/* the ref bit is tricky.  We have to make sure it is set
4728	 * if we have the buffer dirty.   Otherwise the
4729	 * code to free a buffer can end up dropping a dirty
4730	 * page
4731	 *
4732	 * Once the ref bit is set, it won't go away while the
4733	 * buffer is dirty or in writeback, and it also won't
4734	 * go away while we have the reference count on the
4735	 * eb bumped.
4736	 *
4737	 * We can't just set the ref bit without bumping the
4738	 * ref on the eb because free_extent_buffer might
4739	 * see the ref bit and try to clear it.  If this happens
4740	 * free_extent_buffer might end up dropping our original
4741	 * ref by mistake and freeing the page before we are able
4742	 * to add one more ref.
4743	 *
4744	 * So bump the ref count first, then set the bit.  If someone
4745	 * beat us to it, drop the ref we added.
 
 
 
 
 
4746	 */
4747	refs = atomic_read(&eb->refs);
4748	if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4749		return;
4750
4751	spin_lock(&eb->refs_lock);
4752	if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
4753		atomic_inc(&eb->refs);
4754	spin_unlock(&eb->refs_lock);
4755}
4756
4757static void mark_extent_buffer_accessed(struct extent_buffer *eb,
4758		struct page *accessed)
4759{
4760	unsigned long num_pages, i;
4761
4762	check_buffer_tree_ref(eb);
4763
4764	num_pages = num_extent_pages(eb->start, eb->len);
4765	for (i = 0; i < num_pages; i++) {
4766		struct page *p = eb->pages[i];
4767
4768		if (p != accessed)
4769			mark_page_accessed(p);
4770	}
4771}
4772
4773struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
4774					 u64 start)
4775{
4776	struct extent_buffer *eb;
4777
4778	rcu_read_lock();
4779	eb = radix_tree_lookup(&fs_info->buffer_radix,
4780			       start >> PAGE_SHIFT);
4781	if (eb && atomic_inc_not_zero(&eb->refs)) {
4782		rcu_read_unlock();
4783		/*
4784		 * Lock our eb's refs_lock to avoid races with
4785		 * free_extent_buffer. When we get our eb it might be flagged
4786		 * with EXTENT_BUFFER_STALE and another task running
4787		 * free_extent_buffer might have seen that flag set,
4788		 * eb->refs == 2, that the buffer isn't under IO (dirty and
4789		 * writeback flags not set) and it's still in the tree (flag
4790		 * EXTENT_BUFFER_TREE_REF set), therefore being in the process
4791		 * of decrementing the extent buffer's reference count twice.
4792		 * So here we could race and increment the eb's reference count,
4793		 * clear its stale flag, mark it as dirty and drop our reference
4794		 * before the other task finishes executing free_extent_buffer,
4795		 * which would later result in an attempt to free an extent
4796		 * buffer that is dirty.
4797		 */
4798		if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
4799			spin_lock(&eb->refs_lock);
4800			spin_unlock(&eb->refs_lock);
4801		}
4802		mark_extent_buffer_accessed(eb, NULL);
4803		return eb;
4804	}
4805	rcu_read_unlock();
4806
4807	return NULL;
4808}
4809
4810#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
4811struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
4812					       u64 start)
4813{
4814	struct extent_buffer *eb, *exists = NULL;
4815	int ret;
4816
4817	eb = find_extent_buffer(fs_info, start);
4818	if (eb)
4819		return eb;
4820	eb = alloc_dummy_extent_buffer(fs_info, start);
4821	if (!eb)
4822		return NULL;
4823	eb->fs_info = fs_info;
4824again:
4825	ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4826	if (ret)
 
4827		goto free_eb;
 
4828	spin_lock(&fs_info->buffer_lock);
4829	ret = radix_tree_insert(&fs_info->buffer_radix,
4830				start >> PAGE_SHIFT, eb);
4831	spin_unlock(&fs_info->buffer_lock);
4832	radix_tree_preload_end();
4833	if (ret == -EEXIST) {
4834		exists = find_extent_buffer(fs_info, start);
4835		if (exists)
4836			goto free_eb;
4837		else
4838			goto again;
4839	}
4840	check_buffer_tree_ref(eb);
4841	set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
4842
4843	/*
4844	 * We will free dummy extent buffer's if they come into
4845	 * free_extent_buffer with a ref count of 2, but if we are using this we
4846	 * want the buffers to stay in memory until we're done with them, so
4847	 * bump the ref count again.
4848	 */
4849	atomic_inc(&eb->refs);
4850	return eb;
4851free_eb:
4852	btrfs_release_extent_buffer(eb);
4853	return exists;
4854}
4855#endif
4856
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4857struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
4858					  u64 start)
4859{
4860	unsigned long len = fs_info->tree_root->nodesize;
4861	unsigned long num_pages = num_extent_pages(start, len);
4862	unsigned long i;
4863	unsigned long index = start >> PAGE_SHIFT;
4864	struct extent_buffer *eb;
4865	struct extent_buffer *exists = NULL;
4866	struct page *p;
4867	struct address_space *mapping = fs_info->btree_inode->i_mapping;
 
 
 
4868	int uptodate = 1;
4869	int ret;
4870
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4871	eb = find_extent_buffer(fs_info, start);
4872	if (eb)
4873		return eb;
4874
4875	eb = __alloc_extent_buffer(fs_info, start, len);
4876	if (!eb)
4877		return NULL;
4878
4879	for (i = 0; i < num_pages; i++, index++) {
4880		p = find_or_create_page(mapping, index, GFP_NOFS|__GFP_NOFAIL);
4881		if (!p)
4882			goto free_eb;
 
 
4883
4884		spin_lock(&mapping->private_lock);
4885		if (PagePrivate(p)) {
4886			/*
4887			 * We could have already allocated an eb for this page
4888			 * and attached one so lets see if we can get a ref on
4889			 * the existing eb, and if we can we know it's good and
4890			 * we can just return that one, else we know we can just
4891			 * overwrite page->private.
4892			 */
4893			exists = (struct extent_buffer *)p->private;
4894			if (atomic_inc_not_zero(&exists->refs)) {
4895				spin_unlock(&mapping->private_lock);
4896				unlock_page(p);
4897				put_page(p);
4898				mark_extent_buffer_accessed(exists, p);
4899				goto free_eb;
4900			}
4901			exists = NULL;
4902
4903			/*
4904			 * Do this so attach doesn't complain and we need to
4905			 * drop the ref the old guy had.
4906			 */
4907			ClearPagePrivate(p);
4908			WARN_ON(PageDirty(p));
4909			put_page(p);
4910		}
4911		attach_extent_buffer_page(eb, p);
4912		spin_unlock(&mapping->private_lock);
4913		WARN_ON(PageDirty(p));
4914		eb->pages[i] = p;
4915		if (!PageUptodate(p))
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4916			uptodate = 0;
4917
4918		/*
4919		 * see below about how we avoid a nasty race with release page
4920		 * and why we unlock later
 
 
 
4921		 */
4922	}
4923	if (uptodate)
4924		set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
 
 
 
4925again:
4926	ret = radix_tree_preload(GFP_NOFS & ~__GFP_HIGHMEM);
4927	if (ret)
4928		goto free_eb;
4929
4930	spin_lock(&fs_info->buffer_lock);
4931	ret = radix_tree_insert(&fs_info->buffer_radix,
4932				start >> PAGE_SHIFT, eb);
4933	spin_unlock(&fs_info->buffer_lock);
4934	radix_tree_preload_end();
4935	if (ret == -EEXIST) {
4936		exists = find_extent_buffer(fs_info, start);
4937		if (exists)
4938			goto free_eb;
 
4939		else
4940			goto again;
4941	}
4942	/* add one reference for the tree */
4943	check_buffer_tree_ref(eb);
4944	set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
4945
4946	/*
4947	 * there is a race where release page may have
4948	 * tried to find this extent buffer in the radix
4949	 * but failed.  It will tell the VM it is safe to
4950	 * reclaim the, and it will clear the page private bit.
4951	 * We must make sure to set the page private bit properly
4952	 * after the extent buffer is in the radix tree so
4953	 * it doesn't get lost
4954	 */
4955	SetPageChecked(eb->pages[0]);
4956	for (i = 1; i < num_pages; i++) {
4957		p = eb->pages[i];
4958		ClearPageChecked(p);
4959		unlock_page(p);
4960	}
4961	unlock_page(eb->pages[0]);
4962	return eb;
4963
4964free_eb:
4965	WARN_ON(!atomic_dec_and_test(&eb->refs));
4966	for (i = 0; i < num_pages; i++) {
4967		if (eb->pages[i])
4968			unlock_page(eb->pages[i]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4969	}
 
 
 
 
 
4970
4971	btrfs_release_extent_buffer(eb);
4972	return exists;
 
 
 
4973}
4974
4975static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
4976{
4977	struct extent_buffer *eb =
4978			container_of(head, struct extent_buffer, rcu_head);
4979
4980	__free_extent_buffer(eb);
4981}
4982
4983/* Expects to have eb->eb_lock already held */
4984static int release_extent_buffer(struct extent_buffer *eb)
 
4985{
 
 
4986	WARN_ON(atomic_read(&eb->refs) == 0);
4987	if (atomic_dec_and_test(&eb->refs)) {
4988		if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
4989			struct btrfs_fs_info *fs_info = eb->fs_info;
4990
4991			spin_unlock(&eb->refs_lock);
4992
4993			spin_lock(&fs_info->buffer_lock);
4994			radix_tree_delete(&fs_info->buffer_radix,
4995					  eb->start >> PAGE_SHIFT);
4996			spin_unlock(&fs_info->buffer_lock);
4997		} else {
4998			spin_unlock(&eb->refs_lock);
4999		}
5000
 
5001		/* Should be safe to release our pages at this point */
5002		btrfs_release_extent_buffer_page(eb);
5003#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
5004		if (unlikely(test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))) {
5005			__free_extent_buffer(eb);
5006			return 1;
5007		}
5008#endif
5009		call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
5010		return 1;
5011	}
5012	spin_unlock(&eb->refs_lock);
5013
5014	return 0;
5015}
5016
5017void free_extent_buffer(struct extent_buffer *eb)
5018{
5019	int refs;
5020	int old;
5021	if (!eb)
5022		return;
5023
 
5024	while (1) {
5025		refs = atomic_read(&eb->refs);
5026		if (refs <= 3)
 
5027			break;
5028		old = atomic_cmpxchg(&eb->refs, refs, refs - 1);
5029		if (old == refs)
5030			return;
5031	}
5032
5033	spin_lock(&eb->refs_lock);
5034	if (atomic_read(&eb->refs) == 2 &&
5035	    test_bit(EXTENT_BUFFER_DUMMY, &eb->bflags))
5036		atomic_dec(&eb->refs);
5037
5038	if (atomic_read(&eb->refs) == 2 &&
5039	    test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
5040	    !extent_buffer_under_io(eb) &&
5041	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5042		atomic_dec(&eb->refs);
5043
5044	/*
5045	 * I know this is terrible, but it's temporary until we stop tracking
5046	 * the uptodate bits and such for the extent buffers.
5047	 */
5048	release_extent_buffer(eb);
5049}
5050
5051void free_extent_buffer_stale(struct extent_buffer *eb)
5052{
5053	if (!eb)
5054		return;
5055
5056	spin_lock(&eb->refs_lock);
5057	set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
5058
5059	if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
5060	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
5061		atomic_dec(&eb->refs);
5062	release_extent_buffer(eb);
5063}
5064
5065void clear_extent_buffer_dirty(struct extent_buffer *eb)
5066{
5067	unsigned long i;
5068	unsigned long num_pages;
5069	struct page *page;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5070
5071	num_pages = num_extent_pages(eb->start, eb->len);
 
 
 
 
5072
5073	for (i = 0; i < num_pages; i++) {
5074		page = eb->pages[i];
5075		if (!PageDirty(page))
5076			continue;
5077
5078		lock_page(page);
5079		WARN_ON(!PagePrivate(page));
5080
5081		clear_page_dirty_for_io(page);
5082		spin_lock_irq(&page->mapping->tree_lock);
5083		if (!PageDirty(page)) {
5084			radix_tree_tag_clear(&page->mapping->page_tree,
5085						page_index(page),
5086						PAGECACHE_TAG_DIRTY);
5087		}
5088		spin_unlock_irq(&page->mapping->tree_lock);
5089		ClearPageError(page);
5090		unlock_page(page);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5091	}
5092	WARN_ON(atomic_read(&eb->refs) == 0);
5093}
5094
5095int set_extent_buffer_dirty(struct extent_buffer *eb)
5096{
5097	unsigned long i;
5098	unsigned long num_pages;
5099	int was_dirty = 0;
5100
5101	check_buffer_tree_ref(eb);
5102
5103	was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
5104
5105	num_pages = num_extent_pages(eb->start, eb->len);
5106	WARN_ON(atomic_read(&eb->refs) == 0);
5107	WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
 
 
 
 
5108
5109	for (i = 0; i < num_pages; i++)
5110		set_page_dirty(eb->pages[i]);
5111	return was_dirty;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5112}
5113
5114void clear_extent_buffer_uptodate(struct extent_buffer *eb)
5115{
5116	unsigned long i;
5117	struct page *page;
5118	unsigned long num_pages;
5119
5120	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5121	num_pages = num_extent_pages(eb->start, eb->len);
5122	for (i = 0; i < num_pages; i++) {
5123		page = eb->pages[i];
5124		if (page)
5125			ClearPageUptodate(page);
 
 
 
 
 
 
 
 
 
 
5126	}
5127}
5128
5129void set_extent_buffer_uptodate(struct extent_buffer *eb)
5130{
5131	unsigned long i;
5132	struct page *page;
5133	unsigned long num_pages;
5134
5135	set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5136	num_pages = num_extent_pages(eb->start, eb->len);
5137	for (i = 0; i < num_pages; i++) {
5138		page = eb->pages[i];
5139		SetPageUptodate(page);
 
 
 
 
 
 
 
 
5140	}
5141}
5142
5143int extent_buffer_uptodate(struct extent_buffer *eb)
5144{
5145	return test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5146}
5147
5148int read_extent_buffer_pages(struct extent_io_tree *tree,
5149			     struct extent_buffer *eb, u64 start, int wait,
5150			     get_extent_t *get_extent, int mirror_num)
5151{
5152	unsigned long i;
5153	unsigned long start_i;
5154	struct page *page;
5155	int err;
5156	int ret = 0;
5157	int locked_pages = 0;
5158	int all_uptodate = 1;
5159	unsigned long num_pages;
5160	unsigned long num_reads = 0;
5161	struct bio *bio = NULL;
5162	unsigned long bio_flags = 0;
5163
5164	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
5165		return 0;
5166
5167	if (start) {
5168		WARN_ON(start < eb->start);
5169		start_i = (start >> PAGE_SHIFT) -
5170			(eb->start >> PAGE_SHIFT);
5171	} else {
5172		start_i = 0;
5173	}
5174
5175	num_pages = num_extent_pages(eb->start, eb->len);
5176	for (i = start_i; i < num_pages; i++) {
5177		page = eb->pages[i];
5178		if (wait == WAIT_NONE) {
5179			if (!trylock_page(page))
5180				goto unlock_exit;
5181		} else {
5182			lock_page(page);
5183		}
5184		locked_pages++;
5185		if (!PageUptodate(page)) {
5186			num_reads++;
5187			all_uptodate = 0;
5188		}
5189	}
5190	if (all_uptodate) {
5191		if (start_i == 0)
5192			set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
5193		goto unlock_exit;
5194	}
5195
5196	clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
5197	eb->read_mirror = 0;
5198	atomic_set(&eb->io_pages, num_reads);
5199	for (i = start_i; i < num_pages; i++) {
5200		page = eb->pages[i];
5201		if (!PageUptodate(page)) {
5202			ClearPageError(page);
5203			err = __extent_read_full_page(tree, page,
5204						      get_extent, &bio,
5205						      mirror_num, &bio_flags,
5206						      READ | REQ_META);
5207			if (err)
5208				ret = err;
5209		} else {
5210			unlock_page(page);
 
 
 
 
 
 
 
 
 
5211		}
5212	}
 
5213
5214	if (bio) {
5215		err = submit_one_bio(READ | REQ_META, bio, mirror_num,
5216				     bio_flags);
5217		if (err)
5218			return err;
5219	}
5220
5221	if (ret || wait != WAIT_COMPLETE)
5222		return ret;
5223
5224	for (i = start_i; i < num_pages; i++) {
5225		page = eb->pages[i];
5226		wait_on_page_locked(page);
5227		if (!PageUptodate(page))
5228			ret = -EIO;
5229	}
 
5230
5231	return ret;
 
5232
5233unlock_exit:
5234	i = start_i;
5235	while (locked_pages > 0) {
5236		page = eb->pages[i];
5237		i++;
5238		unlock_page(page);
5239		locked_pages--;
5240	}
5241	return ret;
 
 
 
 
 
 
 
 
5242}
5243
5244void read_extent_buffer(struct extent_buffer *eb, void *dstv,
5245			unsigned long start,
5246			unsigned long len)
5247{
 
5248	size_t cur;
5249	size_t offset;
5250	struct page *page;
5251	char *kaddr;
5252	char *dst = (char *)dstv;
5253	size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5254	unsigned long i = (start_offset + start) >> PAGE_SHIFT;
5255
5256	WARN_ON(start > eb->len);
5257	WARN_ON(start + len > eb->start + eb->len);
 
 
 
 
 
 
5258
5259	offset = (start_offset + start) & (PAGE_SIZE - 1);
 
 
 
 
 
5260
5261	while (len > 0) {
5262		page = eb->pages[i];
5263
5264		cur = min(len, (PAGE_SIZE - offset));
5265		kaddr = page_address(page);
5266		memcpy(dst, kaddr + offset, cur);
5267
5268		dst += cur;
5269		len -= cur;
5270		offset = 0;
5271		i++;
5272	}
5273}
5274
5275int read_extent_buffer_to_user(struct extent_buffer *eb, void __user *dstv,
5276			unsigned long start,
5277			unsigned long len)
5278{
 
5279	size_t cur;
5280	size_t offset;
5281	struct page *page;
5282	char *kaddr;
5283	char __user *dst = (char __user *)dstv;
5284	size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5285	unsigned long i = (start_offset + start) >> PAGE_SHIFT;
5286	int ret = 0;
5287
5288	WARN_ON(start > eb->len);
5289	WARN_ON(start + len > eb->start + eb->len);
5290
5291	offset = (start_offset + start) & (PAGE_SIZE - 1);
 
 
 
 
 
 
5292
5293	while (len > 0) {
5294		page = eb->pages[i];
5295
5296		cur = min(len, (PAGE_SIZE - offset));
5297		kaddr = page_address(page);
5298		if (copy_to_user(dst, kaddr + offset, cur)) {
5299			ret = -EFAULT;
5300			break;
5301		}
5302
5303		dst += cur;
5304		len -= cur;
5305		offset = 0;
5306		i++;
5307	}
5308
5309	return ret;
5310}
5311
5312int map_private_extent_buffer(struct extent_buffer *eb, unsigned long start,
5313			       unsigned long min_len, char **map,
5314			       unsigned long *map_start,
5315			       unsigned long *map_len)
5316{
5317	size_t offset = start & (PAGE_SIZE - 1);
5318	char *kaddr;
5319	struct page *p;
5320	size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5321	unsigned long i = (start_offset + start) >> PAGE_SHIFT;
5322	unsigned long end_i = (start_offset + start + min_len - 1) >>
5323		PAGE_SHIFT;
5324
5325	if (i != end_i)
5326		return -EINVAL;
5327
5328	if (i == 0) {
5329		offset = start_offset;
5330		*map_start = 0;
5331	} else {
5332		offset = 0;
5333		*map_start = ((u64)i << PAGE_SHIFT) - start_offset;
5334	}
5335
5336	if (start + min_len > eb->len) {
5337		WARN(1, KERN_ERR "btrfs bad mapping eb start %llu len %lu, "
5338		       "wanted %lu %lu\n",
5339		       eb->start, eb->len, start, min_len);
5340		return -EINVAL;
5341	}
5342
5343	p = eb->pages[i];
5344	kaddr = page_address(p);
5345	*map = kaddr + offset;
5346	*map_len = PAGE_SIZE - offset;
5347	return 0;
5348}
5349
5350int memcmp_extent_buffer(struct extent_buffer *eb, const void *ptrv,
5351			  unsigned long start,
5352			  unsigned long len)
5353{
 
5354	size_t cur;
5355	size_t offset;
5356	struct page *page;
5357	char *kaddr;
5358	char *ptr = (char *)ptrv;
5359	size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5360	unsigned long i = (start_offset + start) >> PAGE_SHIFT;
5361	int ret = 0;
5362
5363	WARN_ON(start > eb->len);
5364	WARN_ON(start + len > eb->start + eb->len);
5365
5366	offset = (start_offset + start) & (PAGE_SIZE - 1);
5367
5368	while (len > 0) {
5369		page = eb->pages[i];
5370
5371		cur = min(len, (PAGE_SIZE - offset));
5372
5373		kaddr = page_address(page);
 
 
5374		ret = memcmp(ptr, kaddr + offset, cur);
5375		if (ret)
5376			break;
5377
5378		ptr += cur;
5379		len -= cur;
5380		offset = 0;
5381		i++;
5382	}
5383	return ret;
5384}
5385
5386void write_extent_buffer(struct extent_buffer *eb, const void *srcv,
5387			 unsigned long start, unsigned long len)
 
 
 
 
 
5388{
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5389	size_t cur;
5390	size_t offset;
5391	struct page *page;
5392	char *kaddr;
5393	char *src = (char *)srcv;
5394	size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5395	unsigned long i = (start_offset + start) >> PAGE_SHIFT;
 
5396
5397	WARN_ON(start > eb->len);
5398	WARN_ON(start + len > eb->start + eb->len);
5399
5400	offset = (start_offset + start) & (PAGE_SIZE - 1);
 
 
 
 
 
 
 
 
5401
5402	while (len > 0) {
5403		page = eb->pages[i];
5404		WARN_ON(!PageUptodate(page));
5405
5406		cur = min(len, PAGE_SIZE - offset);
5407		kaddr = page_address(page);
5408		memcpy(kaddr + offset, src, cur);
 
 
 
5409
5410		src += cur;
5411		len -= cur;
5412		offset = 0;
5413		i++;
5414	}
5415}
5416
5417void memset_extent_buffer(struct extent_buffer *eb, char c,
5418			  unsigned long start, unsigned long len)
5419{
5420	size_t cur;
5421	size_t offset;
5422	struct page *page;
5423	char *kaddr;
5424	size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5425	unsigned long i = (start_offset + start) >> PAGE_SHIFT;
5426
5427	WARN_ON(start > eb->len);
5428	WARN_ON(start + len > eb->start + eb->len);
 
 
 
5429
5430	offset = (start_offset + start) & (PAGE_SIZE - 1);
 
 
 
5431
5432	while (len > 0) {
5433		page = eb->pages[i];
5434		WARN_ON(!PageUptodate(page));
 
5435
5436		cur = min(len, PAGE_SIZE - offset);
5437		kaddr = page_address(page);
5438		memset(kaddr + offset, c, cur);
5439
5440		len -= cur;
5441		offset = 0;
5442		i++;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5443	}
5444}
5445
5446void copy_extent_buffer(struct extent_buffer *dst, struct extent_buffer *src,
 
5447			unsigned long dst_offset, unsigned long src_offset,
5448			unsigned long len)
5449{
 
5450	u64 dst_len = dst->len;
5451	size_t cur;
5452	size_t offset;
5453	struct page *page;
5454	char *kaddr;
5455	size_t start_offset = dst->start & ((u64)PAGE_SIZE - 1);
5456	unsigned long i = (start_offset + dst_offset) >> PAGE_SHIFT;
 
 
 
5457
5458	WARN_ON(src->len != dst_len);
5459
5460	offset = (start_offset + dst_offset) &
5461		(PAGE_SIZE - 1);
5462
5463	while (len > 0) {
5464		page = dst->pages[i];
5465		WARN_ON(!PageUptodate(page));
5466
5467		cur = min(len, (unsigned long)(PAGE_SIZE - offset));
5468
5469		kaddr = page_address(page);
5470		read_extent_buffer(src, kaddr + offset, src_offset, cur);
5471
5472		src_offset += cur;
5473		len -= cur;
5474		offset = 0;
5475		i++;
5476	}
5477}
5478
5479/*
5480 * The extent buffer bitmap operations are done with byte granularity because
5481 * bitmap items are not guaranteed to be aligned to a word and therefore a
5482 * single word in a bitmap may straddle two pages in the extent buffer.
5483 */
5484#define BIT_BYTE(nr) ((nr) / BITS_PER_BYTE)
5485#define BYTE_MASK ((1 << BITS_PER_BYTE) - 1)
5486#define BITMAP_FIRST_BYTE_MASK(start) \
5487	((BYTE_MASK << ((start) & (BITS_PER_BYTE - 1))) & BYTE_MASK)
5488#define BITMAP_LAST_BYTE_MASK(nbits) \
5489	(BYTE_MASK >> (-(nbits) & (BITS_PER_BYTE - 1)))
5490
5491/*
5492 * eb_bitmap_offset() - calculate the page and offset of the byte containing the
5493 * given bit number
5494 * @eb: the extent buffer
5495 * @start: offset of the bitmap item in the extent buffer
5496 * @nr: bit number
5497 * @page_index: return index of the page in the extent buffer that contains the
5498 * given bit number
5499 * @page_offset: return offset into the page given by page_index
5500 *
5501 * This helper hides the ugliness of finding the byte in an extent buffer which
5502 * contains a given bit.
5503 */
5504static inline void eb_bitmap_offset(struct extent_buffer *eb,
5505				    unsigned long start, unsigned long nr,
5506				    unsigned long *page_index,
5507				    size_t *page_offset)
5508{
5509	size_t start_offset = eb->start & ((u64)PAGE_SIZE - 1);
5510	size_t byte_offset = BIT_BYTE(nr);
5511	size_t offset;
5512
5513	/*
5514	 * The byte we want is the offset of the extent buffer + the offset of
5515	 * the bitmap item in the extent buffer + the offset of the byte in the
5516	 * bitmap item.
5517	 */
5518	offset = start_offset + start + byte_offset;
5519
5520	*page_index = offset >> PAGE_SHIFT;
5521	*page_offset = offset & (PAGE_SIZE - 1);
5522}
5523
5524/**
5525 * extent_buffer_test_bit - determine whether a bit in a bitmap item is set
5526 * @eb: the extent buffer
5527 * @start: offset of the bitmap item in the extent buffer
5528 * @nr: bit number to test
 
5529 */
5530int extent_buffer_test_bit(struct extent_buffer *eb, unsigned long start,
5531			   unsigned long nr)
5532{
5533	char *kaddr;
5534	struct page *page;
5535	unsigned long i;
5536	size_t offset;
 
5537
5538	eb_bitmap_offset(eb, start, nr, &i, &offset);
5539	page = eb->pages[i];
5540	WARN_ON(!PageUptodate(page));
5541	kaddr = page_address(page);
5542	return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
5543}
5544
5545/**
5546 * extent_buffer_bitmap_set - set an area of a bitmap
5547 * @eb: the extent buffer
5548 * @start: offset of the bitmap item in the extent buffer
5549 * @pos: bit number of the first bit
5550 * @len: number of bits to set
 
 
 
 
 
 
 
 
 
 
5551 */
5552void extent_buffer_bitmap_set(struct extent_buffer *eb, unsigned long start,
5553			      unsigned long pos, unsigned long len)
5554{
5555	char *kaddr;
5556	struct page *page;
5557	unsigned long i;
5558	size_t offset;
5559	const unsigned int size = pos + len;
5560	int bits_to_set = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
5561	unsigned int mask_to_set = BITMAP_FIRST_BYTE_MASK(pos);
5562
5563	eb_bitmap_offset(eb, start, pos, &i, &offset);
5564	page = eb->pages[i];
5565	WARN_ON(!PageUptodate(page));
5566	kaddr = page_address(page);
5567
5568	while (len >= bits_to_set) {
5569		kaddr[offset] |= mask_to_set;
5570		len -= bits_to_set;
5571		bits_to_set = BITS_PER_BYTE;
5572		mask_to_set = ~0U;
5573		if (++offset >= PAGE_SIZE && len > 0) {
5574			offset = 0;
5575			page = eb->pages[++i];
5576			WARN_ON(!PageUptodate(page));
5577			kaddr = page_address(page);
5578		}
5579	}
5580	if (len) {
5581		mask_to_set &= BITMAP_LAST_BYTE_MASK(size);
5582		kaddr[offset] |= mask_to_set;
5583	}
5584}
5585
5586
5587/**
5588 * extent_buffer_bitmap_clear - clear an area of a bitmap
5589 * @eb: the extent buffer
5590 * @start: offset of the bitmap item in the extent buffer
5591 * @pos: bit number of the first bit
5592 * @len: number of bits to clear
5593 */
5594void extent_buffer_bitmap_clear(struct extent_buffer *eb, unsigned long start,
5595				unsigned long pos, unsigned long len)
5596{
5597	char *kaddr;
5598	struct page *page;
5599	unsigned long i;
5600	size_t offset;
5601	const unsigned int size = pos + len;
5602	int bits_to_clear = BITS_PER_BYTE - (pos % BITS_PER_BYTE);
5603	unsigned int mask_to_clear = BITMAP_FIRST_BYTE_MASK(pos);
5604
5605	eb_bitmap_offset(eb, start, pos, &i, &offset);
5606	page = eb->pages[i];
5607	WARN_ON(!PageUptodate(page));
5608	kaddr = page_address(page);
5609
5610	while (len >= bits_to_clear) {
5611		kaddr[offset] &= ~mask_to_clear;
5612		len -= bits_to_clear;
5613		bits_to_clear = BITS_PER_BYTE;
5614		mask_to_clear = ~0U;
5615		if (++offset >= PAGE_SIZE && len > 0) {
5616			offset = 0;
5617			page = eb->pages[++i];
5618			WARN_ON(!PageUptodate(page));
5619			kaddr = page_address(page);
5620		}
5621	}
5622	if (len) {
5623		mask_to_clear &= BITMAP_LAST_BYTE_MASK(size);
5624		kaddr[offset] &= ~mask_to_clear;
5625	}
5626}
5627
5628static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
5629{
5630	unsigned long distance = (src > dst) ? src - dst : dst - src;
5631	return distance < len;
5632}
5633
5634static void copy_pages(struct page *dst_page, struct page *src_page,
5635		       unsigned long dst_off, unsigned long src_off,
5636		       unsigned long len)
5637{
5638	char *dst_kaddr = page_address(dst_page);
5639	char *src_kaddr;
5640	int must_memmove = 0;
5641
5642	if (dst_page != src_page) {
5643		src_kaddr = page_address(src_page);
5644	} else {
5645		src_kaddr = dst_kaddr;
5646		if (areas_overlap(src_off, dst_off, len))
5647			must_memmove = 1;
5648	}
5649
5650	if (must_memmove)
5651		memmove(dst_kaddr + dst_off, src_kaddr + src_off, len);
5652	else
5653		memcpy(dst_kaddr + dst_off, src_kaddr + src_off, len);
5654}
5655
5656void memcpy_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5657			   unsigned long src_offset, unsigned long len)
5658{
5659	size_t cur;
5660	size_t dst_off_in_page;
5661	size_t src_off_in_page;
5662	size_t start_offset = dst->start & ((u64)PAGE_SIZE - 1);
5663	unsigned long dst_i;
5664	unsigned long src_i;
5665
5666	if (src_offset + len > dst->len) {
5667		btrfs_err(dst->fs_info,
5668			"memmove bogus src_offset %lu move "
5669		       "len %lu dst len %lu", src_offset, len, dst->len);
5670		BUG_ON(1);
5671	}
5672	if (dst_offset + len > dst->len) {
5673		btrfs_err(dst->fs_info,
5674			"memmove bogus dst_offset %lu move "
5675		       "len %lu dst len %lu", dst_offset, len, dst->len);
5676		BUG_ON(1);
5677	}
5678
5679	while (len > 0) {
5680		dst_off_in_page = (start_offset + dst_offset) &
5681			(PAGE_SIZE - 1);
5682		src_off_in_page = (start_offset + src_offset) &
5683			(PAGE_SIZE - 1);
5684
5685		dst_i = (start_offset + dst_offset) >> PAGE_SHIFT;
5686		src_i = (start_offset + src_offset) >> PAGE_SHIFT;
5687
5688		cur = min(len, (unsigned long)(PAGE_SIZE -
5689					       src_off_in_page));
5690		cur = min_t(unsigned long, cur,
5691			(unsigned long)(PAGE_SIZE - dst_off_in_page));
5692
5693		copy_pages(dst->pages[dst_i], dst->pages[src_i],
5694			   dst_off_in_page, src_off_in_page, cur);
5695
5696		src_offset += cur;
5697		dst_offset += cur;
5698		len -= cur;
5699	}
5700}
5701
5702void memmove_extent_buffer(struct extent_buffer *dst, unsigned long dst_offset,
5703			   unsigned long src_offset, unsigned long len)
 
5704{
5705	size_t cur;
5706	size_t dst_off_in_page;
5707	size_t src_off_in_page;
5708	unsigned long dst_end = dst_offset + len - 1;
5709	unsigned long src_end = src_offset + len - 1;
5710	size_t start_offset = dst->start & ((u64)PAGE_SIZE - 1);
5711	unsigned long dst_i;
5712	unsigned long src_i;
5713
5714	if (src_offset + len > dst->len) {
5715		btrfs_err(dst->fs_info, "memmove bogus src_offset %lu move "
5716		       "len %lu len %lu", src_offset, len, dst->len);
5717		BUG_ON(1);
5718	}
5719	if (dst_offset + len > dst->len) {
5720		btrfs_err(dst->fs_info, "memmove bogus dst_offset %lu move "
5721		       "len %lu len %lu", dst_offset, len, dst->len);
5722		BUG_ON(1);
5723	}
5724	if (dst_offset < src_offset) {
5725		memcpy_extent_buffer(dst, dst_offset, src_offset, len);
5726		return;
5727	}
 
 
 
 
 
 
5728	while (len > 0) {
5729		dst_i = (start_offset + dst_end) >> PAGE_SHIFT;
5730		src_i = (start_offset + src_end) >> PAGE_SHIFT;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5731
5732		dst_off_in_page = (start_offset + dst_end) &
5733			(PAGE_SIZE - 1);
5734		src_off_in_page = (start_offset + src_end) &
5735			(PAGE_SIZE - 1);
5736
5737		cur = min_t(unsigned long, len, src_off_in_page + 1);
5738		cur = min(cur, dst_off_in_page + 1);
5739		copy_pages(dst->pages[dst_i], dst->pages[src_i],
5740			   dst_off_in_page - cur + 1,
5741			   src_off_in_page - cur + 1, cur);
5742
5743		dst_end -= cur;
5744		src_end -= cur;
5745		len -= cur;
5746	}
5747}
5748
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5749int try_release_extent_buffer(struct page *page)
5750{
 
5751	struct extent_buffer *eb;
5752
 
 
 
5753	/*
5754	 * We need to make sure noboody is attaching this page to an eb right
5755	 * now.
5756	 */
5757	spin_lock(&page->mapping->private_lock);
5758	if (!PagePrivate(page)) {
5759		spin_unlock(&page->mapping->private_lock);
5760		return 1;
5761	}
5762
5763	eb = (struct extent_buffer *)page->private;
5764	BUG_ON(!eb);
5765
5766	/*
5767	 * This is a little awful but should be ok, we need to make sure that
5768	 * the eb doesn't disappear out from under us while we're looking at
5769	 * this page.
5770	 */
5771	spin_lock(&eb->refs_lock);
5772	if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
5773		spin_unlock(&eb->refs_lock);
5774		spin_unlock(&page->mapping->private_lock);
5775		return 0;
5776	}
5777	spin_unlock(&page->mapping->private_lock);
5778
5779	/*
5780	 * If tree ref isn't set then we know the ref on this eb is a real ref,
5781	 * so just return, this page will likely be freed soon anyway.
5782	 */
5783	if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
5784		spin_unlock(&eb->refs_lock);
5785		return 0;
5786	}
5787
5788	return release_extent_buffer(eb);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
5789}