1// SPDX-License-Identifier: GPL-2.0
   2
   3#include "misc.h"
   4#include "ctree.h"
   5#include "block-group.h"
   6#include "space-info.h"
   7#include "disk-io.h"
   8#include "free-space-cache.h"
   9#include "free-space-tree.h"
  10#include "disk-io.h"
  11#include "volumes.h"
  12#include "transaction.h"
  13#include "ref-verify.h"
  14#include "sysfs.h"
  15#include "tree-log.h"
  16#include "delalloc-space.h"
  17
  18/*
  19 * Return target flags in extended format or 0 if restripe for this chunk_type
  20 * is not in progress
  21 *
  22 * Should be called with balance_lock held
  23 */
  24static u64 get_restripe_target(struct btrfs_fs_info *fs_info, u64 flags)
  25{
  26	struct btrfs_balance_control *bctl = fs_info->balance_ctl;
  27	u64 target = 0;
  28
  29	if (!bctl)
  30		return 0;
  31
  32	if (flags & BTRFS_BLOCK_GROUP_DATA &&
  33	    bctl->data.flags & BTRFS_BALANCE_ARGS_CONVERT) {
  34		target = BTRFS_BLOCK_GROUP_DATA | bctl->data.target;
  35	} else if (flags & BTRFS_BLOCK_GROUP_SYSTEM &&
  36		   bctl->sys.flags & BTRFS_BALANCE_ARGS_CONVERT) {
  37		target = BTRFS_BLOCK_GROUP_SYSTEM | bctl->sys.target;
  38	} else if (flags & BTRFS_BLOCK_GROUP_METADATA &&
  39		   bctl->meta.flags & BTRFS_BALANCE_ARGS_CONVERT) {
  40		target = BTRFS_BLOCK_GROUP_METADATA | bctl->meta.target;
  41	}
  42
  43	return target;
  44}
  45
  46/*
  47 * @flags: available profiles in extended format (see ctree.h)
  48 *
  49 * Return reduced profile in chunk format.  If profile changing is in progress
  50 * (either running or paused) picks the target profile (if it's already
  51 * available), otherwise falls back to plain reducing.
  52 */
  53static u64 btrfs_reduce_alloc_profile(struct btrfs_fs_info *fs_info, u64 flags)
  54{
  55	u64 num_devices = fs_info->fs_devices->rw_devices;
  56	u64 target;
  57	u64 raid_type;
  58	u64 allowed = 0;
  59
  60	/*
  61	 * See if restripe for this chunk_type is in progress, if so try to
  62	 * reduce to the target profile
  63	 */
  64	spin_lock(&fs_info->balance_lock);
  65	target = get_restripe_target(fs_info, flags);
  66	if (target) {
  67		/* Pick target profile only if it's already available */
  68		if ((flags & target) & BTRFS_EXTENDED_PROFILE_MASK) {
  69			spin_unlock(&fs_info->balance_lock);
  70			return extended_to_chunk(target);
  71		}
  72	}
  73	spin_unlock(&fs_info->balance_lock);
  74
  75	/* First, mask out the RAID levels which aren't possible */
  76	for (raid_type = 0; raid_type < BTRFS_NR_RAID_TYPES; raid_type++) {
  77		if (num_devices >= btrfs_raid_array[raid_type].devs_min)
  78			allowed |= btrfs_raid_array[raid_type].bg_flag;
  79	}
  80	allowed &= flags;
  81
  82	if (allowed & BTRFS_BLOCK_GROUP_RAID6)
  83		allowed = BTRFS_BLOCK_GROUP_RAID6;
  84	else if (allowed & BTRFS_BLOCK_GROUP_RAID5)
  85		allowed = BTRFS_BLOCK_GROUP_RAID5;
  86	else if (allowed & BTRFS_BLOCK_GROUP_RAID10)
  87		allowed = BTRFS_BLOCK_GROUP_RAID10;
  88	else if (allowed & BTRFS_BLOCK_GROUP_RAID1)
  89		allowed = BTRFS_BLOCK_GROUP_RAID1;
  90	else if (allowed & BTRFS_BLOCK_GROUP_RAID0)
  91		allowed = BTRFS_BLOCK_GROUP_RAID0;
  92
  93	flags &= ~BTRFS_BLOCK_GROUP_PROFILE_MASK;
  94
  95	return extended_to_chunk(flags | allowed);
  96}
  97
  98static u64 get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags)
  99{
 100	unsigned seq;
 101	u64 flags;
 102
 103	do {
 104		flags = orig_flags;
 105		seq = read_seqbegin(&fs_info->profiles_lock);
 106
 107		if (flags & BTRFS_BLOCK_GROUP_DATA)
 108			flags |= fs_info->avail_data_alloc_bits;
 109		else if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
 110			flags |= fs_info->avail_system_alloc_bits;
 111		else if (flags & BTRFS_BLOCK_GROUP_METADATA)
 112			flags |= fs_info->avail_metadata_alloc_bits;
 113	} while (read_seqretry(&fs_info->profiles_lock, seq));
 114
 115	return btrfs_reduce_alloc_profile(fs_info, flags);
 116}
 117
 118u64 btrfs_get_alloc_profile(struct btrfs_fs_info *fs_info, u64 orig_flags)
 119{
 120	return get_alloc_profile(fs_info, orig_flags);
 121}
 122
 123void btrfs_get_block_group(struct btrfs_block_group_cache *cache)
 124{
 125	atomic_inc(&cache->count);
 126}
 127
 128void btrfs_put_block_group(struct btrfs_block_group_cache *cache)
 129{
 130	if (atomic_dec_and_test(&cache->count)) {
 131		WARN_ON(cache->pinned > 0);
 132		WARN_ON(cache->reserved > 0);
 133
 134		/*
 135		 * If not empty, someone is still holding mutex of
 136		 * full_stripe_lock, which can only be released by caller.
 137		 * And it will definitely cause use-after-free when caller
 138		 * tries to release full stripe lock.
 139		 *
 140		 * No better way to resolve, but only to warn.
 141		 */
 142		WARN_ON(!RB_EMPTY_ROOT(&cache->full_stripe_locks_root.root));
 143		kfree(cache->free_space_ctl);
 144		kfree(cache);
 145	}
 146}
 147
 148/*
 149 * This adds the block group to the fs_info rb tree for the block group cache
 150 */
 151static int btrfs_add_block_group_cache(struct btrfs_fs_info *info,
 152				struct btrfs_block_group_cache *block_group)
 153{
 154	struct rb_node **p;
 155	struct rb_node *parent = NULL;
 156	struct btrfs_block_group_cache *cache;
 157
 158	spin_lock(&info->block_group_cache_lock);
 159	p = &info->block_group_cache_tree.rb_node;
 160
 161	while (*p) {
 162		parent = *p;
 163		cache = rb_entry(parent, struct btrfs_block_group_cache,
 164				 cache_node);
 165		if (block_group->key.objectid < cache->key.objectid) {
 166			p = &(*p)->rb_left;
 167		} else if (block_group->key.objectid > cache->key.objectid) {
 168			p = &(*p)->rb_right;
 169		} else {
 170			spin_unlock(&info->block_group_cache_lock);
 171			return -EEXIST;
 172		}
 173	}
 174
 175	rb_link_node(&block_group->cache_node, parent, p);
 176	rb_insert_color(&block_group->cache_node,
 177			&info->block_group_cache_tree);
 178
 179	if (info->first_logical_byte > block_group->key.objectid)
 180		info->first_logical_byte = block_group->key.objectid;
 181
 182	spin_unlock(&info->block_group_cache_lock);
 183
 184	return 0;
 185}
 186
 187/*
 188 * This will return the block group at or after bytenr if contains is 0, else
 189 * it will return the block group that contains the bytenr
 190 */
 191static struct btrfs_block_group_cache *block_group_cache_tree_search(
 192		struct btrfs_fs_info *info, u64 bytenr, int contains)
 193{
 194	struct btrfs_block_group_cache *cache, *ret = NULL;
 195	struct rb_node *n;
 196	u64 end, start;
 197
 198	spin_lock(&info->block_group_cache_lock);
 199	n = info->block_group_cache_tree.rb_node;
 200
 201	while (n) {
 202		cache = rb_entry(n, struct btrfs_block_group_cache,
 203				 cache_node);
 204		end = cache->key.objectid + cache->key.offset - 1;
 205		start = cache->key.objectid;
 206
 207		if (bytenr < start) {
 208			if (!contains && (!ret || start < ret->key.objectid))
 209				ret = cache;
 210			n = n->rb_left;
 211		} else if (bytenr > start) {
 212			if (contains && bytenr <= end) {
 213				ret = cache;
 214				break;
 215			}
 216			n = n->rb_right;
 217		} else {
 218			ret = cache;
 219			break;
 220		}
 221	}
 222	if (ret) {
 223		btrfs_get_block_group(ret);
 224		if (bytenr == 0 && info->first_logical_byte > ret->key.objectid)
 225			info->first_logical_byte = ret->key.objectid;
 226	}
 227	spin_unlock(&info->block_group_cache_lock);
 228
 229	return ret;
 230}
 231
 232/*
 233 * Return the block group that starts at or after bytenr
 234 */
 235struct btrfs_block_group_cache *btrfs_lookup_first_block_group(
 236		struct btrfs_fs_info *info, u64 bytenr)
 237{
 238	return block_group_cache_tree_search(info, bytenr, 0);
 239}
 240
 241/*
 242 * Return the block group that contains the given bytenr
 243 */
 244struct btrfs_block_group_cache *btrfs_lookup_block_group(
 245		struct btrfs_fs_info *info, u64 bytenr)
 246{
 247	return block_group_cache_tree_search(info, bytenr, 1);
 248}
 249
 250struct btrfs_block_group_cache *btrfs_next_block_group(
 251		struct btrfs_block_group_cache *cache)
 252{
 253	struct btrfs_fs_info *fs_info = cache->fs_info;
 254	struct rb_node *node;
 255
 256	spin_lock(&fs_info->block_group_cache_lock);
 257
 258	/* If our block group was removed, we need a full search. */
 259	if (RB_EMPTY_NODE(&cache->cache_node)) {
 260		const u64 next_bytenr = cache->key.objectid + cache->key.offset;
 261
 262		spin_unlock(&fs_info->block_group_cache_lock);
 263		btrfs_put_block_group(cache);
 264		cache = btrfs_lookup_first_block_group(fs_info, next_bytenr); return cache;
 265	}
 266	node = rb_next(&cache->cache_node);
 267	btrfs_put_block_group(cache);
 268	if (node) {
 269		cache = rb_entry(node, struct btrfs_block_group_cache,
 270				 cache_node);
 271		btrfs_get_block_group(cache);
 272	} else
 273		cache = NULL;
 274	spin_unlock(&fs_info->block_group_cache_lock);
 275	return cache;
 276}
 277
 278bool btrfs_inc_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
 279{
 280	struct btrfs_block_group_cache *bg;
 281	bool ret = true;
 282
 283	bg = btrfs_lookup_block_group(fs_info, bytenr);
 284	if (!bg)
 285		return false;
 286
 287	spin_lock(&bg->lock);
 288	if (bg->ro)
 289		ret = false;
 290	else
 291		atomic_inc(&bg->nocow_writers);
 292	spin_unlock(&bg->lock);
 293
 294	/* No put on block group, done by btrfs_dec_nocow_writers */
 295	if (!ret)
 296		btrfs_put_block_group(bg);
 297
 298	return ret;
 299}
 300
 301void btrfs_dec_nocow_writers(struct btrfs_fs_info *fs_info, u64 bytenr)
 302{
 303	struct btrfs_block_group_cache *bg;
 304
 305	bg = btrfs_lookup_block_group(fs_info, bytenr);
 306	ASSERT(bg);
 307	if (atomic_dec_and_test(&bg->nocow_writers))
 308		wake_up_var(&bg->nocow_writers);
 309	/*
 310	 * Once for our lookup and once for the lookup done by a previous call
 311	 * to btrfs_inc_nocow_writers()
 312	 */
 313	btrfs_put_block_group(bg);
 314	btrfs_put_block_group(bg);
 315}
 316
 317void btrfs_wait_nocow_writers(struct btrfs_block_group_cache *bg)
 318{
 319	wait_var_event(&bg->nocow_writers, !atomic_read(&bg->nocow_writers));
 320}
 321
 322void btrfs_dec_block_group_reservations(struct btrfs_fs_info *fs_info,
 323					const u64 start)
 324{
 325	struct btrfs_block_group_cache *bg;
 326
 327	bg = btrfs_lookup_block_group(fs_info, start);
 328	ASSERT(bg);
 329	if (atomic_dec_and_test(&bg->reservations))
 330		wake_up_var(&bg->reservations);
 331	btrfs_put_block_group(bg);
 332}
 333
 334void btrfs_wait_block_group_reservations(struct btrfs_block_group_cache *bg)
 335{
 336	struct btrfs_space_info *space_info = bg->space_info;
 337
 338	ASSERT(bg->ro);
 339
 340	if (!(bg->flags & BTRFS_BLOCK_GROUP_DATA))
 341		return;
 342
 343	/*
 344	 * Our block group is read only but before we set it to read only,
 345	 * some task might have had allocated an extent from it already, but it
 346	 * has not yet created a respective ordered extent (and added it to a
 347	 * root's list of ordered extents).
 348	 * Therefore wait for any task currently allocating extents, since the
 349	 * block group's reservations counter is incremented while a read lock
 350	 * on the groups' semaphore is held and decremented after releasing
 351	 * the read access on that semaphore and creating the ordered extent.
 352	 */
 353	down_write(&space_info->groups_sem);
 354	up_write(&space_info->groups_sem);
 355
 356	wait_var_event(&bg->reservations, !atomic_read(&bg->reservations));
 357}
 358
 359struct btrfs_caching_control *btrfs_get_caching_control(
 360		struct btrfs_block_group_cache *cache)
 361{
 362	struct btrfs_caching_control *ctl;
 363
 364	spin_lock(&cache->lock);
 365	if (!cache->caching_ctl) {
 366		spin_unlock(&cache->lock);
 367		return NULL;
 368	}
 369
 370	ctl = cache->caching_ctl;
 371	refcount_inc(&ctl->count);
 372	spin_unlock(&cache->lock);
 373	return ctl;
 374}
 375
 376void btrfs_put_caching_control(struct btrfs_caching_control *ctl)
 377{
 378	if (refcount_dec_and_test(&ctl->count))
 379		kfree(ctl);
 380}
 381
 382/*
 383 * When we wait for progress in the block group caching, its because our
 384 * allocation attempt failed at least once.  So, we must sleep and let some
 385 * progress happen before we try again.
 386 *
 387 * This function will sleep at least once waiting for new free space to show
 388 * up, and then it will check the block group free space numbers for our min
 389 * num_bytes.  Another option is to have it go ahead and look in the rbtree for
 390 * a free extent of a given size, but this is a good start.
 391 *
 392 * Callers of this must check if cache->cached == BTRFS_CACHE_ERROR before using
 393 * any of the information in this block group.
 394 */
 395void btrfs_wait_block_group_cache_progress(struct btrfs_block_group_cache *cache,
 396					   u64 num_bytes)
 397{
 398	struct btrfs_caching_control *caching_ctl;
 399
 400	caching_ctl = btrfs_get_caching_control(cache);
 401	if (!caching_ctl)
 402		return;
 403
 404	wait_event(caching_ctl->wait, btrfs_block_group_cache_done(cache) ||
 405		   (cache->free_space_ctl->free_space >= num_bytes));
 406
 407	btrfs_put_caching_control(caching_ctl);
 408}
 409
 410int btrfs_wait_block_group_cache_done(struct btrfs_block_group_cache *cache)
 411{
 412	struct btrfs_caching_control *caching_ctl;
 413	int ret = 0;
 414
 415	caching_ctl = btrfs_get_caching_control(cache);
 416	if (!caching_ctl)
 417		return (cache->cached == BTRFS_CACHE_ERROR) ? -EIO : 0;
 418
 419	wait_event(caching_ctl->wait, btrfs_block_group_cache_done(cache));
 420	if (cache->cached == BTRFS_CACHE_ERROR)
 421		ret = -EIO;
 422	btrfs_put_caching_control(caching_ctl);
 423	return ret;
 424}
 425
 426#ifdef CONFIG_BTRFS_DEBUG
 427static void fragment_free_space(struct btrfs_block_group_cache *block_group)
 428{
 429	struct btrfs_fs_info *fs_info = block_group->fs_info;
 430	u64 start = block_group->key.objectid;
 431	u64 len = block_group->key.offset;
 432	u64 chunk = block_group->flags & BTRFS_BLOCK_GROUP_METADATA ?
 433		fs_info->nodesize : fs_info->sectorsize;
 434	u64 step = chunk << 1;
 435
 436	while (len > chunk) {
 437		btrfs_remove_free_space(block_group, start, chunk);
 438		start += step;
 439		if (len < step)
 440			len = 0;
 441		else
 442			len -= step;
 443	}
 444}
 445#endif
 446
 447/*
 448 * This is only called by btrfs_cache_block_group, since we could have freed
 449 * extents we need to check the pinned_extents for any extents that can't be
 450 * used yet since their free space will be released as soon as the transaction
 451 * commits.
 452 */
 453u64 add_new_free_space(struct btrfs_block_group_cache *block_group,
 454		       u64 start, u64 end)
 455{
 456	struct btrfs_fs_info *info = block_group->fs_info;
 457	u64 extent_start, extent_end, size, total_added = 0;
 458	int ret;
 459
 460	while (start < end) {
 461		ret = find_first_extent_bit(info->pinned_extents, start,
 462					    &extent_start, &extent_end,
 463					    EXTENT_DIRTY | EXTENT_UPTODATE,
 464					    NULL);
 465		if (ret)
 466			break;
 467
 468		if (extent_start <= start) {
 469			start = extent_end + 1;
 470		} else if (extent_start > start && extent_start < end) {
 471			size = extent_start - start;
 472			total_added += size;
 473			ret = btrfs_add_free_space(block_group, start,
 474						   size);
 475			BUG_ON(ret); /* -ENOMEM or logic error */
 476			start = extent_end + 1;
 477		} else {
 478			break;
 479		}
 480	}
 481
 482	if (start < end) {
 483		size = end - start;
 484		total_added += size;
 485		ret = btrfs_add_free_space(block_group, start, size);
 486		BUG_ON(ret); /* -ENOMEM or logic error */
 487	}
 488
 489	return total_added;
 490}
 491
 492static int load_extent_tree_free(struct btrfs_caching_control *caching_ctl)
 493{
 494	struct btrfs_block_group_cache *block_group = caching_ctl->block_group;
 495	struct btrfs_fs_info *fs_info = block_group->fs_info;
 496	struct btrfs_root *extent_root = fs_info->extent_root;
 497	struct btrfs_path *path;
 498	struct extent_buffer *leaf;
 499	struct btrfs_key key;
 500	u64 total_found = 0;
 501	u64 last = 0;
 502	u32 nritems;
 503	int ret;
 504	bool wakeup = true;
 505
 506	path = btrfs_alloc_path();
 507	if (!path)
 508		return -ENOMEM;
 509
 510	last = max_t(u64, block_group->key.objectid, BTRFS_SUPER_INFO_OFFSET);
 511
 512#ifdef CONFIG_BTRFS_DEBUG
 513	/*
 514	 * If we're fragmenting we don't want to make anybody think we can
 515	 * allocate from this block group until we've had a chance to fragment
 516	 * the free space.
 517	 */
 518	if (btrfs_should_fragment_free_space(block_group))
 519		wakeup = false;
 520#endif
 521	/*
 522	 * We don't want to deadlock with somebody trying to allocate a new
 523	 * extent for the extent root while also trying to search the extent
 524	 * root to add free space.  So we skip locking and search the commit
 525	 * root, since its read-only
 526	 */
 527	path->skip_locking = 1;
 528	path->search_commit_root = 1;
 529	path->reada = READA_FORWARD;
 530
 531	key.objectid = last;
 532	key.offset = 0;
 533	key.type = BTRFS_EXTENT_ITEM_KEY;
 534
 535next:
 536	ret = btrfs_search_slot(NULL, extent_root, &key, path, 0, 0);
 537	if (ret < 0)
 538		goto out;
 539
 540	leaf = path->nodes[0];
 541	nritems = btrfs_header_nritems(leaf);
 542
 543	while (1) {
 544		if (btrfs_fs_closing(fs_info) > 1) {
 545			last = (u64)-1;
 546			break;
 547		}
 548
 549		if (path->slots[0] < nritems) {
 550			btrfs_item_key_to_cpu(leaf, &key, path->slots[0]);
 551		} else {
 552			ret = btrfs_find_next_key(extent_root, path, &key, 0, 0);
 553			if (ret)
 554				break;
 555
 556			if (need_resched() ||
 557			    rwsem_is_contended(&fs_info->commit_root_sem)) {
 558				if (wakeup)
 559					caching_ctl->progress = last;
 560				btrfs_release_path(path);
 561				up_read(&fs_info->commit_root_sem);
 562				mutex_unlock(&caching_ctl->mutex);
 563				cond_resched();
 564				mutex_lock(&caching_ctl->mutex);
 565				down_read(&fs_info->commit_root_sem);
 566				goto next;
 567			}
 568
 569			ret = btrfs_next_leaf(extent_root, path);
 570			if (ret < 0)
 571				goto out;
 572			if (ret)
 573				break;
 574			leaf = path->nodes[0];
 575			nritems = btrfs_header_nritems(leaf);
 576			continue;
 577		}
 578
 579		if (key.objectid < last) {
 580			key.objectid = last;
 581			key.offset = 0;
 582			key.type = BTRFS_EXTENT_ITEM_KEY;
 583
 584			if (wakeup)
 585				caching_ctl->progress = last;
 586			btrfs_release_path(path);
 587			goto next;
 588		}
 589
 590		if (key.objectid < block_group->key.objectid) {
 591			path->slots[0]++;
 592			continue;
 593		}
 594
 595		if (key.objectid >= block_group->key.objectid +
 596		    block_group->key.offset)
 597			break;
 598
 599		if (key.type == BTRFS_EXTENT_ITEM_KEY ||
 600		    key.type == BTRFS_METADATA_ITEM_KEY) {
 601			total_found += add_new_free_space(block_group, last,
 602							  key.objectid);
 603			if (key.type == BTRFS_METADATA_ITEM_KEY)
 604				last = key.objectid +
 605					fs_info->nodesize;
 606			else
 607				last = key.objectid + key.offset;
 608
 609			if (total_found > CACHING_CTL_WAKE_UP) {
 610				total_found = 0;
 611				if (wakeup)
 612					wake_up(&caching_ctl->wait);
 613			}
 614		}
 615		path->slots[0]++;
 616	}
 617	ret = 0;
 618
 619	total_found += add_new_free_space(block_group, last,
 620					  block_group->key.objectid +
 621					  block_group->key.offset);
 622	caching_ctl->progress = (u64)-1;
 623
 624out:
 625	btrfs_free_path(path);
 626	return ret;
 627}
 628
 629static noinline void caching_thread(struct btrfs_work *work)
 630{
 631	struct btrfs_block_group_cache *block_group;
 632	struct btrfs_fs_info *fs_info;
 633	struct btrfs_caching_control *caching_ctl;
 634	int ret;
 635
 636	caching_ctl = container_of(work, struct btrfs_caching_control, work);
 637	block_group = caching_ctl->block_group;
 638	fs_info = block_group->fs_info;
 639
 640	mutex_lock(&caching_ctl->mutex);
 641	down_read(&fs_info->commit_root_sem);
 642
 643	if (btrfs_fs_compat_ro(fs_info, FREE_SPACE_TREE))
 644		ret = load_free_space_tree(caching_ctl);
 645	else
 646		ret = load_extent_tree_free(caching_ctl);
 647
 648	spin_lock(&block_group->lock);
 649	block_group->caching_ctl = NULL;
 650	block_group->cached = ret ? BTRFS_CACHE_ERROR : BTRFS_CACHE_FINISHED;
 651	spin_unlock(&block_group->lock);
 652
 653#ifdef CONFIG_BTRFS_DEBUG
 654	if (btrfs_should_fragment_free_space(block_group)) {
 655		u64 bytes_used;
 656
 657		spin_lock(&block_group->space_info->lock);
 658		spin_lock(&block_group->lock);
 659		bytes_used = block_group->key.offset -
 660			btrfs_block_group_used(&block_group->item);
 661		block_group->space_info->bytes_used += bytes_used >> 1;
 662		spin_unlock(&block_group->lock);
 663		spin_unlock(&block_group->space_info->lock);
 664		fragment_free_space(block_group);
 665	}
 666#endif
 667
 668	caching_ctl->progress = (u64)-1;
 669
 670	up_read(&fs_info->commit_root_sem);
 671	btrfs_free_excluded_extents(block_group);
 672	mutex_unlock(&caching_ctl->mutex);
 673
 674	wake_up(&caching_ctl->wait);
 675
 676	btrfs_put_caching_control(caching_ctl);
 677	btrfs_put_block_group(block_group);
 678}
 679
 680int btrfs_cache_block_group(struct btrfs_block_group_cache *cache,
 681			    int load_cache_only)
 682{
 683	DEFINE_WAIT(wait);
 684	struct btrfs_fs_info *fs_info = cache->fs_info;
 685	struct btrfs_caching_control *caching_ctl;
 686	int ret = 0;
 687
 688	caching_ctl = kzalloc(sizeof(*caching_ctl), GFP_NOFS);
 689	if (!caching_ctl)
 690		return -ENOMEM;
 691
 692	INIT_LIST_HEAD(&caching_ctl->list);
 693	mutex_init(&caching_ctl->mutex);
 694	init_waitqueue_head(&caching_ctl->wait);
 695	caching_ctl->block_group = cache;
 696	caching_ctl->progress = cache->key.objectid;
 697	refcount_set(&caching_ctl->count, 1);
 698	btrfs_init_work(&caching_ctl->work, btrfs_cache_helper,
 699			caching_thread, NULL, NULL);
 700
 701	spin_lock(&cache->lock);
 702	/*
 703	 * This should be a rare occasion, but this could happen I think in the
 704	 * case where one thread starts to load the space cache info, and then
 705	 * some other thread starts a transaction commit which tries to do an
 706	 * allocation while the other thread is still loading the space cache
 707	 * info.  The previous loop should have kept us from choosing this block
 708	 * group, but if we've moved to the state where we will wait on caching
 709	 * block groups we need to first check if we're doing a fast load here,
 710	 * so we can wait for it to finish, otherwise we could end up allocating
 711	 * from a block group who's cache gets evicted for one reason or
 712	 * another.
 713	 */
 714	while (cache->cached == BTRFS_CACHE_FAST) {
 715		struct btrfs_caching_control *ctl;
 716
 717		ctl = cache->caching_ctl;
 718		refcount_inc(&ctl->count);
 719		prepare_to_wait(&ctl->wait, &wait, TASK_UNINTERRUPTIBLE);
 720		spin_unlock(&cache->lock);
 721
 722		schedule();
 723
 724		finish_wait(&ctl->wait, &wait);
 725		btrfs_put_caching_control(ctl);
 726		spin_lock(&cache->lock);
 727	}
 728
 729	if (cache->cached != BTRFS_CACHE_NO) {
 730		spin_unlock(&cache->lock);
 731		kfree(caching_ctl);
 732		return 0;
 733	}
 734	WARN_ON(cache->caching_ctl);
 735	cache->caching_ctl = caching_ctl;
 736	cache->cached = BTRFS_CACHE_FAST;
 737	spin_unlock(&cache->lock);
 738
 739	if (btrfs_test_opt(fs_info, SPACE_CACHE)) {
 740		mutex_lock(&caching_ctl->mutex);
 741		ret = load_free_space_cache(cache);
 742
 743		spin_lock(&cache->lock);
 744		if (ret == 1) {
 745			cache->caching_ctl = NULL;
 746			cache->cached = BTRFS_CACHE_FINISHED;
 747			cache->last_byte_to_unpin = (u64)-1;
 748			caching_ctl->progress = (u64)-1;
 749		} else {
 750			if (load_cache_only) {
 751				cache->caching_ctl = NULL;
 752				cache->cached = BTRFS_CACHE_NO;
 753			} else {
 754				cache->cached = BTRFS_CACHE_STARTED;
 755				cache->has_caching_ctl = 1;
 756			}
 757		}
 758		spin_unlock(&cache->lock);
 759#ifdef CONFIG_BTRFS_DEBUG
 760		if (ret == 1 &&
 761		    btrfs_should_fragment_free_space(cache)) {
 762			u64 bytes_used;
 763
 764			spin_lock(&cache->space_info->lock);
 765			spin_lock(&cache->lock);
 766			bytes_used = cache->key.offset -
 767				btrfs_block_group_used(&cache->item);
 768			cache->space_info->bytes_used += bytes_used >> 1;
 769			spin_unlock(&cache->lock);
 770			spin_unlock(&cache->space_info->lock);
 771			fragment_free_space(cache);
 772		}
 773#endif
 774		mutex_unlock(&caching_ctl->mutex);
 775
 776		wake_up(&caching_ctl->wait);
 777		if (ret == 1) {
 778			btrfs_put_caching_control(caching_ctl);
 779			btrfs_free_excluded_extents(cache);
 780			return 0;
 781		}
 782	} else {
 783		/*
 784		 * We're either using the free space tree or no caching at all.
 785		 * Set cached to the appropriate value and wakeup any waiters.
 786		 */
 787		spin_lock(&cache->lock);
 788		if (load_cache_only) {
 789			cache->caching_ctl = NULL;
 790			cache->cached = BTRFS_CACHE_NO;
 791		} else {
 792			cache->cached = BTRFS_CACHE_STARTED;
 793			cache->has_caching_ctl = 1;
 794		}
 795		spin_unlock(&cache->lock);
 796		wake_up(&caching_ctl->wait);
 797	}
 798
 799	if (load_cache_only) {
 800		btrfs_put_caching_control(caching_ctl);
 801		return 0;
 802	}
 803
 804	down_write(&fs_info->commit_root_sem);
 805	refcount_inc(&caching_ctl->count);
 806	list_add_tail(&caching_ctl->list, &fs_info->caching_block_groups);
 807	up_write(&fs_info->commit_root_sem);
 808
 809	btrfs_get_block_group(cache);
 810
 811	btrfs_queue_work(fs_info->caching_workers, &caching_ctl->work);
 812
 813	return ret;
 814}
 815
 816static void clear_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
 817{
 818	u64 extra_flags = chunk_to_extended(flags) &
 819				BTRFS_EXTENDED_PROFILE_MASK;
 820
 821	write_seqlock(&fs_info->profiles_lock);
 822	if (flags & BTRFS_BLOCK_GROUP_DATA)
 823		fs_info->avail_data_alloc_bits &= ~extra_flags;
 824	if (flags & BTRFS_BLOCK_GROUP_METADATA)
 825		fs_info->avail_metadata_alloc_bits &= ~extra_flags;
 826	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
 827		fs_info->avail_system_alloc_bits &= ~extra_flags;
 828	write_sequnlock(&fs_info->profiles_lock);
 829}
 830
 831/*
 832 * Clear incompat bits for the following feature(s):
 833 *
 834 * - RAID56 - in case there's neither RAID5 nor RAID6 profile block group
 835 *            in the whole filesystem
 836 */
 837static void clear_incompat_bg_bits(struct btrfs_fs_info *fs_info, u64 flags)
 838{
 839	if (flags & BTRFS_BLOCK_GROUP_RAID56_MASK) {
 840		struct list_head *head = &fs_info->space_info;
 841		struct btrfs_space_info *sinfo;
 842
 843		list_for_each_entry_rcu(sinfo, head, list) {
 844			bool found = false;
 845
 846			down_read(&sinfo->groups_sem);
 847			if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID5]))
 848				found = true;
 849			if (!list_empty(&sinfo->block_groups[BTRFS_RAID_RAID6]))
 850				found = true;
 851			up_read(&sinfo->groups_sem);
 852
 853			if (found)
 854				return;
 855		}
 856		btrfs_clear_fs_incompat(fs_info, RAID56);
 857	}
 858}
 859
 860int btrfs_remove_block_group(struct btrfs_trans_handle *trans,
 861			     u64 group_start, struct extent_map *em)
 862{
 863	struct btrfs_fs_info *fs_info = trans->fs_info;
 864	struct btrfs_root *root = fs_info->extent_root;
 865	struct btrfs_path *path;
 866	struct btrfs_block_group_cache *block_group;
 867	struct btrfs_free_cluster *cluster;
 868	struct btrfs_root *tree_root = fs_info->tree_root;
 869	struct btrfs_key key;
 870	struct inode *inode;
 871	struct kobject *kobj = NULL;
 872	int ret;
 873	int index;
 874	int factor;
 875	struct btrfs_caching_control *caching_ctl = NULL;
 876	bool remove_em;
 877	bool remove_rsv = false;
 878
 879	block_group = btrfs_lookup_block_group(fs_info, group_start);
 880	BUG_ON(!block_group);
 881	BUG_ON(!block_group->ro);
 882
 883	trace_btrfs_remove_block_group(block_group);
 884	/*
 885	 * Free the reserved super bytes from this block group before
 886	 * remove it.
 887	 */
 888	btrfs_free_excluded_extents(block_group);
 889	btrfs_free_ref_tree_range(fs_info, block_group->key.objectid,
 890				  block_group->key.offset);
 891
 892	memcpy(&key, &block_group->key, sizeof(key));
 893	index = btrfs_bg_flags_to_raid_index(block_group->flags);
 894	factor = btrfs_bg_type_to_factor(block_group->flags);
 895
 896	/* make sure this block group isn't part of an allocation cluster */
 897	cluster = &fs_info->data_alloc_cluster;
 898	spin_lock(&cluster->refill_lock);
 899	btrfs_return_cluster_to_free_space(block_group, cluster);
 900	spin_unlock(&cluster->refill_lock);
 901
 902	/*
 903	 * make sure this block group isn't part of a metadata
 904	 * allocation cluster
 905	 */
 906	cluster = &fs_info->meta_alloc_cluster;
 907	spin_lock(&cluster->refill_lock);
 908	btrfs_return_cluster_to_free_space(block_group, cluster);
 909	spin_unlock(&cluster->refill_lock);
 910
 911	path = btrfs_alloc_path();
 912	if (!path) {
 913		ret = -ENOMEM;
 914		goto out;
 915	}
 916
 917	/*
 918	 * get the inode first so any iput calls done for the io_list
 919	 * aren't the final iput (no unlinks allowed now)
 920	 */
 921	inode = lookup_free_space_inode(block_group, path);
 922
 923	mutex_lock(&trans->transaction->cache_write_mutex);
 924	/*
 925	 * Make sure our free space cache IO is done before removing the
 926	 * free space inode
 927	 */
 928	spin_lock(&trans->transaction->dirty_bgs_lock);
 929	if (!list_empty(&block_group->io_list)) {
 930		list_del_init(&block_group->io_list);
 931
 932		WARN_ON(!IS_ERR(inode) && inode != block_group->io_ctl.inode);
 933
 934		spin_unlock(&trans->transaction->dirty_bgs_lock);
 935		btrfs_wait_cache_io(trans, block_group, path);
 936		btrfs_put_block_group(block_group);
 937		spin_lock(&trans->transaction->dirty_bgs_lock);
 938	}
 939
 940	if (!list_empty(&block_group->dirty_list)) {
 941		list_del_init(&block_group->dirty_list);
 942		remove_rsv = true;
 943		btrfs_put_block_group(block_group);
 944	}
 945	spin_unlock(&trans->transaction->dirty_bgs_lock);
 946	mutex_unlock(&trans->transaction->cache_write_mutex);
 947
 948	if (!IS_ERR(inode)) {
 949		ret = btrfs_orphan_add(trans, BTRFS_I(inode));
 950		if (ret) {
 951			btrfs_add_delayed_iput(inode);
 952			goto out;
 953		}
 954		clear_nlink(inode);
 955		/* One for the block groups ref */
 956		spin_lock(&block_group->lock);
 957		if (block_group->iref) {
 958			block_group->iref = 0;
 959			block_group->inode = NULL;
 960			spin_unlock(&block_group->lock);
 961			iput(inode);
 962		} else {
 963			spin_unlock(&block_group->lock);
 964		}
 965		/* One for our lookup ref */
 966		btrfs_add_delayed_iput(inode);
 967	}
 968
 969	key.objectid = BTRFS_FREE_SPACE_OBJECTID;
 970	key.offset = block_group->key.objectid;
 971	key.type = 0;
 972
 973	ret = btrfs_search_slot(trans, tree_root, &key, path, -1, 1);
 974	if (ret < 0)
 975		goto out;
 976	if (ret > 0)
 977		btrfs_release_path(path);
 978	if (ret == 0) {
 979		ret = btrfs_del_item(trans, tree_root, path);
 980		if (ret)
 981			goto out;
 982		btrfs_release_path(path);
 983	}
 984
 985	spin_lock(&fs_info->block_group_cache_lock);
 986	rb_erase(&block_group->cache_node,
 987		 &fs_info->block_group_cache_tree);
 988	RB_CLEAR_NODE(&block_group->cache_node);
 989
 990	if (fs_info->first_logical_byte == block_group->key.objectid)
 991		fs_info->first_logical_byte = (u64)-1;
 992	spin_unlock(&fs_info->block_group_cache_lock);
 993
 994	down_write(&block_group->space_info->groups_sem);
 995	/*
 996	 * we must use list_del_init so people can check to see if they
 997	 * are still on the list after taking the semaphore
 998	 */
 999	list_del_init(&block_group->list);
1000	if (list_empty(&block_group->space_info->block_groups[index])) {
1001		kobj = block_group->space_info->block_group_kobjs[index];
1002		block_group->space_info->block_group_kobjs[index] = NULL;
1003		clear_avail_alloc_bits(fs_info, block_group->flags);
1004	}
1005	up_write(&block_group->space_info->groups_sem);
1006	clear_incompat_bg_bits(fs_info, block_group->flags);
1007	if (kobj) {
1008		kobject_del(kobj);
1009		kobject_put(kobj);
1010	}
1011
1012	if (block_group->has_caching_ctl)
1013		caching_ctl = btrfs_get_caching_control(block_group);
1014	if (block_group->cached == BTRFS_CACHE_STARTED)
1015		btrfs_wait_block_group_cache_done(block_group);
1016	if (block_group->has_caching_ctl) {
1017		down_write(&fs_info->commit_root_sem);
1018		if (!caching_ctl) {
1019			struct btrfs_caching_control *ctl;
1020
1021			list_for_each_entry(ctl,
1022				    &fs_info->caching_block_groups, list)
1023				if (ctl->block_group == block_group) {
1024					caching_ctl = ctl;
1025					refcount_inc(&caching_ctl->count);
1026					break;
1027				}
1028		}
1029		if (caching_ctl)
1030			list_del_init(&caching_ctl->list);
1031		up_write(&fs_info->commit_root_sem);
1032		if (caching_ctl) {
1033			/* Once for the caching bgs list and once for us. */
1034			btrfs_put_caching_control(caching_ctl);
1035			btrfs_put_caching_control(caching_ctl);
1036		}
1037	}
1038
1039	spin_lock(&trans->transaction->dirty_bgs_lock);
1040	WARN_ON(!list_empty(&block_group->dirty_list));
1041	WARN_ON(!list_empty(&block_group->io_list));
1042	spin_unlock(&trans->transaction->dirty_bgs_lock);
1043
1044	btrfs_remove_free_space_cache(block_group);
1045
1046	spin_lock(&block_group->space_info->lock);
1047	list_del_init(&block_group->ro_list);
1048
1049	if (btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
1050		WARN_ON(block_group->space_info->total_bytes
1051			< block_group->key.offset);
1052		WARN_ON(block_group->space_info->bytes_readonly
1053			< block_group->key.offset);
1054		WARN_ON(block_group->space_info->disk_total
1055			< block_group->key.offset * factor);
1056	}
1057	block_group->space_info->total_bytes -= block_group->key.offset;
1058	block_group->space_info->bytes_readonly -= block_group->key.offset;
1059	block_group->space_info->disk_total -= block_group->key.offset * factor;
1060
1061	spin_unlock(&block_group->space_info->lock);
1062
1063	memcpy(&key, &block_group->key, sizeof(key));
1064
1065	mutex_lock(&fs_info->chunk_mutex);
1066	spin_lock(&block_group->lock);
1067	block_group->removed = 1;
1068	/*
1069	 * At this point trimming can't start on this block group, because we
1070	 * removed the block group from the tree fs_info->block_group_cache_tree
1071	 * so no one can't find it anymore and even if someone already got this
1072	 * block group before we removed it from the rbtree, they have already
1073	 * incremented block_group->trimming - if they didn't, they won't find
1074	 * any free space entries because we already removed them all when we
1075	 * called btrfs_remove_free_space_cache().
1076	 *
1077	 * And we must not remove the extent map from the fs_info->mapping_tree
1078	 * to prevent the same logical address range and physical device space
1079	 * ranges from being reused for a new block group. This is because our
1080	 * fs trim operation (btrfs_trim_fs() / btrfs_ioctl_fitrim()) is
1081	 * completely transactionless, so while it is trimming a range the
1082	 * currently running transaction might finish and a new one start,
1083	 * allowing for new block groups to be created that can reuse the same
1084	 * physical device locations unless we take this special care.
1085	 *
1086	 * There may also be an implicit trim operation if the file system
1087	 * is mounted with -odiscard. The same protections must remain
1088	 * in place until the extents have been discarded completely when
1089	 * the transaction commit has completed.
1090	 */
1091	remove_em = (atomic_read(&block_group->trimming) == 0);
1092	spin_unlock(&block_group->lock);
1093
1094	mutex_unlock(&fs_info->chunk_mutex);
1095
1096	ret = remove_block_group_free_space(trans, block_group);
1097	if (ret)
1098		goto out;
1099
1100	btrfs_put_block_group(block_group);
1101	btrfs_put_block_group(block_group);
1102
1103	ret = btrfs_search_slot(trans, root, &key, path, -1, 1);
1104	if (ret > 0)
1105		ret = -EIO;
1106	if (ret < 0)
1107		goto out;
1108
1109	ret = btrfs_del_item(trans, root, path);
1110	if (ret)
1111		goto out;
1112
1113	if (remove_em) {
1114		struct extent_map_tree *em_tree;
1115
1116		em_tree = &fs_info->mapping_tree;
1117		write_lock(&em_tree->lock);
1118		remove_extent_mapping(em_tree, em);
1119		write_unlock(&em_tree->lock);
1120		/* once for the tree */
1121		free_extent_map(em);
1122	}
1123out:
1124	if (remove_rsv)
1125		btrfs_delayed_refs_rsv_release(fs_info, 1);
1126	btrfs_free_path(path);
1127	return ret;
1128}
1129
1130struct btrfs_trans_handle *btrfs_start_trans_remove_block_group(
1131		struct btrfs_fs_info *fs_info, const u64 chunk_offset)
1132{
1133	struct extent_map_tree *em_tree = &fs_info->mapping_tree;
1134	struct extent_map *em;
1135	struct map_lookup *map;
1136	unsigned int num_items;
1137
1138	read_lock(&em_tree->lock);
1139	em = lookup_extent_mapping(em_tree, chunk_offset, 1);
1140	read_unlock(&em_tree->lock);
1141	ASSERT(em && em->start == chunk_offset);
1142
1143	/*
1144	 * We need to reserve 3 + N units from the metadata space info in order
1145	 * to remove a block group (done at btrfs_remove_chunk() and at
1146	 * btrfs_remove_block_group()), which are used for:
1147	 *
1148	 * 1 unit for adding the free space inode's orphan (located in the tree
1149	 * of tree roots).
1150	 * 1 unit for deleting the block group item (located in the extent
1151	 * tree).
1152	 * 1 unit for deleting the free space item (located in tree of tree
1153	 * roots).
1154	 * N units for deleting N device extent items corresponding to each
1155	 * stripe (located in the device tree).
1156	 *
1157	 * In order to remove a block group we also need to reserve units in the
1158	 * system space info in order to update the chunk tree (update one or
1159	 * more device items and remove one chunk item), but this is done at
1160	 * btrfs_remove_chunk() through a call to check_system_chunk().
1161	 */
1162	map = em->map_lookup;
1163	num_items = 3 + map->num_stripes;
1164	free_extent_map(em);
1165
1166	return btrfs_start_transaction_fallback_global_rsv(fs_info->extent_root,
1167							   num_items, 1);
1168}
1169
1170/*
1171 * Mark block group @cache read-only, so later write won't happen to block
1172 * group @cache.
1173 *
1174 * If @force is not set, this function will only mark the block group readonly
1175 * if we have enough free space (1M) in other metadata/system block groups.
1176 * If @force is not set, this function will mark the block group readonly
1177 * without checking free space.
1178 *
1179 * NOTE: This function doesn't care if other block groups can contain all the
1180 * data in this block group. That check should be done by relocation routine,
1181 * not this function.
1182 */
1183static int inc_block_group_ro(struct btrfs_block_group_cache *cache, int force)
1184{
1185	struct btrfs_space_info *sinfo = cache->space_info;
1186	u64 num_bytes;
1187	u64 sinfo_used;
1188	u64 min_allocable_bytes;
1189	int ret = -ENOSPC;
1190
1191	/*
1192	 * We need some metadata space and system metadata space for
1193	 * allocating chunks in some corner cases until we force to set
1194	 * it to be readonly.
1195	 */
1196	if ((sinfo->flags &
1197	     (BTRFS_BLOCK_GROUP_SYSTEM | BTRFS_BLOCK_GROUP_METADATA)) &&
1198	    !force)
1199		min_allocable_bytes = SZ_1M;
1200	else
1201		min_allocable_bytes = 0;
1202
1203	spin_lock(&sinfo->lock);
1204	spin_lock(&cache->lock);
1205
1206	if (cache->ro) {
1207		cache->ro++;
1208		ret = 0;
1209		goto out;
1210	}
1211
1212	num_bytes = cache->key.offset - cache->reserved - cache->pinned -
1213		    cache->bytes_super - btrfs_block_group_used(&cache->item);
1214	sinfo_used = btrfs_space_info_used(sinfo, true);
1215
1216	/*
1217	 * sinfo_used + num_bytes should always <= sinfo->total_bytes.
1218	 *
1219	 * Here we make sure if we mark this bg RO, we still have enough
1220	 * free space as buffer (if min_allocable_bytes is not 0).
1221	 */
1222	if (sinfo_used + num_bytes + min_allocable_bytes <=
1223	    sinfo->total_bytes) {
1224		sinfo->bytes_readonly += num_bytes;
1225		cache->ro++;
1226		list_add_tail(&cache->ro_list, &sinfo->ro_bgs);
1227		ret = 0;
1228	}
1229out:
1230	spin_unlock(&cache->lock);
1231	spin_unlock(&sinfo->lock);
1232	if (ret == -ENOSPC && btrfs_test_opt(cache->fs_info, ENOSPC_DEBUG)) {
1233		btrfs_info(cache->fs_info,
1234			"unable to make block group %llu ro",
1235			cache->key.objectid);
1236		btrfs_info(cache->fs_info,
1237			"sinfo_used=%llu bg_num_bytes=%llu min_allocable=%llu",
1238			sinfo_used, num_bytes, min_allocable_bytes);
1239		btrfs_dump_space_info(cache->fs_info, cache->space_info, 0, 0);
1240	}
1241	return ret;
1242}
1243
1244/*
1245 * Process the unused_bgs list and remove any that don't have any allocated
1246 * space inside of them.
1247 */
1248void btrfs_delete_unused_bgs(struct btrfs_fs_info *fs_info)
1249{
1250	struct btrfs_block_group_cache *block_group;
1251	struct btrfs_space_info *space_info;
1252	struct btrfs_trans_handle *trans;
1253	int ret = 0;
1254
1255	if (!test_bit(BTRFS_FS_OPEN, &fs_info->flags))
1256		return;
1257
1258	spin_lock(&fs_info->unused_bgs_lock);
1259	while (!list_empty(&fs_info->unused_bgs)) {
1260		u64 start, end;
1261		int trimming;
1262
1263		block_group = list_first_entry(&fs_info->unused_bgs,
1264					       struct btrfs_block_group_cache,
1265					       bg_list);
1266		list_del_init(&block_group->bg_list);
1267
1268		space_info = block_group->space_info;
1269
1270		if (ret || btrfs_mixed_space_info(space_info)) {
1271			btrfs_put_block_group(block_group);
1272			continue;
1273		}
1274		spin_unlock(&fs_info->unused_bgs_lock);
1275
1276		mutex_lock(&fs_info->delete_unused_bgs_mutex);
1277
1278		/* Don't want to race with allocators so take the groups_sem */
1279		down_write(&space_info->groups_sem);
1280		spin_lock(&block_group->lock);
1281		if (block_group->reserved || block_group->pinned ||
1282		    btrfs_block_group_used(&block_group->item) ||
1283		    block_group->ro ||
1284		    list_is_singular(&block_group->list)) {
1285			/*
1286			 * We want to bail if we made new allocations or have
1287			 * outstanding allocations in this block group.  We do
1288			 * the ro check in case balance is currently acting on
1289			 * this block group.
1290			 */
1291			trace_btrfs_skip_unused_block_group(block_group);
1292			spin_unlock(&block_group->lock);
1293			up_write(&space_info->groups_sem);
1294			goto next;
1295		}
1296		spin_unlock(&block_group->lock);
1297
1298		/* We don't want to force the issue, only flip if it's ok. */
1299		ret = inc_block_group_ro(block_group, 0);
1300		up_write(&space_info->groups_sem);
1301		if (ret < 0) {
1302			ret = 0;
1303			goto next;
1304		}
1305
1306		/*
1307		 * Want to do this before we do anything else so we can recover
1308		 * properly if we fail to join the transaction.
1309		 */
1310		trans = btrfs_start_trans_remove_block_group(fs_info,
1311						     block_group->key.objectid);
1312		if (IS_ERR(trans)) {
1313			btrfs_dec_block_group_ro(block_group);
1314			ret = PTR_ERR(trans);
1315			goto next;
1316		}
1317
1318		/*
1319		 * We could have pending pinned extents for this block group,
1320		 * just delete them, we don't care about them anymore.
1321		 */
1322		start = block_group->key.objectid;
1323		end = start + block_group->key.offset - 1;
1324		/*
1325		 * Hold the unused_bg_unpin_mutex lock to avoid racing with
1326		 * btrfs_finish_extent_commit(). If we are at transaction N,
1327		 * another task might be running finish_extent_commit() for the
1328		 * previous transaction N - 1, and have seen a range belonging
1329		 * to the block group in freed_extents[] before we were able to
1330		 * clear the whole block group range from freed_extents[]. This
1331		 * means that task can lookup for the block group after we
1332		 * unpinned it from freed_extents[] and removed it, leading to
1333		 * a BUG_ON() at btrfs_unpin_extent_range().
1334		 */
1335		mutex_lock(&fs_info->unused_bg_unpin_mutex);
1336		ret = clear_extent_bits(&fs_info->freed_extents[0], start, end,
1337				  EXTENT_DIRTY);
1338		if (ret) {
1339			mutex_unlock(&fs_info->unused_bg_unpin_mutex);
1340			btrfs_dec_block_group_ro(block_group);
1341			goto end_trans;
1342		}
1343		ret = clear_extent_bits(&fs_info->freed_extents[1], start, end,
1344				  EXTENT_DIRTY);
1345		if (ret) {
1346			mutex_unlock(&fs_info->unused_bg_unpin_mutex);
1347			btrfs_dec_block_group_ro(block_group);
1348			goto end_trans;
1349		}
1350		mutex_unlock(&fs_info->unused_bg_unpin_mutex);
1351
1352		/* Reset pinned so btrfs_put_block_group doesn't complain */
1353		spin_lock(&space_info->lock);
1354		spin_lock(&block_group->lock);
1355
1356		btrfs_space_info_update_bytes_pinned(fs_info, space_info,
1357						     -block_group->pinned);
1358		space_info->bytes_readonly += block_group->pinned;
1359		percpu_counter_add_batch(&space_info->total_bytes_pinned,
1360				   -block_group->pinned,
1361				   BTRFS_TOTAL_BYTES_PINNED_BATCH);
1362		block_group->pinned = 0;
1363
1364		spin_unlock(&block_group->lock);
1365		spin_unlock(&space_info->lock);
1366
1367		/* DISCARD can flip during remount */
1368		trimming = btrfs_test_opt(fs_info, DISCARD);
1369
1370		/* Implicit trim during transaction commit. */
1371		if (trimming)
1372			btrfs_get_block_group_trimming(block_group);
1373
1374		/*
1375		 * Btrfs_remove_chunk will abort the transaction if things go
1376		 * horribly wrong.
1377		 */
1378		ret = btrfs_remove_chunk(trans, block_group->key.objectid);
1379
1380		if (ret) {
1381			if (trimming)
1382				btrfs_put_block_group_trimming(block_group);
1383			goto end_trans;
1384		}
1385
1386		/*
1387		 * If we're not mounted with -odiscard, we can just forget
1388		 * about this block group. Otherwise we'll need to wait
1389		 * until transaction commit to do the actual discard.
1390		 */
1391		if (trimming) {
1392			spin_lock(&fs_info->unused_bgs_lock);
1393			/*
1394			 * A concurrent scrub might have added us to the list
1395			 * fs_info->unused_bgs, so use a list_move operation
1396			 * to add the block group to the deleted_bgs list.
1397			 */
1398			list_move(&block_group->bg_list,
1399				  &trans->transaction->deleted_bgs);
1400			spin_unlock(&fs_info->unused_bgs_lock);
1401			btrfs_get_block_group(block_group);
1402		}
1403end_trans:
1404		btrfs_end_transaction(trans);
1405next:
1406		mutex_unlock(&fs_info->delete_unused_bgs_mutex);
1407		btrfs_put_block_group(block_group);
1408		spin_lock(&fs_info->unused_bgs_lock);
1409	}
1410	spin_unlock(&fs_info->unused_bgs_lock);
1411}
1412
1413void btrfs_mark_bg_unused(struct btrfs_block_group_cache *bg)
1414{
1415	struct btrfs_fs_info *fs_info = bg->fs_info;
1416
1417	spin_lock(&fs_info->unused_bgs_lock);
1418	if (list_empty(&bg->bg_list)) {
1419		btrfs_get_block_group(bg);
1420		trace_btrfs_add_unused_block_group(bg);
1421		list_add_tail(&bg->bg_list, &fs_info->unused_bgs);
1422	}
1423	spin_unlock(&fs_info->unused_bgs_lock);
1424}
1425
1426static int find_first_block_group(struct btrfs_fs_info *fs_info,
1427				  struct btrfs_path *path,
1428				  struct btrfs_key *key)
1429{
1430	struct btrfs_root *root = fs_info->extent_root;
1431	int ret = 0;
1432	struct btrfs_key found_key;
1433	struct extent_buffer *leaf;
1434	struct btrfs_block_group_item bg;
1435	u64 flags;
1436	int slot;
1437
1438	ret = btrfs_search_slot(NULL, root, key, path, 0, 0);
1439	if (ret < 0)
1440		goto out;
1441
1442	while (1) {
1443		slot = path->slots[0];
1444		leaf = path->nodes[0];
1445		if (slot >= btrfs_header_nritems(leaf)) {
1446			ret = btrfs_next_leaf(root, path);
1447			if (ret == 0)
1448				continue;
1449			if (ret < 0)
1450				goto out;
1451			break;
1452		}
1453		btrfs_item_key_to_cpu(leaf, &found_key, slot);
1454
1455		if (found_key.objectid >= key->objectid &&
1456		    found_key.type == BTRFS_BLOCK_GROUP_ITEM_KEY) {
1457			struct extent_map_tree *em_tree;
1458			struct extent_map *em;
1459
1460			em_tree = &root->fs_info->mapping_tree;
1461			read_lock(&em_tree->lock);
1462			em = lookup_extent_mapping(em_tree, found_key.objectid,
1463						   found_key.offset);
1464			read_unlock(&em_tree->lock);
1465			if (!em) {
1466				btrfs_err(fs_info,
1467			"logical %llu len %llu found bg but no related chunk",
1468					  found_key.objectid, found_key.offset);
1469				ret = -ENOENT;
1470			} else if (em->start != found_key.objectid ||
1471				   em->len != found_key.offset) {
1472				btrfs_err(fs_info,
1473		"block group %llu len %llu mismatch with chunk %llu len %llu",
1474					  found_key.objectid, found_key.offset,
1475					  em->start, em->len);
1476				ret = -EUCLEAN;
1477			} else {
1478				read_extent_buffer(leaf, &bg,
1479					btrfs_item_ptr_offset(leaf, slot),
1480					sizeof(bg));
1481				flags = btrfs_block_group_flags(&bg) &
1482					BTRFS_BLOCK_GROUP_TYPE_MASK;
1483
1484				if (flags != (em->map_lookup->type &
1485					      BTRFS_BLOCK_GROUP_TYPE_MASK)) {
1486					btrfs_err(fs_info,
1487"block group %llu len %llu type flags 0x%llx mismatch with chunk type flags 0x%llx",
1488						found_key.objectid,
1489						found_key.offset, flags,
1490						(BTRFS_BLOCK_GROUP_TYPE_MASK &
1491						 em->map_lookup->type));
1492					ret = -EUCLEAN;
1493				} else {
1494					ret = 0;
1495				}
1496			}
1497			free_extent_map(em);
1498			goto out;
1499		}
1500		path->slots[0]++;
1501	}
1502out:
1503	return ret;
1504}
1505
1506static void set_avail_alloc_bits(struct btrfs_fs_info *fs_info, u64 flags)
1507{
1508	u64 extra_flags = chunk_to_extended(flags) &
1509				BTRFS_EXTENDED_PROFILE_MASK;
1510
1511	write_seqlock(&fs_info->profiles_lock);
1512	if (flags & BTRFS_BLOCK_GROUP_DATA)
1513		fs_info->avail_data_alloc_bits |= extra_flags;
1514	if (flags & BTRFS_BLOCK_GROUP_METADATA)
1515		fs_info->avail_metadata_alloc_bits |= extra_flags;
1516	if (flags & BTRFS_BLOCK_GROUP_SYSTEM)
1517		fs_info->avail_system_alloc_bits |= extra_flags;
1518	write_sequnlock(&fs_info->profiles_lock);
1519}
1520
1521static int exclude_super_stripes(struct btrfs_block_group_cache *cache)
1522{
1523	struct btrfs_fs_info *fs_info = cache->fs_info;
1524	u64 bytenr;
1525	u64 *logical;
1526	int stripe_len;
1527	int i, nr, ret;
1528
1529	if (cache->key.objectid < BTRFS_SUPER_INFO_OFFSET) {
1530		stripe_len = BTRFS_SUPER_INFO_OFFSET - cache->key.objectid;
1531		cache->bytes_super += stripe_len;
1532		ret = btrfs_add_excluded_extent(fs_info, cache->key.objectid,
1533						stripe_len);
1534		if (ret)
1535			return ret;
1536	}
1537
1538	for (i = 0; i < BTRFS_SUPER_MIRROR_MAX; i++) {
1539		bytenr = btrfs_sb_offset(i);
1540		ret = btrfs_rmap_block(fs_info, cache->key.objectid,
1541				       bytenr, &logical, &nr, &stripe_len);
1542		if (ret)
1543			return ret;
1544
1545		while (nr--) {
1546			u64 start, len;
1547
1548			if (logical[nr] > cache->key.objectid +
1549			    cache->key.offset)
1550				continue;
1551
1552			if (logical[nr] + stripe_len <= cache->key.objectid)
1553				continue;
1554
1555			start = logical[nr];
1556			if (start < cache->key.objectid) {
1557				start = cache->key.objectid;
1558				len = (logical[nr] + stripe_len) - start;
1559			} else {
1560				len = min_t(u64, stripe_len,
1561					    cache->key.objectid +
1562					    cache->key.offset - start);
1563			}
1564
1565			cache->bytes_super += len;
1566			ret = btrfs_add_excluded_extent(fs_info, start, len);
1567			if (ret) {
1568				kfree(logical);
1569				return ret;
1570			}
1571		}
1572
1573		kfree(logical);
1574	}
1575	return 0;
1576}
1577
1578static void link_block_group(struct btrfs_block_group_cache *cache)
1579{
1580	struct btrfs_space_info *space_info = cache->space_info;
1581	int index = btrfs_bg_flags_to_raid_index(cache->flags);
1582	bool first = false;
1583
1584	down_write(&space_info->groups_sem);
1585	if (list_empty(&space_info->block_groups[index]))
1586		first = true;
1587	list_add_tail(&cache->list, &space_info->block_groups[index]);
1588	up_write(&space_info->groups_sem);
1589
1590	if (first)
1591		btrfs_sysfs_add_block_group_type(cache);
1592}
1593
1594static struct btrfs_block_group_cache *btrfs_create_block_group_cache(
1595		struct btrfs_fs_info *fs_info, u64 start, u64 size)
1596{
1597	struct btrfs_block_group_cache *cache;
1598
1599	cache = kzalloc(sizeof(*cache), GFP_NOFS);
1600	if (!cache)
1601		return NULL;
1602
1603	cache->free_space_ctl = kzalloc(sizeof(*cache->free_space_ctl),
1604					GFP_NOFS);
1605	if (!cache->free_space_ctl) {
1606		kfree(cache);
1607		return NULL;
1608	}
1609
1610	cache->key.objectid = start;
1611	cache->key.offset = size;
1612	cache->key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
1613
1614	cache->fs_info = fs_info;
1615	cache->full_stripe_len = btrfs_full_stripe_len(fs_info, start);
1616	set_free_space_tree_thresholds(cache);
1617
1618	atomic_set(&cache->count, 1);
1619	spin_lock_init(&cache->lock);
1620	init_rwsem(&cache->data_rwsem);
1621	INIT_LIST_HEAD(&cache->list);
1622	INIT_LIST_HEAD(&cache->cluster_list);
1623	INIT_LIST_HEAD(&cache->bg_list);
1624	INIT_LIST_HEAD(&cache->ro_list);
1625	INIT_LIST_HEAD(&cache->dirty_list);
1626	INIT_LIST_HEAD(&cache->io_list);
1627	btrfs_init_free_space_ctl(cache);
1628	atomic_set(&cache->trimming, 0);
1629	mutex_init(&cache->free_space_lock);
1630	btrfs_init_full_stripe_locks_tree(&cache->full_stripe_locks_root);
1631
1632	return cache;
1633}
1634
1635/*
1636 * Iterate all chunks and verify that each of them has the corresponding block
1637 * group
1638 */
1639static int check_chunk_block_group_mappings(struct btrfs_fs_info *fs_info)
1640{
1641	struct extent_map_tree *map_tree = &fs_info->mapping_tree;
1642	struct extent_map *em;
1643	struct btrfs_block_group_cache *bg;
1644	u64 start = 0;
1645	int ret = 0;
1646
1647	while (1) {
1648		read_lock(&map_tree->lock);
1649		/*
1650		 * lookup_extent_mapping will return the first extent map
1651		 * intersecting the range, so setting @len to 1 is enough to
1652		 * get the first chunk.
1653		 */
1654		em = lookup_extent_mapping(map_tree, start, 1);
1655		read_unlock(&map_tree->lock);
1656		if (!em)
1657			break;
1658
1659		bg = btrfs_lookup_block_group(fs_info, em->start);
1660		if (!bg) {
1661			btrfs_err(fs_info,
1662	"chunk start=%llu len=%llu doesn't have corresponding block group",
1663				     em->start, em->len);
1664			ret = -EUCLEAN;
1665			free_extent_map(em);
1666			break;
1667		}
1668		if (bg->key.objectid != em->start ||
1669		    bg->key.offset != em->len ||
1670		    (bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK) !=
1671		    (em->map_lookup->type & BTRFS_BLOCK_GROUP_TYPE_MASK)) {
1672			btrfs_err(fs_info,
1673"chunk start=%llu len=%llu flags=0x%llx doesn't match block group start=%llu len=%llu flags=0x%llx",
1674				em->start, em->len,
1675				em->map_lookup->type & BTRFS_BLOCK_GROUP_TYPE_MASK,
1676				bg->key.objectid, bg->key.offset,
1677				bg->flags & BTRFS_BLOCK_GROUP_TYPE_MASK);
1678			ret = -EUCLEAN;
1679			free_extent_map(em);
1680			btrfs_put_block_group(bg);
1681			break;
1682		}
1683		start = em->start + em->len;
1684		free_extent_map(em);
1685		btrfs_put_block_group(bg);
1686	}
1687	return ret;
1688}
1689
1690int btrfs_read_block_groups(struct btrfs_fs_info *info)
1691{
1692	struct btrfs_path *path;
1693	int ret;
1694	struct btrfs_block_group_cache *cache;
1695	struct btrfs_space_info *space_info;
1696	struct btrfs_key key;
1697	struct btrfs_key found_key;
1698	struct extent_buffer *leaf;
1699	int need_clear = 0;
1700	u64 cache_gen;
1701	u64 feature;
1702	int mixed;
1703
1704	feature = btrfs_super_incompat_flags(info->super_copy);
1705	mixed = !!(feature & BTRFS_FEATURE_INCOMPAT_MIXED_GROUPS);
1706
1707	key.objectid = 0;
1708	key.offset = 0;
1709	key.type = BTRFS_BLOCK_GROUP_ITEM_KEY;
1710	path = btrfs_alloc_path();
1711	if (!path)
1712		return -ENOMEM;
1713	path->reada = READA_FORWARD;
1714
1715	cache_gen = btrfs_super_cache_generation(info->super_copy);
1716	if (btrfs_test_opt(info, SPACE_CACHE) &&
1717	    btrfs_super_generation(info->super_copy) != cache_gen)
1718		need_clear = 1;
1719	if (btrfs_test_opt(info, CLEAR_CACHE))
1720		need_clear = 1;
1721
1722	while (1) {
1723		ret = find_first_block_group(info, path, &key);
1724		if (ret > 0)
1725			break;
1726		if (ret != 0)
1727			goto error;
1728
1729		leaf = path->nodes[0];
1730		btrfs_item_key_to_cpu(leaf, &found_key, path->slots[0]);
1731
1732		cache = btrfs_create_block_group_cache(info, found_key.objectid,
1733						       found_key.offset);
1734		if (!cache) {
1735			ret = -ENOMEM;
1736			goto error;
1737		}
1738
1739		if (need_clear) {
1740			/*
1741			 * When we mount with old space cache, we need to
1742			 * set BTRFS_DC_CLEAR and set dirty flag.
1743			 *
1744			 * a) Setting 'BTRFS_DC_CLEAR' makes sure that we
1745			 *    truncate the old free space cache inode and
1746			 *    setup a new one.
1747			 * b) Setting 'dirty flag' makes sure that we flush
1748			 *    the new space cache info onto disk.
1749			 */
1750			if (btrfs_test_opt(info, SPACE_CACHE))
1751				cache->disk_cache_state = BTRFS_DC_CLEAR;
1752		}
1753
1754		read_extent_buffer(leaf, &cache->item,
1755				   btrfs_item_ptr_offset(leaf, path->slots[0]),
1756				   sizeof(cache->item));
1757		cache->flags = btrfs_block_group_flags(&cache->item);
1758		if (!mixed &&
1759		    ((cache->flags & BTRFS_BLOCK_GROUP_METADATA) &&
1760		    (cache->flags & BTRFS_BLOCK_GROUP_DATA))) {
1761			btrfs_err(info,
1762"bg %llu is a mixed block group but filesystem hasn't enabled mixed block groups",
1763				  cache->key.objectid);
1764			btrfs_put_block_group(cache);
1765			ret = -EINVAL;
1766			goto error;
1767		}
1768
1769		key.objectid = found_key.objectid + found_key.offset;
1770		btrfs_release_path(path);
1771
1772		/*
1773		 * We need to exclude the super stripes now so that the space
1774		 * info has super bytes accounted for, otherwise we'll think
1775		 * we have more space than we actually do.
1776		 */
1777		ret = exclude_super_stripes(cache);
1778		if (ret) {
1779			/*
1780			 * We may have excluded something, so call this just in
1781			 * case.
1782			 */
1783			btrfs_free_excluded_extents(cache);
1784			btrfs_put_block_group(cache);
1785			goto error;
1786		}
1787
1788		/*
1789		 * Check for two cases, either we are full, and therefore
1790		 * don't need to bother with the caching work since we won't
1791		 * find any space, or we are empty, and we can just add all
1792		 * the space in and be done with it.  This saves us _a_lot_ of
1793		 * time, particularly in the full case.
1794		 */
1795		if (found_key.offset == btrfs_block_group_used(&cache->item)) {
1796			cache->last_byte_to_unpin = (u64)-1;
1797			cache->cached = BTRFS_CACHE_FINISHED;
1798			btrfs_free_excluded_extents(cache);
1799		} else if (btrfs_block_group_used(&cache->item) == 0) {
1800			cache->last_byte_to_unpin = (u64)-1;
1801			cache->cached = BTRFS_CACHE_FINISHED;
1802			add_new_free_space(cache, found_key.objectid,
1803					   found_key.objectid +
1804					   found_key.offset);
1805			btrfs_free_excluded_extents(cache);
1806		}
1807
1808		ret = btrfs_add_block_group_cache(info, cache);
1809		if (ret) {
1810			btrfs_remove_free_space_cache(cache);
1811			btrfs_put_block_group(cache);
1812			goto error;
1813		}
1814
1815		trace_btrfs_add_block_group(info, cache, 0);
1816		btrfs_update_space_info(info, cache->flags, found_key.offset,
1817					btrfs_block_group_used(&cache->item),
1818					cache->bytes_super, &space_info);
1819
1820		cache->space_info = space_info;
1821
1822		link_block_group(cache);
1823
1824		set_avail_alloc_bits(info, cache->flags);
1825		if (btrfs_chunk_readonly(info, cache->key.objectid)) {
1826			inc_block_group_ro(cache, 1);
1827		} else if (btrfs_block_group_used(&cache->item) == 0) {
1828			ASSERT(list_empty(&cache->bg_list));
1829			btrfs_mark_bg_unused(cache);
1830		}
1831	}
1832
1833	list_for_each_entry_rcu(space_info, &info->space_info, list) {
1834		if (!(btrfs_get_alloc_profile(info, space_info->flags) &
1835		      (BTRFS_BLOCK_GROUP_RAID10 |
1836		       BTRFS_BLOCK_GROUP_RAID1_MASK |
1837		       BTRFS_BLOCK_GROUP_RAID56_MASK |
1838		       BTRFS_BLOCK_GROUP_DUP)))
1839			continue;
1840		/*
1841		 * Avoid allocating from un-mirrored block group if there are
1842		 * mirrored block groups.
1843		 */
1844		list_for_each_entry(cache,
1845				&space_info->block_groups[BTRFS_RAID_RAID0],
1846				list)
1847			inc_block_group_ro(cache, 1);
1848		list_for_each_entry(cache,
1849				&space_info->block_groups[BTRFS_RAID_SINGLE],
1850				list)
1851			inc_block_group_ro(cache, 1);
1852	}
1853
1854	btrfs_init_global_block_rsv(info);
1855	ret = check_chunk_block_group_mappings(info);
1856error:
1857	btrfs_free_path(path);
1858	return ret;
1859}
1860
1861void btrfs_create_pending_block_groups(struct btrfs_trans_handle *trans)
1862{
1863	struct btrfs_fs_info *fs_info = trans->fs_info;
1864	struct btrfs_block_group_cache *block_group;
1865	struct btrfs_root *extent_root = fs_info->extent_root;
1866	struct btrfs_block_group_item item;
1867	struct btrfs_key key;
1868	int ret = 0;
1869
1870	if (!trans->can_flush_pending_bgs)
1871		return;
1872
1873	while (!list_empty(&trans->new_bgs)) {
1874		block_group = list_first_entry(&trans->new_bgs,
1875					       struct btrfs_block_group_cache,
1876					       bg_list);
1877		if (ret)
1878			goto next;
1879
1880		spin_lock(&block_group->lock);
1881		memcpy(&item, &block_group->item, sizeof(item));
1882		memcpy(&key, &block_group->key, sizeof(key));
1883		spin_unlock(&block_group->lock);
1884
1885		ret = btrfs_insert_item(trans, extent_root, &key, &item,
1886					sizeof(item));
1887		if (ret)
1888			btrfs_abort_transaction(trans, ret);
1889		ret = btrfs_finish_chunk_alloc(trans, key.objectid, key.offset);
1890		if (ret)
1891			btrfs_abort_transaction(trans, ret);
1892		add_block_group_free_space(trans, block_group);
1893		/* Already aborted the transaction if it failed. */
1894next:
1895		btrfs_delayed_refs_rsv_release(fs_info, 1);
1896		list_del_init(&block_group->bg_list);
1897	}
1898	btrfs_trans_release_chunk_metadata(trans);
1899}
1900
1901int btrfs_make_block_group(struct btrfs_trans_handle *trans, u64 bytes_used,
1902			   u64 type, u64 chunk_offset, u64 size)
1903{
1904	struct btrfs_fs_info *fs_info = trans->fs_info;
1905	struct btrfs_block_group_cache *cache;
1906	int ret;
1907
1908	btrfs_set_log_full_commit(trans);
1909
1910	cache = btrfs_create_block_group_cache(fs_info, chunk_offset, size);
1911	if (!cache)
1912		return -ENOMEM;
1913
1914	btrfs_set_block_group_used(&cache->item, bytes_used);
1915	btrfs_set_block_group_chunk_objectid(&cache->item,
1916					     BTRFS_FIRST_CHUNK_TREE_OBJECTID);
1917	btrfs_set_block_group_flags(&cache->item, type);
1918
1919	cache->flags = type;
1920	cache->last_byte_to_unpin = (u64)-1;
1921	cache->cached = BTRFS_CACHE_FINISHED;
1922	cache->needs_free_space = 1;
1923	ret = exclude_super_stripes(cache);
1924	if (ret) {
1925		/* We may have excluded something, so call this just in case */
1926		btrfs_free_excluded_extents(cache);
1927		btrfs_put_block_group(cache);
1928		return ret;
1929	}
1930
1931	add_new_free_space(cache, chunk_offset, chunk_offset + size);
1932
1933	btrfs_free_excluded_extents(cache);
1934
1935#ifdef CONFIG_BTRFS_DEBUG
1936	if (btrfs_should_fragment_free_space(cache)) {
1937		u64 new_bytes_used = size - bytes_used;
1938
1939		bytes_used += new_bytes_used >> 1;
1940		fragment_free_space(cache);
1941	}
1942#endif
1943	/*
1944	 * Ensure the corresponding space_info object is created and
1945	 * assigned to our block group. We want our bg to be added to the rbtree
1946	 * with its ->space_info set.
1947	 */
1948	cache->space_info = btrfs_find_space_info(fs_info, cache->flags);
1949	ASSERT(cache->space_info);
1950
1951	ret = btrfs_add_block_group_cache(fs_info, cache);
1952	if (ret) {
1953		btrfs_remove_free_space_cache(cache);
1954		btrfs_put_block_group(cache);
1955		return ret;
1956	}
1957
1958	/*
1959	 * Now that our block group has its ->space_info set and is inserted in
1960	 * the rbtree, update the space info's counters.
1961	 */
1962	trace_btrfs_add_block_group(fs_info, cache, 1);
1963	btrfs_update_space_info(fs_info, cache->flags, size, bytes_used,
1964				cache->bytes_super, &cache->space_info);
1965	btrfs_update_global_block_rsv(fs_info);
1966
1967	link_block_group(cache);
1968
1969	list_add_tail(&cache->bg_list, &trans->new_bgs);
1970	trans->delayed_ref_updates++;
1971	btrfs_update_delayed_refs_rsv(trans);
1972
1973	set_avail_alloc_bits(fs_info, type);
1974	return 0;
1975}
1976
1977static u64 update_block_group_flags(struct btrfs_fs_info *fs_info, u64 flags)
1978{
1979	u64 num_devices;
1980	u64 stripped;
1981
1982	/*
1983	 * if restripe for this chunk_type is on pick target profile and
1984	 * return, otherwise do the usual balance
1985	 */
1986	stripped = get_restripe_target(fs_info, flags);
1987	if (stripped)
1988		return extended_to_chunk(stripped);
1989
1990	num_devices = fs_info->fs_devices->rw_devices;
1991
1992	stripped = BTRFS_BLOCK_GROUP_RAID0 | BTRFS_BLOCK_GROUP_RAID56_MASK |
1993		BTRFS_BLOCK_GROUP_RAID1_MASK | BTRFS_BLOCK_GROUP_RAID10;
1994
1995	if (num_devices == 1) {
1996		stripped |= BTRFS_BLOCK_GROUP_DUP;
1997		stripped = flags & ~stripped;
1998
1999		/* turn raid0 into single device chunks */
2000		if (flags & BTRFS_BLOCK_GROUP_RAID0)
2001			return stripped;
2002
2003		/* turn mirroring into duplication */
2004		if (flags & (BTRFS_BLOCK_GROUP_RAID1_MASK |
2005			     BTRFS_BLOCK_GROUP_RAID10))
2006			return stripped | BTRFS_BLOCK_GROUP_DUP;
2007	} else {
2008		/* they already had raid on here, just return */
2009		if (flags & stripped)
2010			return flags;
2011
2012		stripped |= BTRFS_BLOCK_GROUP_DUP;
2013		stripped = flags & ~stripped;
2014
2015		/* switch duplicated blocks with raid1 */
2016		if (flags & BTRFS_BLOCK_GROUP_DUP)
2017			return stripped | BTRFS_BLOCK_GROUP_RAID1;
2018
2019		/* this is drive concat, leave it alone */
2020	}
2021
2022	return flags;
2023}
2024
2025int btrfs_inc_block_group_ro(struct btrfs_block_group_cache *cache)
2026
2027{
2028	struct btrfs_fs_info *fs_info = cache->fs_info;
2029	struct btrfs_trans_handle *trans;
2030	u64 alloc_flags;
2031	int ret;
2032
2033again:
2034	trans = btrfs_join_transaction(fs_info->extent_root);
2035	if (IS_ERR(trans))
2036		return PTR_ERR(trans);
2037
2038	/*
2039	 * we're not allowed to set block groups readonly after the dirty
2040	 * block groups cache has started writing.  If it already started,
2041	 * back off and let this transaction commit
2042	 */
2043	mutex_lock(&fs_info->ro_block_group_mutex);
2044	if (test_bit(BTRFS_TRANS_DIRTY_BG_RUN, &trans->transaction->flags)) {
2045		u64 transid = trans->transid;
2046
2047		mutex_unlock(&fs_info->ro_block_group_mutex);
2048		btrfs_end_transaction(trans);
2049
2050		ret = btrfs_wait_for_commit(fs_info, transid);
2051		if (ret)
2052			return ret;
2053		goto again;
2054	}
2055
2056	/*
2057	 * if we are changing raid levels, try to allocate a corresponding
2058	 * block group with the new raid level.
2059	 */
2060	alloc_flags = update_block_group_flags(fs_info, cache->flags);
2061	if (alloc_flags != cache->flags) {
2062		ret = btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE);
2063		/*
2064		 * ENOSPC is allowed here, we may have enough space
2065		 * already allocated at the new raid level to
2066		 * carry on
2067		 */
2068		if (ret == -ENOSPC)
2069			ret = 0;
2070		if (ret < 0)
2071			goto out;
2072	}
2073
2074	ret = inc_block_group_ro(cache, 0);
2075	if (!ret)
2076		goto out;
2077	alloc_flags = btrfs_get_alloc_profile(fs_info, cache->space_info->flags);
2078	ret = btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE);
2079	if (ret < 0)
2080		goto out;
2081	ret = inc_block_group_ro(cache, 0);
2082out:
2083	if (cache->flags & BTRFS_BLOCK_GROUP_SYSTEM) {
2084		alloc_flags = update_block_group_flags(fs_info, cache->flags);
2085		mutex_lock(&fs_info->chunk_mutex);
2086		check_system_chunk(trans, alloc_flags);
2087		mutex_unlock(&fs_info->chunk_mutex);
2088	}
2089	mutex_unlock(&fs_info->ro_block_group_mutex);
2090
2091	btrfs_end_transaction(trans);
2092	return ret;
2093}
2094
2095void btrfs_dec_block_group_ro(struct btrfs_block_group_cache *cache)
2096{
2097	struct btrfs_space_info *sinfo = cache->space_info;
2098	u64 num_bytes;
2099
2100	BUG_ON(!cache->ro);
2101
2102	spin_lock(&sinfo->lock);
2103	spin_lock(&cache->lock);
2104	if (!--cache->ro) {
2105		num_bytes = cache->key.offset - cache->reserved -
2106			    cache->pinned - cache->bytes_super -
2107			    btrfs_block_group_used(&cache->item);
2108		sinfo->bytes_readonly -= num_bytes;
2109		list_del_init(&cache->ro_list);
2110	}
2111	spin_unlock(&cache->lock);
2112	spin_unlock(&sinfo->lock);
2113}
2114
2115static int write_one_cache_group(struct btrfs_trans_handle *trans,
2116				 struct btrfs_path *path,
2117				 struct btrfs_block_group_cache *cache)
2118{
2119	struct btrfs_fs_info *fs_info = trans->fs_info;
2120	int ret;
2121	struct btrfs_root *extent_root = fs_info->extent_root;
2122	unsigned long bi;
2123	struct extent_buffer *leaf;
2124
2125	ret = btrfs_search_slot(trans, extent_root, &cache->key, path, 0, 1);
2126	if (ret) {
2127		if (ret > 0)
2128			ret = -ENOENT;
2129		goto fail;
2130	}
2131
2132	leaf = path->nodes[0];
2133	bi = btrfs_item_ptr_offset(leaf, path->slots[0]);
2134	write_extent_buffer(leaf, &cache->item, bi, sizeof(cache->item));
2135	btrfs_mark_buffer_dirty(leaf);
2136fail:
2137	btrfs_release_path(path);
2138	return ret;
2139
2140}
2141
2142static int cache_save_setup(struct btrfs_block_group_cache *block_group,
2143			    struct btrfs_trans_handle *trans,
2144			    struct btrfs_path *path)
2145{
2146	struct btrfs_fs_info *fs_info = block_group->fs_info;
2147	struct btrfs_root *root = fs_info->tree_root;
2148	struct inode *inode = NULL;
2149	struct extent_changeset *data_reserved = NULL;
2150	u64 alloc_hint = 0;
2151	int dcs = BTRFS_DC_ERROR;
2152	u64 num_pages = 0;
2153	int retries = 0;
2154	int ret = 0;
2155
2156	/*
2157	 * If this block group is smaller than 100 megs don't bother caching the
2158	 * block group.
2159	 */
2160	if (block_group->key.offset < (100 * SZ_1M)) {
2161		spin_lock(&block_group->lock);
2162		block_group->disk_cache_state = BTRFS_DC_WRITTEN;
2163		spin_unlock(&block_group->lock);
2164		return 0;
2165	}
2166
2167	if (trans->aborted)
2168		return 0;
2169again:
2170	inode = lookup_free_space_inode(block_group, path);
2171	if (IS_ERR(inode) && PTR_ERR(inode) != -ENOENT) {
2172		ret = PTR_ERR(inode);
2173		btrfs_release_path(path);
2174		goto out;
2175	}
2176
2177	if (IS_ERR(inode)) {
2178		BUG_ON(retries);
2179		retries++;
2180
2181		if (block_group->ro)
2182			goto out_free;
2183
2184		ret = create_free_space_inode(trans, block_group, path);
2185		if (ret)
2186			goto out_free;
2187		goto again;
2188	}
2189
2190	/*
2191	 * We want to set the generation to 0, that way if anything goes wrong
2192	 * from here on out we know not to trust this cache when we load up next
2193	 * time.
2194	 */
2195	BTRFS_I(inode)->generation = 0;
2196	ret = btrfs_update_inode(trans, root, inode);
2197	if (ret) {
2198		/*
2199		 * So theoretically we could recover from this, simply set the
2200		 * super cache generation to 0 so we know to invalidate the
2201		 * cache, but then we'd have to keep track of the block groups
2202		 * that fail this way so we know we _have_ to reset this cache
2203		 * before the next commit or risk reading stale cache.  So to
2204		 * limit our exposure to horrible edge cases lets just abort the
2205		 * transaction, this only happens in really bad situations
2206		 * anyway.
2207		 */
2208		btrfs_abort_transaction(trans, ret);
2209		goto out_put;
2210	}
2211	WARN_ON(ret);
2212
2213	/* We've already setup this transaction, go ahead and exit */
2214	if (block_group->cache_generation == trans->transid &&
2215	    i_size_read(inode)) {
2216		dcs = BTRFS_DC_SETUP;
2217		goto out_put;
2218	}
2219
2220	if (i_size_read(inode) > 0) {
2221		ret = btrfs_check_trunc_cache_free_space(fs_info,
2222					&fs_info->global_block_rsv);
2223		if (ret)
2224			goto out_put;
2225
2226		ret = btrfs_truncate_free_space_cache(trans, NULL, inode);
2227		if (ret)
2228			goto out_put;
2229	}
2230
2231	spin_lock(&block_group->lock);
2232	if (block_group->cached != BTRFS_CACHE_FINISHED ||
2233	    !btrfs_test_opt(fs_info, SPACE_CACHE)) {
2234		/*
2235		 * don't bother trying to write stuff out _if_
2236		 * a) we're not cached,
2237		 * b) we're with nospace_cache mount option,
2238		 * c) we're with v2 space_cache (FREE_SPACE_TREE).
2239		 */
2240		dcs = BTRFS_DC_WRITTEN;
2241		spin_unlock(&block_group->lock);
2242		goto out_put;
2243	}
2244	spin_unlock(&block_group->lock);
2245
2246	/*
2247	 * We hit an ENOSPC when setting up the cache in this transaction, just
2248	 * skip doing the setup, we've already cleared the cache so we're safe.
2249	 */
2250	if (test_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags)) {
2251		ret = -ENOSPC;
2252		goto out_put;
2253	}
2254
2255	/*
2256	 * Try to preallocate enough space based on how big the block group is.
2257	 * Keep in mind this has to include any pinned space which could end up
2258	 * taking up quite a bit since it's not folded into the other space
2259	 * cache.
2260	 */
2261	num_pages = div_u64(block_group->key.offset, SZ_256M);
2262	if (!num_pages)
2263		num_pages = 1;
2264
2265	num_pages *= 16;
2266	num_pages *= PAGE_SIZE;
2267
2268	ret = btrfs_check_data_free_space(inode, &data_reserved, 0, num_pages);
2269	if (ret)
2270		goto out_put;
2271
2272	ret = btrfs_prealloc_file_range_trans(inode, trans, 0, 0, num_pages,
2273					      num_pages, num_pages,
2274					      &alloc_hint);
2275	/*
2276	 * Our cache requires contiguous chunks so that we don't modify a bunch
2277	 * of metadata or split extents when writing the cache out, which means
2278	 * we can enospc if we are heavily fragmented in addition to just normal
2279	 * out of space conditions.  So if we hit this just skip setting up any
2280	 * other block groups for this transaction, maybe we'll unpin enough
2281	 * space the next time around.
2282	 */
2283	if (!ret)
2284		dcs = BTRFS_DC_SETUP;
2285	else if (ret == -ENOSPC)
2286		set_bit(BTRFS_TRANS_CACHE_ENOSPC, &trans->transaction->flags);
2287
2288out_put:
2289	iput(inode);
2290out_free:
2291	btrfs_release_path(path);
2292out:
2293	spin_lock(&block_group->lock);
2294	if (!ret && dcs == BTRFS_DC_SETUP)
2295		block_group->cache_generation = trans->transid;
2296	block_group->disk_cache_state = dcs;
2297	spin_unlock(&block_group->lock);
2298
2299	extent_changeset_free(data_reserved);
2300	return ret;
2301}
2302
2303int btrfs_setup_space_cache(struct btrfs_trans_handle *trans)
2304{
2305	struct btrfs_fs_info *fs_info = trans->fs_info;
2306	struct btrfs_block_group_cache *cache, *tmp;
2307	struct btrfs_transaction *cur_trans = trans->transaction;
2308	struct btrfs_path *path;
2309
2310	if (list_empty(&cur_trans->dirty_bgs) ||
2311	    !btrfs_test_opt(fs_info, SPACE_CACHE))
2312		return 0;
2313
2314	path = btrfs_alloc_path();
2315	if (!path)
2316		return -ENOMEM;
2317
2318	/* Could add new block groups, use _safe just in case */
2319	list_for_each_entry_safe(cache, tmp, &cur_trans->dirty_bgs,
2320				 dirty_list) {
2321		if (cache->disk_cache_state == BTRFS_DC_CLEAR)
2322			cache_save_setup(cache, trans, path);
2323	}
2324
2325	btrfs_free_path(path);
2326	return 0;
2327}
2328
2329/*
2330 * Transaction commit does final block group cache writeback during a critical
2331 * section where nothing is allowed to change the FS.  This is required in
2332 * order for the cache to actually match the block group, but can introduce a
2333 * lot of latency into the commit.
2334 *
2335 * So, btrfs_start_dirty_block_groups is here to kick off block group cache IO.
2336 * There's a chance we'll have to redo some of it if the block group changes
2337 * again during the commit, but it greatly reduces the commit latency by
2338 * getting rid of the easy block groups while we're still allowing others to
2339 * join the commit.
2340 */
2341int btrfs_start_dirty_block_groups(struct btrfs_trans_handle *trans)
2342{
2343	struct btrfs_fs_info *fs_info = trans->fs_info;
2344	struct btrfs_block_group_cache *cache;
2345	struct btrfs_transaction *cur_trans = trans->transaction;
2346	int ret = 0;
2347	int should_put;
2348	struct btrfs_path *path = NULL;
2349	LIST_HEAD(dirty);
2350	struct list_head *io = &cur_trans->io_bgs;
2351	int num_started = 0;
2352	int loops = 0;
2353
2354	spin_lock(&cur_trans->dirty_bgs_lock);
2355	if (list_empty(&cur_trans->dirty_bgs)) {
2356		spin_unlock(&cur_trans->dirty_bgs_lock);
2357		return 0;
2358	}
2359	list_splice_init(&cur_trans->dirty_bgs, &dirty);
2360	spin_unlock(&cur_trans->dirty_bgs_lock);
2361
2362again:
2363	/* Make sure all the block groups on our dirty list actually exist */
2364	btrfs_create_pending_block_groups(trans);
2365
2366	if (!path) {
2367		path = btrfs_alloc_path();
2368		if (!path)
2369			return -ENOMEM;
2370	}
2371
2372	/*
2373	 * cache_write_mutex is here only to save us from balance or automatic
2374	 * removal of empty block groups deleting this block group while we are
2375	 * writing out the cache
2376	 */
2377	mutex_lock(&trans->transaction->cache_write_mutex);
2378	while (!list_empty(&dirty)) {
2379		bool drop_reserve = true;
2380
2381		cache = list_first_entry(&dirty,
2382					 struct btrfs_block_group_cache,
2383					 dirty_list);
2384		/*
2385		 * This can happen if something re-dirties a block group that
2386		 * is already under IO.  Just wait for it to finish and then do
2387		 * it all again
2388		 */
2389		if (!list_empty(&cache->io_list)) {
2390			list_del_init(&cache->io_list);
2391			btrfs_wait_cache_io(trans, cache, path);
2392			btrfs_put_block_group(cache);
2393		}
2394
2395
2396		/*
2397		 * btrfs_wait_cache_io uses the cache->dirty_list to decide if
2398		 * it should update the cache_state.  Don't delete until after
2399		 * we wait.
2400		 *
2401		 * Since we're not running in the commit critical section
2402		 * we need the dirty_bgs_lock to protect from update_block_group
2403		 */
2404		spin_lock(&cur_trans->dirty_bgs_lock);
2405		list_del_init(&cache->dirty_list);
2406		spin_unlock(&cur_trans->dirty_bgs_lock);
2407
2408		should_put = 1;
2409
2410		cache_save_setup(cache, trans, path);
2411
2412		if (cache->disk_cache_state == BTRFS_DC_SETUP) {
2413			cache->io_ctl.inode = NULL;
2414			ret = btrfs_write_out_cache(trans, cache, path);
2415			if (ret == 0 && cache->io_ctl.inode) {
2416				num_started++;
2417				should_put = 0;
2418
2419				/*
2420				 * The cache_write_mutex is protecting the
2421				 * io_list, also refer to the definition of
2422				 * btrfs_transaction::io_bgs for more details
2423				 */
2424				list_add_tail(&cache->io_list, io);
2425			} else {
2426				/*
2427				 * If we failed to write the cache, the
2428				 * generation will be bad and life goes on
2429				 */
2430				ret = 0;
2431			}
2432		}
2433		if (!ret) {
2434			ret = write_one_cache_group(trans, path, cache);
2435			/*
2436			 * Our block group might still be attached to the list
2437			 * of new block groups in the transaction handle of some
2438			 * other task (struct btrfs_trans_handle->new_bgs). This
2439			 * means its block group item isn't yet in the extent
2440			 * tree. If this happens ignore the error, as we will
2441			 * try again later in the critical section of the
2442			 * transaction commit.
2443			 */
2444			if (ret == -ENOENT) {
2445				ret = 0;
2446				spin_lock(&cur_trans->dirty_bgs_lock);
2447				if (list_empty(&cache->dirty_list)) {
2448					list_add_tail(&cache->dirty_list,
2449						      &cur_trans->dirty_bgs);
2450					btrfs_get_block_group(cache);
2451					drop_reserve = false;
2452				}
2453				spin_unlock(&cur_trans->dirty_bgs_lock);
2454			} else if (ret) {
2455				btrfs_abort_transaction(trans, ret);
2456			}
2457		}
2458
2459		/* If it's not on the io list, we need to put the block group */
2460		if (should_put)
2461			btrfs_put_block_group(cache);
2462		if (drop_reserve)
2463			btrfs_delayed_refs_rsv_release(fs_info, 1);
2464
2465		if (ret)
2466			break;
2467
2468		/*
2469		 * Avoid blocking other tasks for too long. It might even save
2470		 * us from writing caches for block groups that are going to be
2471		 * removed.
2472		 */
2473		mutex_unlock(&trans->transaction->cache_write_mutex);
2474		mutex_lock(&trans->transaction->cache_write_mutex);
2475	}
2476	mutex_unlock(&trans->transaction->cache_write_mutex);
2477
2478	/*
2479	 * Go through delayed refs for all the stuff we've just kicked off
2480	 * and then loop back (just once)
2481	 */
2482	ret = btrfs_run_delayed_refs(trans, 0);
2483	if (!ret && loops == 0) {
2484		loops++;
2485		spin_lock(&cur_trans->dirty_bgs_lock);
2486		list_splice_init(&cur_trans->dirty_bgs, &dirty);
2487		/*
2488		 * dirty_bgs_lock protects us from concurrent block group
2489		 * deletes too (not just cache_write_mutex).
2490		 */
2491		if (!list_empty(&dirty)) {
2492			spin_unlock(&cur_trans->dirty_bgs_lock);
2493			goto again;
2494		}
2495		spin_unlock(&cur_trans->dirty_bgs_lock);
2496	} else if (ret < 0) {
2497		btrfs_cleanup_dirty_bgs(cur_trans, fs_info);
2498	}
2499
2500	btrfs_free_path(path);
2501	return ret;
2502}
2503
2504int btrfs_write_dirty_block_groups(struct btrfs_trans_handle *trans)
2505{
2506	struct btrfs_fs_info *fs_info = trans->fs_info;
2507	struct btrfs_block_group_cache *cache;
2508	struct btrfs_transaction *cur_trans = trans->transaction;
2509	int ret = 0;
2510	int should_put;
2511	struct btrfs_path *path;
2512	struct list_head *io = &cur_trans->io_bgs;
2513	int num_started = 0;
2514
2515	path = btrfs_alloc_path();
2516	if (!path)
2517		return -ENOMEM;
2518
2519	/*
2520	 * Even though we are in the critical section of the transaction commit,
2521	 * we can still have concurrent tasks adding elements to this
2522	 * transaction's list of dirty block groups. These tasks correspond to
2523	 * endio free space workers started when writeback finishes for a
2524	 * space cache, which run inode.c:btrfs_finish_ordered_io(), and can
2525	 * allocate new block groups as a result of COWing nodes of the root
2526	 * tree when updating the free space inode. The writeback for the space
2527	 * caches is triggered by an earlier call to
2528	 * btrfs_start_dirty_block_groups() and iterations of the following
2529	 * loop.
2530	 * Also we want to do the cache_save_setup first and then run the
2531	 * delayed refs to make sure we have the best chance at doing this all
2532	 * in one shot.
2533	 */
2534	spin_lock(&cur_trans->dirty_bgs_lock);
2535	while (!list_empty(&cur_trans->dirty_bgs)) {
2536		cache = list_first_entry(&cur_trans->dirty_bgs,
2537					 struct btrfs_block_group_cache,
2538					 dirty_list);
2539
2540		/*
2541		 * This can happen if cache_save_setup re-dirties a block group
2542		 * that is already under IO.  Just wait for it to finish and
2543		 * then do it all again
2544		 */
2545		if (!list_empty(&cache->io_list)) {
2546			spin_unlock(&cur_trans->dirty_bgs_lock);
2547			list_del_init(&cache->io_list);
2548			btrfs_wait_cache_io(trans, cache, path);
2549			btrfs_put_block_group(cache);
2550			spin_lock(&cur_trans->dirty_bgs_lock);
2551		}
2552
2553		/*
2554		 * Don't remove from the dirty list until after we've waited on
2555		 * any pending IO
2556		 */
2557		list_del_init(&cache->dirty_list);
2558		spin_unlock(&cur_trans->dirty_bgs_lock);
2559		should_put = 1;
2560
2561		cache_save_setup(cache, trans, path);
2562
2563		if (!ret)
2564			ret = btrfs_run_delayed_refs(trans,
2565						     (unsigned long) -1);
2566
2567		if (!ret && cache->disk_cache_state == BTRFS_DC_SETUP) {
2568			cache->io_ctl.inode = NULL;
2569			ret = btrfs_write_out_cache(trans, cache, path);
2570			if (ret == 0 && cache->io_ctl.inode) {
2571				num_started++;
2572				should_put = 0;
2573				list_add_tail(&cache->io_list, io);
2574			} else {
2575				/*
2576				 * If we failed to write the cache, the
2577				 * generation will be bad and life goes on
2578				 */
2579				ret = 0;
2580			}
2581		}
2582		if (!ret) {
2583			ret = write_one_cache_group(trans, path, cache);
2584			/*
2585			 * One of the free space endio workers might have
2586			 * created a new block group while updating a free space
2587			 * cache's inode (at inode.c:btrfs_finish_ordered_io())
2588			 * and hasn't released its transaction handle yet, in
2589			 * which case the new block group is still attached to
2590			 * its transaction handle and its creation has not
2591			 * finished yet (no block group item in the extent tree
2592			 * yet, etc). If this is the case, wait for all free
2593			 * space endio workers to finish and retry. This is a
2594			 * a very rare case so no need for a more efficient and
2595			 * complex approach.
2596			 */
2597			if (ret == -ENOENT) {
2598				wait_event(cur_trans->writer_wait,
2599				   atomic_read(&cur_trans->num_writers) == 1);
2600				ret = write_one_cache_group(trans, path, cache);
2601			}
2602			if (ret)
2603				btrfs_abort_transaction(trans, ret);
2604		}
2605
2606		/* If its not on the io list, we need to put the block group */
2607		if (should_put)
2608			btrfs_put_block_group(cache);
2609		btrfs_delayed_refs_rsv_release(fs_info, 1);
2610		spin_lock(&cur_trans->dirty_bgs_lock);
2611	}
2612	spin_unlock(&cur_trans->dirty_bgs_lock);
2613
2614	/*
2615	 * Refer to the definition of io_bgs member for details why it's safe
2616	 * to use it without any locking
2617	 */
2618	while (!list_empty(io)) {
2619		cache = list_first_entry(io, struct btrfs_block_group_cache,
2620					 io_list);
2621		list_del_init(&cache->io_list);
2622		btrfs_wait_cache_io(trans, cache, path);
2623		btrfs_put_block_group(cache);
2624	}
2625
2626	btrfs_free_path(path);
2627	return ret;
2628}
2629
2630int btrfs_update_block_group(struct btrfs_trans_handle *trans,
2631			     u64 bytenr, u64 num_bytes, int alloc)
2632{
2633	struct btrfs_fs_info *info = trans->fs_info;
2634	struct btrfs_block_group_cache *cache = NULL;
2635	u64 total = num_bytes;
2636	u64 old_val;
2637	u64 byte_in_group;
2638	int factor;
2639	int ret = 0;
2640
2641	/* Block accounting for super block */
2642	spin_lock(&info->delalloc_root_lock);
2643	old_val = btrfs_super_bytes_used(info->super_copy);
2644	if (alloc)
2645		old_val += num_bytes;
2646	else
2647		old_val -= num_bytes;
2648	btrfs_set_super_bytes_used(info->super_copy, old_val);
2649	spin_unlock(&info->delalloc_root_lock);
2650
2651	while (total) {
2652		cache = btrfs_lookup_block_group(info, bytenr);
2653		if (!cache) {
2654			ret = -ENOENT;
2655			break;
2656		}
2657		factor = btrfs_bg_type_to_factor(cache->flags);
2658
2659		/*
2660		 * If this block group has free space cache written out, we
2661		 * need to make sure to load it if we are removing space.  This
2662		 * is because we need the unpinning stage to actually add the
2663		 * space back to the block group, otherwise we will leak space.
2664		 */
2665		if (!alloc && cache->cached == BTRFS_CACHE_NO)
2666			btrfs_cache_block_group(cache, 1);
2667
2668		byte_in_group = bytenr - cache->key.objectid;
2669		WARN_ON(byte_in_group > cache->key.offset);
2670
2671		spin_lock(&cache->space_info->lock);
2672		spin_lock(&cache->lock);
2673
2674		if (btrfs_test_opt(info, SPACE_CACHE) &&
2675		    cache->disk_cache_state < BTRFS_DC_CLEAR)
2676			cache->disk_cache_state = BTRFS_DC_CLEAR;
2677
2678		old_val = btrfs_block_group_used(&cache->item);
2679		num_bytes = min(total, cache->key.offset - byte_in_group);
2680		if (alloc) {
2681			old_val += num_bytes;
2682			btrfs_set_block_group_used(&cache->item, old_val);
2683			cache->reserved -= num_bytes;
2684			cache->space_info->bytes_reserved -= num_bytes;
2685			cache->space_info->bytes_used += num_bytes;
2686			cache->space_info->disk_used += num_bytes * factor;
2687			spin_unlock(&cache->lock);
2688			spin_unlock(&cache->space_info->lock);
2689		} else {
2690			old_val -= num_bytes;
2691			btrfs_set_block_group_used(&cache->item, old_val);
2692			cache->pinned += num_bytes;
2693			btrfs_space_info_update_bytes_pinned(info,
2694					cache->space_info, num_bytes);
2695			cache->space_info->bytes_used -= num_bytes;
2696			cache->space_info->disk_used -= num_bytes * factor;
2697			spin_unlock(&cache->lock);
2698			spin_unlock(&cache->space_info->lock);
2699
2700			percpu_counter_add_batch(
2701					&cache->space_info->total_bytes_pinned,
2702					num_bytes,
2703					BTRFS_TOTAL_BYTES_PINNED_BATCH);
2704			set_extent_dirty(info->pinned_extents,
2705					 bytenr, bytenr + num_bytes - 1,
2706					 GFP_NOFS | __GFP_NOFAIL);
2707		}
2708
2709		spin_lock(&trans->transaction->dirty_bgs_lock);
2710		if (list_empty(&cache->dirty_list)) {
2711			list_add_tail(&cache->dirty_list,
2712				      &trans->transaction->dirty_bgs);
2713			trans->delayed_ref_updates++;
2714			btrfs_get_block_group(cache);
2715		}
2716		spin_unlock(&trans->transaction->dirty_bgs_lock);
2717
2718		/*
2719		 * No longer have used bytes in this block group, queue it for
2720		 * deletion. We do this after adding the block group to the
2721		 * dirty list to avoid races between cleaner kthread and space
2722		 * cache writeout.
2723		 */
2724		if (!alloc && old_val == 0)
2725			btrfs_mark_bg_unused(cache);
2726
2727		btrfs_put_block_group(cache);
2728		total -= num_bytes;
2729		bytenr += num_bytes;
2730	}
2731
2732	/* Modified block groups are accounted for in the delayed_refs_rsv. */
2733	btrfs_update_delayed_refs_rsv(trans);
2734	return ret;
2735}
2736
2737/**
2738 * btrfs_add_reserved_bytes - update the block_group and space info counters
2739 * @cache:	The cache we are manipulating
2740 * @ram_bytes:  The number of bytes of file content, and will be same to
2741 *              @num_bytes except for the compress path.
2742 * @num_bytes:	The number of bytes in question
2743 * @delalloc:   The blocks are allocated for the delalloc write
2744 *
2745 * This is called by the allocator when it reserves space. If this is a
2746 * reservation and the block group has become read only we cannot make the
2747 * reservation and return -EAGAIN, otherwise this function always succeeds.
2748 */
2749int btrfs_add_reserved_bytes(struct btrfs_block_group_cache *cache,
2750			     u64 ram_bytes, u64 num_bytes, int delalloc)
2751{
2752	struct btrfs_space_info *space_info = cache->space_info;
2753	int ret = 0;
2754
2755	spin_lock(&space_info->lock);
2756	spin_lock(&cache->lock);
2757	if (cache->ro) {
2758		ret = -EAGAIN;
2759	} else {
2760		cache->reserved += num_bytes;
2761		space_info->bytes_reserved += num_bytes;
2762		trace_btrfs_space_reservation(cache->fs_info, "space_info",
2763					      space_info->flags, num_bytes, 1);
2764		btrfs_space_info_update_bytes_may_use(cache->fs_info,
2765						      space_info, -ram_bytes);
2766		if (delalloc)
2767			cache->delalloc_bytes += num_bytes;
2768	}
2769	spin_unlock(&cache->lock);
2770	spin_unlock(&space_info->lock);
2771	return ret;
2772}
2773
2774/**
2775 * btrfs_free_reserved_bytes - update the block_group and space info counters
2776 * @cache:      The cache we are manipulating
2777 * @num_bytes:  The number of bytes in question
2778 * @delalloc:   The blocks are allocated for the delalloc write
2779 *
2780 * This is called by somebody who is freeing space that was never actually used
2781 * on disk.  For example if you reserve some space for a new leaf in transaction
2782 * A and before transaction A commits you free that leaf, you call this with
2783 * reserve set to 0 in order to clear the reservation.
2784 */
2785void btrfs_free_reserved_bytes(struct btrfs_block_group_cache *cache,
2786			       u64 num_bytes, int delalloc)
2787{
2788	struct btrfs_space_info *space_info = cache->space_info;
2789
2790	spin_lock(&space_info->lock);
2791	spin_lock(&cache->lock);
2792	if (cache->ro)
2793		space_info->bytes_readonly += num_bytes;
2794	cache->reserved -= num_bytes;
2795	space_info->bytes_reserved -= num_bytes;
2796	space_info->max_extent_size = 0;
2797
2798	if (delalloc)
2799		cache->delalloc_bytes -= num_bytes;
2800	spin_unlock(&cache->lock);
2801	spin_unlock(&space_info->lock);
2802}
2803
2804static void force_metadata_allocation(struct btrfs_fs_info *info)
2805{
2806	struct list_head *head = &info->space_info;
2807	struct btrfs_space_info *found;
2808
2809	rcu_read_lock();
2810	list_for_each_entry_rcu(found, head, list) {
2811		if (found->flags & BTRFS_BLOCK_GROUP_METADATA)
2812			found->force_alloc = CHUNK_ALLOC_FORCE;
2813	}
2814	rcu_read_unlock();
2815}
2816
2817static int should_alloc_chunk(struct btrfs_fs_info *fs_info,
2818			      struct btrfs_space_info *sinfo, int force)
2819{
2820	u64 bytes_used = btrfs_space_info_used(sinfo, false);
2821	u64 thresh;
2822
2823	if (force == CHUNK_ALLOC_FORCE)
2824		return 1;
2825
2826	/*
2827	 * in limited mode, we want to have some free space up to
2828	 * about 1% of the FS size.
2829	 */
2830	if (force == CHUNK_ALLOC_LIMITED) {
2831		thresh = btrfs_super_total_bytes(fs_info->super_copy);
2832		thresh = max_t(u64, SZ_64M, div_factor_fine(thresh, 1));
2833
2834		if (sinfo->total_bytes - bytes_used < thresh)
2835			return 1;
2836	}
2837
2838	if (bytes_used + SZ_2M < div_factor(sinfo->total_bytes, 8))
2839		return 0;
2840	return 1;
2841}
2842
2843int btrfs_force_chunk_alloc(struct btrfs_trans_handle *trans, u64 type)
2844{
2845	u64 alloc_flags = btrfs_get_alloc_profile(trans->fs_info, type);
2846
2847	return btrfs_chunk_alloc(trans, alloc_flags, CHUNK_ALLOC_FORCE);
2848}
2849
2850/*
2851 * If force is CHUNK_ALLOC_FORCE:
2852 *    - return 1 if it successfully allocates a chunk,
2853 *    - return errors including -ENOSPC otherwise.
2854 * If force is NOT CHUNK_ALLOC_FORCE:
2855 *    - return 0 if it doesn't need to allocate a new chunk,
2856 *    - return 1 if it successfully allocates a chunk,
2857 *    - return errors including -ENOSPC otherwise.
2858 */
2859int btrfs_chunk_alloc(struct btrfs_trans_handle *trans, u64 flags,
2860		      enum btrfs_chunk_alloc_enum force)
2861{
2862	struct btrfs_fs_info *fs_info = trans->fs_info;
2863	struct btrfs_space_info *space_info;
2864	bool wait_for_alloc = false;
2865	bool should_alloc = false;
2866	int ret = 0;
2867
2868	/* Don't re-enter if we're already allocating a chunk */
2869	if (trans->allocating_chunk)
2870		return -ENOSPC;
2871
2872	space_info = btrfs_find_space_info(fs_info, flags);
2873	ASSERT(space_info);
2874
2875	do {
2876		spin_lock(&space_info->lock);
2877		if (force < space_info->force_alloc)
2878			force = space_info->force_alloc;
2879		should_alloc = should_alloc_chunk(fs_info, space_info, force);
2880		if (space_info->full) {
2881			/* No more free physical space */
2882			if (should_alloc)
2883				ret = -ENOSPC;
2884			else
2885				ret = 0;
2886			spin_unlock(&space_info->lock);
2887			return ret;
2888		} else if (!should_alloc) {
2889			spin_unlock(&space_info->lock);
2890			return 0;
2891		} else if (space_info->chunk_alloc) {
2892			/*
2893			 * Someone is already allocating, so we need to block
2894			 * until this someone is finished and then loop to
2895			 * recheck if we should continue with our allocation
2896			 * attempt.
2897			 */
2898			wait_for_alloc = true;
2899			spin_unlock(&space_info->lock);
2900			mutex_lock(&fs_info->chunk_mutex);
2901			mutex_unlock(&fs_info->chunk_mutex);
2902		} else {
2903			/* Proceed with allocation */
2904			space_info->chunk_alloc = 1;
2905			wait_for_alloc = false;
2906			spin_unlock(&space_info->lock);
2907		}
2908
2909		cond_resched();
2910	} while (wait_for_alloc);
2911
2912	mutex_lock(&fs_info->chunk_mutex);
2913	trans->allocating_chunk = true;
2914
2915	/*
2916	 * If we have mixed data/metadata chunks we want to make sure we keep
2917	 * allocating mixed chunks instead of individual chunks.
2918	 */
2919	if (btrfs_mixed_space_info(space_info))
2920		flags |= (BTRFS_BLOCK_GROUP_DATA | BTRFS_BLOCK_GROUP_METADATA);
2921
2922	/*
2923	 * if we're doing a data chunk, go ahead and make sure that
2924	 * we keep a reasonable number of metadata chunks allocated in the
2925	 * FS as well.
2926	 */
2927	if (flags & BTRFS_BLOCK_GROUP_DATA && fs_info->metadata_ratio) {
2928		fs_info->data_chunk_allocations++;
2929		if (!(fs_info->data_chunk_allocations %
2930		      fs_info->metadata_ratio))
2931			force_metadata_allocation(fs_info);
2932	}
2933
2934	/*
2935	 * Check if we have enough space in SYSTEM chunk because we may need
2936	 * to update devices.
2937	 */
2938	check_system_chunk(trans, flags);
2939
2940	ret = btrfs_alloc_chunk(trans, flags);
2941	trans->allocating_chunk = false;
2942
2943	spin_lock(&space_info->lock);
2944	if (ret < 0) {
2945		if (ret == -ENOSPC)
2946			space_info->full = 1;
2947		else
2948			goto out;
2949	} else {
2950		ret = 1;
2951		space_info->max_extent_size = 0;
2952	}
2953
2954	space_info->force_alloc = CHUNK_ALLOC_NO_FORCE;
2955out:
2956	space_info->chunk_alloc = 0;
2957	spin_unlock(&space_info->lock);
2958	mutex_unlock(&fs_info->chunk_mutex);
2959	/*
2960	 * When we allocate a new chunk we reserve space in the chunk block
2961	 * reserve to make sure we can COW nodes/leafs in the chunk tree or
2962	 * add new nodes/leafs to it if we end up needing to do it when
2963	 * inserting the chunk item and updating device items as part of the
2964	 * second phase of chunk allocation, performed by
2965	 * btrfs_finish_chunk_alloc(). So make sure we don't accumulate a
2966	 * large number of new block groups to create in our transaction
2967	 * handle's new_bgs list to avoid exhausting the chunk block reserve
2968	 * in extreme cases - like having a single transaction create many new
2969	 * block groups when starting to write out the free space caches of all
2970	 * the block groups that were made dirty during the lifetime of the
2971	 * transaction.
2972	 */
2973	if (trans->chunk_bytes_reserved >= (u64)SZ_2M)
2974		btrfs_create_pending_block_groups(trans);
2975
2976	return ret;
2977}
2978
2979static u64 get_profile_num_devs(struct btrfs_fs_info *fs_info, u64 type)
2980{
2981	u64 num_dev;
2982
2983	num_dev = btrfs_raid_array[btrfs_bg_flags_to_raid_index(type)].devs_max;
2984	if (!num_dev)
2985		num_dev = fs_info->fs_devices->rw_devices;
2986
2987	return num_dev;
2988}
2989
2990/*
2991 * If @is_allocation is true, reserve space in the system space info necessary
2992 * for allocating a chunk, otherwise if it's false, reserve space necessary for
2993 * removing a chunk.
2994 */
2995void check_system_chunk(struct btrfs_trans_handle *trans, u64 type)
2996{
2997	struct btrfs_fs_info *fs_info = trans->fs_info;
2998	struct btrfs_space_info *info;
2999	u64 left;
3000	u64 thresh;
3001	int ret = 0;
3002	u64 num_devs;
3003
3004	/*
3005	 * Needed because we can end up allocating a system chunk and for an
3006	 * atomic and race free space reservation in the chunk block reserve.
3007	 */
3008	lockdep_assert_held(&fs_info->chunk_mutex);
3009
3010	info = btrfs_find_space_info(fs_info, BTRFS_BLOCK_GROUP_SYSTEM);
3011	spin_lock(&info->lock);
3012	left = info->total_bytes - btrfs_space_info_used(info, true);
3013	spin_unlock(&info->lock);
3014
3015	num_devs = get_profile_num_devs(fs_info, type);
3016
3017	/* num_devs device items to update and 1 chunk item to add or remove */
3018	thresh = btrfs_calc_metadata_size(fs_info, num_devs) +
3019		btrfs_calc_insert_metadata_size(fs_info, 1);
3020
3021	if (left < thresh && btrfs_test_opt(fs_info, ENOSPC_DEBUG)) {
3022		btrfs_info(fs_info, "left=%llu, need=%llu, flags=%llu",
3023			   left, thresh, type);
3024		btrfs_dump_space_info(fs_info, info, 0, 0);
3025	}
3026
3027	if (left < thresh) {
3028		u64 flags = btrfs_system_alloc_profile(fs_info);
3029
3030		/*
3031		 * Ignore failure to create system chunk. We might end up not
3032		 * needing it, as we might not need to COW all nodes/leafs from
3033		 * the paths we visit in the chunk tree (they were already COWed
3034		 * or created in the current transaction for example).
3035		 */
3036		ret = btrfs_alloc_chunk(trans, flags);
3037	}
3038
3039	if (!ret) {
3040		ret = btrfs_block_rsv_add(fs_info->chunk_root,
3041					  &fs_info->chunk_block_rsv,
3042					  thresh, BTRFS_RESERVE_NO_FLUSH);
3043		if (!ret)
3044			trans->chunk_bytes_reserved += thresh;
3045	}
3046}
3047
3048void btrfs_put_block_group_cache(struct btrfs_fs_info *info)
3049{
3050	struct btrfs_block_group_cache *block_group;
3051	u64 last = 0;
3052
3053	while (1) {
3054		struct inode *inode;
3055
3056		block_group = btrfs_lookup_first_block_group(info, last);
3057		while (block_group) {
3058			btrfs_wait_block_group_cache_done(block_group);
3059			spin_lock(&block_group->lock);
3060			if (block_group->iref)
3061				break;
3062			spin_unlock(&block_group->lock);
3063			block_group = btrfs_next_block_group(block_group);
3064		}
3065		if (!block_group) {
3066			if (last == 0)
3067				break;
3068			last = 0;
3069			continue;
3070		}
3071
3072		inode = block_group->inode;
3073		block_group->iref = 0;
3074		block_group->inode = NULL;
3075		spin_unlock(&block_group->lock);
3076		ASSERT(block_group->io_ctl.inode == NULL);
3077		iput(inode);
3078		last = block_group->key.objectid + block_group->key.offset;
3079		btrfs_put_block_group(block_group);
3080	}
3081}
3082
3083/*
3084 * Must be called only after stopping all workers, since we could have block
3085 * group caching kthreads running, and therefore they could race with us if we
3086 * freed the block groups before stopping them.
3087 */
3088int btrfs_free_block_groups(struct btrfs_fs_info *info)
3089{
3090	struct btrfs_block_group_cache *block_group;
3091	struct btrfs_space_info *space_info;
3092	struct btrfs_caching_control *caching_ctl;
3093	struct rb_node *n;
3094
3095	down_write(&info->commit_root_sem);
3096	while (!list_empty(&info->caching_block_groups)) {
3097		caching_ctl = list_entry(info->caching_block_groups.next,
3098					 struct btrfs_caching_control, list);
3099		list_del(&caching_ctl->list);
3100		btrfs_put_caching_control(caching_ctl);
3101	}
3102	up_write(&info->commit_root_sem);
3103
3104	spin_lock(&info->unused_bgs_lock);
3105	while (!list_empty(&info->unused_bgs)) {
3106		block_group = list_first_entry(&info->unused_bgs,
3107					       struct btrfs_block_group_cache,
3108					       bg_list);
3109		list_del_init(&block_group->bg_list);
3110		btrfs_put_block_group(block_group);
3111	}
3112	spin_unlock(&info->unused_bgs_lock);
3113
3114	spin_lock(&info->block_group_cache_lock);
3115	while ((n = rb_last(&info->block_group_cache_tree)) != NULL) {
3116		block_group = rb_entry(n, struct btrfs_block_group_cache,
3117				       cache_node);
3118		rb_erase(&block_group->cache_node,
3119			 &info->block_group_cache_tree);
3120		RB_CLEAR_NODE(&block_group->cache_node);
3121		spin_unlock(&info->block_group_cache_lock);
3122
3123		down_write(&block_group->space_info->groups_sem);
3124		list_del(&block_group->list);
3125		up_write(&block_group->space_info->groups_sem);
3126
3127		/*
3128		 * We haven't cached this block group, which means we could
3129		 * possibly have excluded extents on this block group.
3130		 */
3131		if (block_group->cached == BTRFS_CACHE_NO ||
3132		    block_group->cached == BTRFS_CACHE_ERROR)
3133			btrfs_free_excluded_extents(block_group);
3134
3135		btrfs_remove_free_space_cache(block_group);
3136		ASSERT(block_group->cached != BTRFS_CACHE_STARTED);
3137		ASSERT(list_empty(&block_group->dirty_list));
3138		ASSERT(list_empty(&block_group->io_list));
3139		ASSERT(list_empty(&block_group->bg_list));
3140		ASSERT(atomic_read(&block_group->count) == 1);
3141		btrfs_put_block_group(block_group);
3142
3143		spin_lock(&info->block_group_cache_lock);
3144	}
3145	spin_unlock(&info->block_group_cache_lock);
3146
3147	/*
3148	 * Now that all the block groups are freed, go through and free all the
3149	 * space_info structs.  This is only called during the final stages of
3150	 * unmount, and so we know nobody is using them.  We call
3151	 * synchronize_rcu() once before we start, just to be on the safe side.
3152	 */
3153	synchronize_rcu();
3154
3155	btrfs_release_global_block_rsv(info);
3156
3157	while (!list_empty(&info->space_info)) {
3158		space_info = list_entry(info->space_info.next,
3159					struct btrfs_space_info,
3160					list);
3161
3162		/*
3163		 * Do not hide this behind enospc_debug, this is actually
3164		 * important and indicates a real bug if this happens.
3165		 */
3166		if (WARN_ON(space_info->bytes_pinned > 0 ||
3167			    space_info->bytes_reserved > 0 ||
3168			    space_info->bytes_may_use > 0))
3169			btrfs_dump_space_info(info, space_info, 0, 0);
3170		list_del(&space_info->list);
3171		btrfs_sysfs_remove_space_info(space_info);
3172	}
3173	return 0;
3174}