1// SPDX-License-Identifier: GPL-2.0
   2
   3#include <linux/bitops.h>
   4#include <linux/slab.h>
   5#include <linux/bio.h>
   6#include <linux/mm.h>
   7#include <linux/pagemap.h>
   8#include <linux/page-flags.h>
   9#include <linux/sched/mm.h>
  10#include <linux/spinlock.h>
  11#include <linux/blkdev.h>
  12#include <linux/swap.h>
  13#include <linux/writeback.h>
  14#include <linux/pagevec.h>
  15#include <linux/prefetch.h>
  16#include <linux/fsverity.h>
  17#include "extent_io.h"
  18#include "extent-io-tree.h"
  19#include "extent_map.h"
  20#include "ctree.h"
  21#include "btrfs_inode.h"
  22#include "bio.h"
 
  23#include "locking.h"
 
  24#include "backref.h"
  25#include "disk-io.h"
  26#include "subpage.h"
  27#include "zoned.h"
  28#include "block-group.h"
  29#include "compression.h"
  30#include "fs.h"
  31#include "accessors.h"
  32#include "file-item.h"
  33#include "file.h"
  34#include "dev-replace.h"
  35#include "super.h"
  36#include "transaction.h"
  37
 
  38static struct kmem_cache *extent_buffer_cache;
 
 
 
 
 
 
  39
  40#ifdef CONFIG_BTRFS_DEBUG
  41static inline void btrfs_leak_debug_add_eb(struct extent_buffer *eb)
 
 
 
 
 
 
  42{
  43	struct btrfs_fs_info *fs_info = eb->fs_info;
  44	unsigned long flags;
  45
  46	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
  47	list_add(&eb->leak_list, &fs_info->allocated_ebs);
  48	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
  49}
  50
  51static inline void btrfs_leak_debug_del_eb(struct extent_buffer *eb)
 
  52{
  53	struct btrfs_fs_info *fs_info = eb->fs_info;
  54	unsigned long flags;
  55
  56	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
  57	list_del(&eb->leak_list);
  58	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
  59}
  60
  61void btrfs_extent_buffer_leak_debug_check(struct btrfs_fs_info *fs_info)
 
  62{
 
  63	struct extent_buffer *eb;
  64	unsigned long flags;
  65
  66	/*
  67	 * If we didn't get into open_ctree our allocated_ebs will not be
  68	 * initialized, so just skip this.
  69	 */
  70	if (!fs_info->allocated_ebs.next)
  71		return;
 
 
 
  72
  73	WARN_ON(!list_empty(&fs_info->allocated_ebs));
  74	spin_lock_irqsave(&fs_info->eb_leak_lock, flags);
  75	while (!list_empty(&fs_info->allocated_ebs)) {
  76		eb = list_first_entry(&fs_info->allocated_ebs,
  77				      struct extent_buffer, leak_list);
  78		pr_err(
  79	"BTRFS: buffer leak start %llu len %u refs %d bflags %lu owner %llu\n",
  80		       eb->start, eb->len, atomic_read(&eb->refs), eb->bflags,
  81		       btrfs_header_owner(eb));
  82		list_del(&eb->leak_list);
  83		WARN_ON_ONCE(1);
  84		kmem_cache_free(extent_buffer_cache, eb);
  85	}
  86	spin_unlock_irqrestore(&fs_info->eb_leak_lock, flags);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
  87}
  88#else
  89#define btrfs_leak_debug_add_eb(eb)			do {} while (0)
  90#define btrfs_leak_debug_del_eb(eb)			do {} while (0)
 
 
  91#endif
  92
  93/*
  94 * Structure to record info about the bio being assembled, and other info like
  95 * how many bytes are there before stripe/ordered extent boundary.
  96 */
  97struct btrfs_bio_ctrl {
  98	struct btrfs_bio *bbio;
  99	enum btrfs_compression_type compress_type;
 100	u32 len_to_oe_boundary;
 101	blk_opf_t opf;
 102	btrfs_bio_end_io_t end_io_func;
 103	struct writeback_control *wbc;
 104
 105	/*
 106	 * The sectors of the page which are going to be submitted by
 107	 * extent_writepage_io().
 108	 * This is to avoid touching ranges covered by compression/inline.
 
 
 
 
 
 
 
 
 
 
 109	 */
 110	unsigned long submit_bitmap;
 
 
 
 111};
 112
 113static void submit_one_bio(struct btrfs_bio_ctrl *bio_ctrl)
 
 
 114{
 115	struct btrfs_bio *bbio = bio_ctrl->bbio;
 116
 117	if (!bbio)
 
 
 
 
 118		return;
 
 
 
 
 
 
 119
 120	/* Caller should ensure the bio has at least some range added */
 121	ASSERT(bbio->bio.bi_iter.bi_size);
 
 
 
 
 
 
 122
 123	if (btrfs_op(&bbio->bio) == BTRFS_MAP_READ &&
 124	    bio_ctrl->compress_type != BTRFS_COMPRESS_NONE)
 125		btrfs_submit_compressed_read(bbio);
 126	else
 127		btrfs_submit_bbio(bbio, 0);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 128
 129	/* The bbio is owned by the end_io handler now */
 130	bio_ctrl->bbio = NULL;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 131}
 132
 133/*
 134 * Submit or fail the current bio in the bio_ctrl structure.
 
 
 
 
 
 
 135 */
 136static void submit_write_bio(struct btrfs_bio_ctrl *bio_ctrl, int ret)
 
 137{
 138	struct btrfs_bio *bbio = bio_ctrl->bbio;
 
 139
 140	if (!bbio)
 141		return;
 142
 143	if (ret) {
 144		ASSERT(ret < 0);
 145		btrfs_bio_end_io(bbio, errno_to_blk_status(ret));
 146		/* The bio is owned by the end_io handler now */
 147		bio_ctrl->bbio = NULL;
 148	} else {
 149		submit_one_bio(bio_ctrl);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 150	}
 151}
 152
 153int __init extent_buffer_init_cachep(void)
 
 154{
 155	extent_buffer_cache = kmem_cache_create("btrfs_extent_buffer",
 156						sizeof(struct extent_buffer), 0, 0,
 157						NULL);
 158	if (!extent_buffer_cache)
 159		return -ENOMEM;
 
 
 
 
 
 160
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 161	return 0;
 162}
 163
 164void __cold extent_buffer_free_cachep(void)
 
 165{
 166	/*
 167	 * Make sure all delayed rcu free are flushed before we
 168	 * destroy caches.
 169	 */
 170	rcu_barrier();
 171	kmem_cache_destroy(extent_buffer_cache);
 172}
 173
 174static void process_one_folio(struct btrfs_fs_info *fs_info,
 175			      struct folio *folio, const struct folio *locked_folio,
 176			      unsigned long page_ops, u64 start, u64 end)
 
 
 
 
 
 
 
 
 
 
 
 
 
 177{
 178	u32 len;
 179
 180	ASSERT(end + 1 - start != 0 && end + 1 - start < U32_MAX);
 181	len = end + 1 - start;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 182
 183	if (page_ops & PAGE_SET_ORDERED)
 184		btrfs_folio_clamp_set_ordered(fs_info, folio, start, len);
 185	if (page_ops & PAGE_START_WRITEBACK) {
 186		btrfs_folio_clamp_clear_dirty(fs_info, folio, start, len);
 187		btrfs_folio_clamp_set_writeback(fs_info, folio, start, len);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 188	}
 189	if (page_ops & PAGE_END_WRITEBACK)
 190		btrfs_folio_clamp_clear_writeback(fs_info, folio, start, len);
 191
 192	if (folio != locked_folio && (page_ops & PAGE_UNLOCK))
 193		btrfs_folio_end_lock(fs_info, folio, start, len);
 
 
 
 
 
 194}
 195
 196static void __process_folios_contig(struct address_space *mapping,
 197				    const struct folio *locked_folio, u64 start,
 198				    u64 end, unsigned long page_ops)
 199{
 200	struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host);
 201	pgoff_t start_index = start >> PAGE_SHIFT;
 202	pgoff_t end_index = end >> PAGE_SHIFT;
 203	pgoff_t index = start_index;
 204	struct folio_batch fbatch;
 205	int i;
 206
 207	folio_batch_init(&fbatch);
 208	while (index <= end_index) {
 209		int found_folios;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 210
 211		found_folios = filemap_get_folios_contig(mapping, &index,
 212				end_index, &fbatch);
 213		for (i = 0; i < found_folios; i++) {
 214			struct folio *folio = fbatch.folios[i];
 
 215
 216			process_one_folio(fs_info, folio, locked_folio,
 217					  page_ops, start, end);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 218		}
 219		folio_batch_release(&fbatch);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 220		cond_resched();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 221	}
 
 
 222}
 223
 224static noinline void __unlock_for_delalloc(const struct inode *inode,
 225					   const struct folio *locked_folio,
 226					   u64 start, u64 end)
 227{
 228	unsigned long index = start >> PAGE_SHIFT;
 229	unsigned long end_index = end >> PAGE_SHIFT;
 230
 231	ASSERT(locked_folio);
 232	if (index == locked_folio->index && end_index == index)
 233		return;
 
 
 
 
 
 234
 235	__process_folios_contig(inode->i_mapping, locked_folio, start, end,
 236				PAGE_UNLOCK);
 
 
 
 
 
 
 
 
 237}
 238
 239static noinline int lock_delalloc_folios(struct inode *inode,
 240					 const struct folio *locked_folio,
 241					 u64 start, u64 end)
 242{
 243	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
 244	struct address_space *mapping = inode->i_mapping;
 245	pgoff_t start_index = start >> PAGE_SHIFT;
 246	pgoff_t end_index = end >> PAGE_SHIFT;
 247	pgoff_t index = start_index;
 248	u64 processed_end = start;
 249	struct folio_batch fbatch;
 250
 251	if (index == locked_folio->index && index == end_index)
 252		return 0;
 
 
 
 
 
 
 
 
 253
 254	folio_batch_init(&fbatch);
 255	while (index <= end_index) {
 256		unsigned int found_folios, i;
 
 
 
 
 
 
 
 
 
 
 
 
 
 257
 258		found_folios = filemap_get_folios_contig(mapping, &index,
 259				end_index, &fbatch);
 260		if (found_folios == 0)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 261			goto out;
 
 262
 263		for (i = 0; i < found_folios; i++) {
 264			struct folio *folio = fbatch.folios[i];
 265			u64 range_start;
 266			u32 range_len;
 
 
 
 
 
 
 
 
 267
 268			if (folio == locked_folio)
 269				continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 270
 271			folio_lock(folio);
 272			if (!folio_test_dirty(folio) || folio->mapping != mapping) {
 273				folio_unlock(folio);
 
 
 
 
 
 
 
 
 
 
 
 274				goto out;
 275			}
 276			range_start = max_t(u64, folio_pos(folio), start);
 277			range_len = min_t(u64, folio_pos(folio) + folio_size(folio),
 278					  end + 1) - range_start;
 279			btrfs_folio_set_lock(fs_info, folio, range_start, range_len);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 280
 281			processed_end = range_start + range_len - 1;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 282		}
 283		folio_batch_release(&fbatch);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 284		cond_resched();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 285	}
 286
 287	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 288out:
 289	folio_batch_release(&fbatch);
 290	if (processed_end > start)
 291		__unlock_for_delalloc(inode, locked_folio, start,
 292				      processed_end);
 293	return -EAGAIN;
 294}
 295
 296/*
 297 * Find and lock a contiguous range of bytes in the file marked as delalloc, no
 298 * more than @max_bytes.
 
 299 *
 300 * @start:	The original start bytenr to search.
 301 *		Will store the extent range start bytenr.
 302 * @end:	The original end bytenr of the search range
 303 *		Will store the extent range end bytenr.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 304 *
 305 * Return true if we find a delalloc range which starts inside the original
 306 * range, and @start/@end will store the delalloc range start/end.
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 307 *
 308 * Return false if we can't find any delalloc range which starts inside the
 309 * original range, and @start/@end will be the non-delalloc range start/end.
 310 */
 311EXPORT_FOR_TESTS
 312noinline_for_stack bool find_lock_delalloc_range(struct inode *inode,
 313						 struct folio *locked_folio,
 314						 u64 *start, u64 *end)
 315{
 316	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
 317	struct extent_io_tree *tree = &BTRFS_I(inode)->io_tree;
 318	const u64 orig_start = *start;
 319	const u64 orig_end = *end;
 320	/* The sanity tests may not set a valid fs_info. */
 321	u64 max_bytes = fs_info ? fs_info->max_extent_size : BTRFS_MAX_EXTENT_SIZE;
 322	u64 delalloc_start;
 323	u64 delalloc_end;
 324	bool found;
 325	struct extent_state *cached_state = NULL;
 326	int ret;
 327	int loops = 0;
 328
 329	/* Caller should pass a valid @end to indicate the search range end */
 330	ASSERT(orig_end > orig_start);
 331
 332	/* The range should at least cover part of the folio */
 333	ASSERT(!(orig_start >= folio_pos(locked_folio) + folio_size(locked_folio) ||
 334		 orig_end <= folio_pos(locked_folio)));
 335again:
 336	/* step one, find a bunch of delalloc bytes starting at start */
 337	delalloc_start = *start;
 338	delalloc_end = 0;
 339	found = btrfs_find_delalloc_range(tree, &delalloc_start, &delalloc_end,
 340					  max_bytes, &cached_state);
 341	if (!found || delalloc_end <= *start || delalloc_start > orig_end) {
 342		*start = delalloc_start;
 343
 344		/* @delalloc_end can be -1, never go beyond @orig_end */
 345		*end = min(delalloc_end, orig_end);
 346		free_extent_state(cached_state);
 347		return false;
 348	}
 349
 350	/*
 351	 * start comes from the offset of locked_folio.  We have to lock
 352	 * folios in order, so we can't process delalloc bytes before
 353	 * locked_folio
 354	 */
 355	if (delalloc_start < *start)
 356		delalloc_start = *start;
 357
 358	/*
 359	 * make sure to limit the number of folios we try to lock down
 360	 */
 361	if (delalloc_end + 1 - delalloc_start > max_bytes)
 362		delalloc_end = delalloc_start + max_bytes - 1;
 363
 364	/* step two, lock all the folioss after the folios that has start */
 365	ret = lock_delalloc_folios(inode, locked_folio, delalloc_start,
 366				   delalloc_end);
 367	ASSERT(!ret || ret == -EAGAIN);
 368	if (ret == -EAGAIN) {
 369		/* some of the folios are gone, lets avoid looping by
 370		 * shortening the size of the delalloc range we're searching
 371		 */
 372		free_extent_state(cached_state);
 373		cached_state = NULL;
 374		if (!loops) {
 375			max_bytes = PAGE_SIZE;
 376			loops = 1;
 377			goto again;
 378		} else {
 379			found = false;
 380			goto out_failed;
 381		}
 382	}
 
 383
 384	/* step three, lock the state bits for the whole range */
 385	lock_extent(tree, delalloc_start, delalloc_end, &cached_state);
 386
 387	/* then test to make sure it is all still delalloc */
 388	ret = test_range_bit(tree, delalloc_start, delalloc_end,
 389			     EXTENT_DELALLOC, cached_state);
 390
 391	unlock_extent(tree, delalloc_start, delalloc_end, &cached_state);
 392	if (!ret) {
 393		__unlock_for_delalloc(inode, locked_folio, delalloc_start,
 394				      delalloc_end);
 
 
 395		cond_resched();
 396		goto again;
 397	}
 
 398	*start = delalloc_start;
 399	*end = delalloc_end;
 400out_failed:
 401	return found;
 402}
 403
 404void extent_clear_unlock_delalloc(struct btrfs_inode *inode, u64 start, u64 end,
 405				  const struct folio *locked_folio,
 406				  struct extent_state **cached,
 407				  u32 clear_bits, unsigned long page_ops)
 408{
 409	clear_extent_bit(&inode->io_tree, start, end, clear_bits, cached);
 
 
 
 
 
 
 410
 411	__process_folios_contig(inode->vfs_inode.i_mapping, locked_folio, start,
 412				end, page_ops);
 413}
 414
 415static bool btrfs_verify_folio(struct folio *folio, u64 start, u32 len)
 416{
 417	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
 
 
 
 
 
 
 
 
 418
 419	if (!fsverity_active(folio->mapping->host) ||
 420	    btrfs_folio_test_uptodate(fs_info, folio, start, len) ||
 421	    start >= i_size_read(folio->mapping->host))
 422		return true;
 423	return fsverity_verify_folio(folio);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 424}
 425
 426static void end_folio_read(struct folio *folio, bool uptodate, u64 start, u32 len)
 427{
 428	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
 
 
 
 
 
 
 
 
 
 
 
 
 429
 430	ASSERT(folio_pos(folio) <= start &&
 431	       start + len <= folio_pos(folio) + PAGE_SIZE);
 432
 433	if (uptodate && btrfs_verify_folio(folio, start, len))
 434		btrfs_folio_set_uptodate(fs_info, folio, start, len);
 435	else
 436		btrfs_folio_clear_uptodate(fs_info, folio, start, len);
 
 
 
 
 
 
 
 
 437
 438	if (!btrfs_is_subpage(fs_info, folio->mapping))
 439		folio_unlock(folio);
 440	else
 441		btrfs_folio_end_lock(fs_info, folio, start, len);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 442}
 443
 444/*
 445 * After a write IO is done, we need to:
 446 *
 447 * - clear the uptodate bits on error
 448 * - clear the writeback bits in the extent tree for the range
 449 * - filio_end_writeback()  if there is no more pending io for the folio
 450 *
 451 * Scheduling is not allowed, so the extent state tree is expected
 452 * to have one and only one object corresponding to this IO.
 453 */
 454static void end_bbio_data_write(struct btrfs_bio *bbio)
 
 455{
 456	struct btrfs_fs_info *fs_info = bbio->fs_info;
 457	struct bio *bio = &bbio->bio;
 458	int error = blk_status_to_errno(bio->bi_status);
 459	struct folio_iter fi;
 460	const u32 sectorsize = fs_info->sectorsize;
 461
 462	ASSERT(!bio_flagged(bio, BIO_CLONED));
 463	bio_for_each_folio_all(fi, bio) {
 464		struct folio *folio = fi.folio;
 465		u64 start = folio_pos(folio) + fi.offset;
 466		u32 len = fi.length;
 467
 468		/* Only order 0 (single page) folios are allowed for data. */
 469		ASSERT(folio_order(folio) == 0);
 470
 471		/* Our read/write should always be sector aligned. */
 472		if (!IS_ALIGNED(fi.offset, sectorsize))
 473			btrfs_err(fs_info,
 474		"partial page write in btrfs with offset %zu and length %zu",
 475				  fi.offset, fi.length);
 476		else if (!IS_ALIGNED(fi.length, sectorsize))
 477			btrfs_info(fs_info,
 478		"incomplete page write with offset %zu and length %zu",
 479				   fi.offset, fi.length);
 480
 481		btrfs_finish_ordered_extent(bbio->ordered, folio, start, len,
 482					    !error);
 483		if (error)
 484			mapping_set_error(folio->mapping, error);
 485		btrfs_folio_clear_writeback(fs_info, folio, start, len);
 486	}
 487
 488	bio_put(bio);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 489}
 490
 491static void begin_folio_read(struct btrfs_fs_info *fs_info, struct folio *folio)
 
 492{
 493	ASSERT(folio_test_locked(folio));
 494	if (!btrfs_is_subpage(fs_info, folio->mapping))
 495		return;
 496
 497	ASSERT(folio_test_private(folio));
 498	btrfs_folio_set_lock(fs_info, folio, folio_pos(folio), PAGE_SIZE);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 499}
 500
 501/*
 502 * After a data read IO is done, we need to:
 503 *
 504 * - clear the uptodate bits on error
 505 * - set the uptodate bits if things worked
 506 * - set the folio up to date if all extents in the tree are uptodate
 507 * - clear the lock bit in the extent tree
 508 * - unlock the folio if there are no other extents locked for it
 509 *
 510 * Scheduling is not allowed, so the extent state tree is expected
 511 * to have one and only one object corresponding to this IO.
 512 */
 513static void end_bbio_data_read(struct btrfs_bio *bbio)
 
 514{
 515	struct btrfs_fs_info *fs_info = bbio->fs_info;
 516	struct bio *bio = &bbio->bio;
 517	struct folio_iter fi;
 518	const u32 sectorsize = fs_info->sectorsize;
 519
 520	ASSERT(!bio_flagged(bio, BIO_CLONED));
 521	bio_for_each_folio_all(fi, &bbio->bio) {
 522		bool uptodate = !bio->bi_status;
 523		struct folio *folio = fi.folio;
 524		struct inode *inode = folio->mapping->host;
 525		u64 start;
 526		u64 end;
 527		u32 len;
 528
 529		/* For now only order 0 folios are supported for data. */
 530		ASSERT(folio_order(folio) == 0);
 531		btrfs_debug(fs_info,
 532			"%s: bi_sector=%llu, err=%d, mirror=%u",
 533			__func__, bio->bi_iter.bi_sector, bio->bi_status,
 534			bbio->mirror_num);
 535
 536		/*
 537		 * We always issue full-sector reads, but if some block in a
 538		 * folio fails to read, blk_update_request() will advance
 539		 * bv_offset and adjust bv_len to compensate.  Print a warning
 540		 * for unaligned offsets, and an error if they don't add up to
 541		 * a full sector.
 542		 */
 543		if (!IS_ALIGNED(fi.offset, sectorsize))
 544			btrfs_err(fs_info,
 545		"partial page read in btrfs with offset %zu and length %zu",
 546				  fi.offset, fi.length);
 547		else if (!IS_ALIGNED(fi.offset + fi.length, sectorsize))
 548			btrfs_info(fs_info,
 549		"incomplete page read with offset %zu and length %zu",
 550				   fi.offset, fi.length);
 551
 552		start = folio_pos(folio) + fi.offset;
 553		end = start + fi.length - 1;
 554		len = fi.length;
 555
 556		if (likely(uptodate)) {
 557			loff_t i_size = i_size_read(inode);
 558			pgoff_t end_index = i_size >> folio_shift(folio);
 
 559
 560			/*
 561			 * Zero out the remaining part if this range straddles
 562			 * i_size.
 563			 *
 564			 * Here we should only zero the range inside the folio,
 565			 * not touch anything else.
 566			 *
 567			 * NOTE: i_size is exclusive while end is inclusive.
 568			 */
 569			if (folio_index(folio) == end_index && i_size <= end) {
 570				u32 zero_start = max(offset_in_folio(folio, i_size),
 571						     offset_in_folio(folio, start));
 572				u32 zero_len = offset_in_folio(folio, end) + 1 -
 573					       zero_start;
 574
 575				folio_zero_range(folio, zero_start, zero_len);
 576			}
 
 
 
 
 
 577		}
 578
 579		/* Update page status and unlock. */
 580		end_folio_read(folio, uptodate, start, len);
 
 
 
 
 
 
 
 
 
 
 581	}
 582	bio_put(bio);
 
 583}
 584
 585/*
 586 * Populate every free slot in a provided array with folios using GFP_NOFS.
 587 *
 588 * @nr_folios:   number of folios to allocate
 589 * @folio_array: the array to fill with folios; any existing non-NULL entries in
 590 *		 the array will be skipped
 591 *
 592 * Return: 0        if all folios were able to be allocated;
 593 *         -ENOMEM  otherwise, the partially allocated folios would be freed and
 594 *                  the array slots zeroed
 595 */
 596int btrfs_alloc_folio_array(unsigned int nr_folios, struct folio **folio_array)
 597{
 598	for (int i = 0; i < nr_folios; i++) {
 599		if (folio_array[i])
 600			continue;
 601		folio_array[i] = folio_alloc(GFP_NOFS, 0);
 602		if (!folio_array[i])
 603			goto error;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 604	}
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 605	return 0;
 606error:
 607	for (int i = 0; i < nr_folios; i++) {
 608		if (folio_array[i])
 609			folio_put(folio_array[i]);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 610	}
 611	return -ENOMEM;
 
 612}
 613
 614/*
 615 * Populate every free slot in a provided array with pages, using GFP_NOFS.
 616 *
 617 * @nr_pages:   number of pages to allocate
 618 * @page_array: the array to fill with pages; any existing non-null entries in
 619 *		the array will be skipped
 620 * @nofail:	whether using __GFP_NOFAIL flag
 621 *
 622 * Return: 0        if all pages were able to be allocated;
 623 *         -ENOMEM  otherwise, the partially allocated pages would be freed and
 624 *                  the array slots zeroed
 625 */
 626int btrfs_alloc_page_array(unsigned int nr_pages, struct page **page_array,
 627			   bool nofail)
 628{
 629	const gfp_t gfp = nofail ? (GFP_NOFS | __GFP_NOFAIL) : GFP_NOFS;
 630	unsigned int allocated;
 631
 632	for (allocated = 0; allocated < nr_pages;) {
 633		unsigned int last = allocated;
 634
 635		allocated = alloc_pages_bulk_array(gfp, nr_pages, page_array);
 636		if (unlikely(allocated == last)) {
 637			/* No progress, fail and do cleanup. */
 638			for (int i = 0; i < allocated; i++) {
 639				__free_page(page_array[i]);
 640				page_array[i] = NULL;
 641			}
 642			return -ENOMEM;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 643		}
 644	}
 
 
 
 
 645	return 0;
 646}
 647
 648/*
 649 * Populate needed folios for the extent buffer.
 650 *
 651 * For now, the folios populated are always in order 0 (aka, single page).
 
 652 */
 653static int alloc_eb_folio_array(struct extent_buffer *eb, bool nofail)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 654{
 655	struct page *page_array[INLINE_EXTENT_BUFFER_PAGES] = { 0 };
 656	int num_pages = num_extent_pages(eb);
 
 
 
 657	int ret;
 
 658
 659	ret = btrfs_alloc_page_array(num_pages, page_array, nofail);
 660	if (ret < 0)
 661		return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 662
 663	for (int i = 0; i < num_pages; i++)
 664		eb->folios[i] = page_folio(page_array[i]);
 665	eb->folio_size = PAGE_SIZE;
 666	eb->folio_shift = PAGE_SHIFT;
 667	return 0;
 668}
 669
 670static bool btrfs_bio_is_contig(struct btrfs_bio_ctrl *bio_ctrl,
 671				struct folio *folio, u64 disk_bytenr,
 672				unsigned int pg_offset)
 673{
 674	struct bio *bio = &bio_ctrl->bbio->bio;
 675	struct bio_vec *bvec = bio_last_bvec_all(bio);
 676	const sector_t sector = disk_bytenr >> SECTOR_SHIFT;
 677	struct folio *bv_folio = page_folio(bvec->bv_page);
 678
 679	if (bio_ctrl->compress_type != BTRFS_COMPRESS_NONE) {
 
 
 680		/*
 681		 * For compression, all IO should have its logical bytenr set
 682		 * to the starting bytenr of the compressed extent.
 
 683		 */
 684		return bio->bi_iter.bi_sector == sector;
 
 
 685	}
 686
 687	/*
 688	 * The contig check requires the following conditions to be met:
 689	 *
 690	 * 1) The folios are belonging to the same inode
 691	 *    This is implied by the call chain.
 692	 *
 693	 * 2) The range has adjacent logical bytenr
 694	 *
 695	 * 3) The range has adjacent file offset
 696	 *    This is required for the usage of btrfs_bio->file_offset.
 697	 */
 698	return bio_end_sector(bio) == sector &&
 699		folio_pos(bv_folio) + bvec->bv_offset + bvec->bv_len ==
 700		folio_pos(folio) + pg_offset;
 701}
 702
 703static void alloc_new_bio(struct btrfs_inode *inode,
 704			  struct btrfs_bio_ctrl *bio_ctrl,
 705			  u64 disk_bytenr, u64 file_offset)
 706{
 707	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 708	struct btrfs_bio *bbio;
 709
 710	bbio = btrfs_bio_alloc(BIO_MAX_VECS, bio_ctrl->opf, fs_info,
 711			       bio_ctrl->end_io_func, NULL);
 712	bbio->bio.bi_iter.bi_sector = disk_bytenr >> SECTOR_SHIFT;
 713	bbio->inode = inode;
 714	bbio->file_offset = file_offset;
 715	bio_ctrl->bbio = bbio;
 716	bio_ctrl->len_to_oe_boundary = U32_MAX;
 717
 718	/* Limit data write bios to the ordered boundary. */
 719	if (bio_ctrl->wbc) {
 720		struct btrfs_ordered_extent *ordered;
 721
 722		ordered = btrfs_lookup_ordered_extent(inode, file_offset);
 723		if (ordered) {
 724			bio_ctrl->len_to_oe_boundary = min_t(u32, U32_MAX,
 725					ordered->file_offset +
 726					ordered->disk_num_bytes - file_offset);
 727			bbio->ordered = ordered;
 728		}
 729
 730		/*
 731		 * Pick the last added device to support cgroup writeback.  For
 732		 * multi-device file systems this means blk-cgroup policies have
 733		 * to always be set on the last added/replaced device.
 734		 * This is a bit odd but has been like that for a long time.
 
 
 
 
 
 
 
 
 
 
 
 735		 */
 736		bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
 737		wbc_init_bio(bio_ctrl->wbc, &bbio->bio);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 738	}
 739}
 740
 741/*
 742 * @disk_bytenr: logical bytenr where the write will be
 743 * @page:	page to add to the bio
 744 * @size:	portion of page that we want to write to
 745 * @pg_offset:	offset of the new bio or to check whether we are adding
 746 *              a contiguous page to the previous one
 747 *
 748 * The will either add the page into the existing @bio_ctrl->bbio, or allocate a
 749 * new one in @bio_ctrl->bbio.
 750 * The mirror number for this IO should already be initizlied in
 751 * @bio_ctrl->mirror_num.
 752 */
 753static void submit_extent_folio(struct btrfs_bio_ctrl *bio_ctrl,
 754			       u64 disk_bytenr, struct folio *folio,
 755			       size_t size, unsigned long pg_offset)
 756{
 757	struct btrfs_inode *inode = folio_to_inode(folio);
 758
 759	ASSERT(pg_offset + size <= PAGE_SIZE);
 760	ASSERT(bio_ctrl->end_io_func);
 761
 762	if (bio_ctrl->bbio &&
 763	    !btrfs_bio_is_contig(bio_ctrl, folio, disk_bytenr, pg_offset))
 764		submit_one_bio(bio_ctrl);
 765
 766	do {
 767		u32 len = size;
 768
 769		/* Allocate new bio if needed */
 770		if (!bio_ctrl->bbio) {
 771			alloc_new_bio(inode, bio_ctrl, disk_bytenr,
 772				      folio_pos(folio) + pg_offset);
 
 
 
 
 
 
 
 
 
 
 
 773		}
 774
 775		/* Cap to the current ordered extent boundary if there is one. */
 776		if (len > bio_ctrl->len_to_oe_boundary) {
 777			ASSERT(bio_ctrl->compress_type == BTRFS_COMPRESS_NONE);
 778			ASSERT(is_data_inode(inode));
 779			len = bio_ctrl->len_to_oe_boundary;
 780		}
 781
 782		if (!bio_add_folio(&bio_ctrl->bbio->bio, folio, len, pg_offset)) {
 783			/* bio full: move on to a new one */
 784			submit_one_bio(bio_ctrl);
 785			continue;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 786		}
 
 
 
 
 
 787
 788		if (bio_ctrl->wbc)
 789			wbc_account_cgroup_owner(bio_ctrl->wbc, folio,
 790						 len);
 791
 792		size -= len;
 793		pg_offset += len;
 794		disk_bytenr += len;
 
 
 
 
 795
 796		/*
 797		 * len_to_oe_boundary defaults to U32_MAX, which isn't folio or
 798		 * sector aligned.  alloc_new_bio() then sets it to the end of
 799		 * our ordered extent for writes into zoned devices.
 800		 *
 801		 * When len_to_oe_boundary is tracking an ordered extent, we
 802		 * trust the ordered extent code to align things properly, and
 803		 * the check above to cap our write to the ordered extent
 804		 * boundary is correct.
 805		 *
 806		 * When len_to_oe_boundary is U32_MAX, the cap above would
 807		 * result in a 4095 byte IO for the last folio right before
 808		 * we hit the bio limit of UINT_MAX.  bio_add_folio() has all
 809		 * the checks required to make sure we don't overflow the bio,
 810		 * and we should just ignore len_to_oe_boundary completely
 811		 * unless we're using it to track an ordered extent.
 812		 *
 813		 * It's pretty hard to make a bio sized U32_MAX, but it can
 814		 * happen when the page cache is able to feed us contiguous
 815		 * folios for large extents.
 816		 */
 817		if (bio_ctrl->len_to_oe_boundary != U32_MAX)
 818			bio_ctrl->len_to_oe_boundary -= len;
 819
 820		/* Ordered extent boundary: move on to a new bio. */
 821		if (bio_ctrl->len_to_oe_boundary == 0)
 822			submit_one_bio(bio_ctrl);
 823	} while (size);
 
 
 824}
 825
 826static int attach_extent_buffer_folio(struct extent_buffer *eb,
 827				      struct folio *folio,
 828				      struct btrfs_subpage *prealloc)
 
 
 
 
 829{
 830	struct btrfs_fs_info *fs_info = eb->fs_info;
 831	int ret = 0;
 832
 833	/*
 834	 * If the page is mapped to btree inode, we should hold the private
 835	 * lock to prevent race.
 836	 * For cloned or dummy extent buffers, their pages are not mapped and
 837	 * will not race with any other ebs.
 838	 */
 839	if (folio->mapping)
 840		lockdep_assert_held(&folio->mapping->i_private_lock);
 841
 842	if (fs_info->nodesize >= PAGE_SIZE) {
 843		if (!folio_test_private(folio))
 844			folio_attach_private(folio, eb);
 845		else
 846			WARN_ON(folio_get_private(folio) != eb);
 847		return 0;
 848	}
 849
 850	/* Already mapped, just free prealloc */
 851	if (folio_test_private(folio)) {
 852		btrfs_free_subpage(prealloc);
 853		return 0;
 
 
 
 854	}
 
 
 855
 856	if (prealloc)
 857		/* Has preallocated memory for subpage */
 858		folio_attach_private(folio, prealloc);
 859	else
 860		/* Do new allocation to attach subpage */
 861		ret = btrfs_attach_subpage(fs_info, folio, BTRFS_SUBPAGE_METADATA);
 862	return ret;
 
 
 
 
 
 
 
 
 
 
 
 
 863}
 864
 865int set_page_extent_mapped(struct page *page)
 
 866{
 867	return set_folio_extent_mapped(page_folio(page));
 
 
 
 
 
 
 
 
 
 
 868}
 869
 870int set_folio_extent_mapped(struct folio *folio)
 
 
 871{
 872	struct btrfs_fs_info *fs_info;
 
 
 
 
 873
 874	ASSERT(folio->mapping);
 875
 876	if (folio_test_private(folio))
 877		return 0;
 878
 879	fs_info = folio_to_fs_info(folio);
 880
 881	if (btrfs_is_subpage(fs_info, folio->mapping))
 882		return btrfs_attach_subpage(fs_info, folio, BTRFS_SUBPAGE_DATA);
 
 
 
 883
 884	folio_attach_private(folio, (void *)EXTENT_FOLIO_PRIVATE);
 885	return 0;
 886}
 887
 888void clear_folio_extent_mapped(struct folio *folio)
 
 
 889{
 890	struct btrfs_fs_info *fs_info;
 
 
 
 
 
 891
 892	ASSERT(folio->mapping);
 893
 894	if (!folio_test_private(folio))
 895		return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 896
 897	fs_info = folio_to_fs_info(folio);
 898	if (btrfs_is_subpage(fs_info, folio->mapping))
 899		return btrfs_detach_subpage(fs_info, folio);
 
 900
 901	folio_detach_private(folio);
 902}
 903
 904static struct extent_map *get_extent_map(struct btrfs_inode *inode,
 905					 struct folio *folio, u64 start,
 906					 u64 len, struct extent_map **em_cached)
 907{
 908	struct extent_map *em;
 
 
 
 
 
 
 
 909
 910	ASSERT(em_cached);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 911
 912	if (*em_cached) {
 913		em = *em_cached;
 914		if (extent_map_in_tree(em) && start >= em->start &&
 915		    start < extent_map_end(em)) {
 916			refcount_inc(&em->refs);
 917			return em;
 918		}
 919
 920		free_extent_map(em);
 921		*em_cached = NULL;
 922	}
 923
 924	em = btrfs_get_extent(inode, folio, start, len);
 925	if (!IS_ERR(em)) {
 926		BUG_ON(*em_cached);
 927		refcount_inc(&em->refs);
 928		*em_cached = em;
 929	}
 930
 931	return em;
 932}
 933/*
 934 * basic readpage implementation.  Locked extent state structs are inserted
 935 * into the tree that are removed when the IO is done (by the end_io
 936 * handlers)
 937 * XXX JDM: This needs looking at to ensure proper page locking
 938 * return 0 on success, otherwise return error
 939 */
 940static int btrfs_do_readpage(struct folio *folio, struct extent_map **em_cached,
 941		      struct btrfs_bio_ctrl *bio_ctrl, u64 *prev_em_start)
 942{
 943	struct inode *inode = folio->mapping->host;
 944	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
 945	u64 start = folio_pos(folio);
 946	const u64 end = start + PAGE_SIZE - 1;
 
 
 
 
 
 947	u64 cur = start;
 948	u64 extent_offset;
 949	u64 last_byte = i_size_read(inode);
 950	u64 block_start;
 
 
 951	struct extent_map *em;
 952	int ret = 0;
 
 
 953	size_t pg_offset = 0;
 954	size_t iosize;
 955	size_t blocksize = fs_info->sectorsize;
 956
 957	ret = set_folio_extent_mapped(folio);
 958	if (ret < 0) {
 959		folio_unlock(folio);
 960		return ret;
 
 
 
 
 
 
 
 961	}
 962
 963	if (folio->index == last_byte >> folio_shift(folio)) {
 964		size_t zero_offset = offset_in_folio(folio, last_byte);
 
 965
 966		if (zero_offset) {
 967			iosize = folio_size(folio) - zero_offset;
 968			folio_zero_range(folio, zero_offset, iosize);
 
 
 
 969		}
 970	}
 971	bio_ctrl->end_io_func = end_bbio_data_read;
 972	begin_folio_read(fs_info, folio);
 973	while (cur <= end) {
 974		enum btrfs_compression_type compress_type = BTRFS_COMPRESS_NONE;
 975		bool force_bio_submit = false;
 976		u64 disk_bytenr;
 977
 978		ASSERT(IS_ALIGNED(cur, fs_info->sectorsize));
 979		if (cur >= last_byte) {
 980			iosize = folio_size(folio) - pg_offset;
 981			folio_zero_range(folio, pg_offset, iosize);
 982			end_folio_read(folio, true, cur, iosize);
 
 
 
 
 
 
 
 
 
 
 983			break;
 984		}
 985		em = get_extent_map(BTRFS_I(inode), folio, cur, end - cur + 1, em_cached);
 986		if (IS_ERR(em)) {
 987			end_folio_read(folio, false, cur, end + 1 - cur);
 988			return PTR_ERR(em);
 
 
 989		}
 990		extent_offset = cur - em->start;
 991		BUG_ON(extent_map_end(em) <= cur);
 992		BUG_ON(end < cur);
 993
 994		compress_type = extent_map_compression(em);
 
 
 
 
 995
 996		iosize = min(extent_map_end(em) - cur, end - cur + 1);
 
 997		iosize = ALIGN(iosize, blocksize);
 998		if (compress_type != BTRFS_COMPRESS_NONE)
 999			disk_bytenr = em->disk_bytenr;
1000		else
1001			disk_bytenr = extent_map_block_start(em) + extent_offset;
1002		block_start = extent_map_block_start(em);
1003		if (em->flags & EXTENT_FLAG_PREALLOC)
 
 
 
 
1004			block_start = EXTENT_MAP_HOLE;
1005
1006		/*
1007		 * If we have a file range that points to a compressed extent
1008		 * and it's followed by a consecutive file range that points
1009		 * to the same compressed extent (possibly with a different
1010		 * offset and/or length, so it either points to the whole extent
1011		 * or only part of it), we must make sure we do not submit a
1012		 * single bio to populate the folios for the 2 ranges because
1013		 * this makes the compressed extent read zero out the folios
1014		 * belonging to the 2nd range. Imagine the following scenario:
1015		 *
1016		 *  File layout
1017		 *  [0 - 8K]                     [8K - 24K]
1018		 *    |                               |
1019		 *    |                               |
1020		 * points to extent X,         points to extent X,
1021		 * offset 4K, length of 8K     offset 0, length 16K
1022		 *
1023		 * [extent X, compressed length = 4K uncompressed length = 16K]
1024		 *
1025		 * If the bio to read the compressed extent covers both ranges,
1026		 * it will decompress extent X into the folios belonging to the
1027		 * first range and then it will stop, zeroing out the remaining
1028		 * folios that belong to the other range that points to extent X.
1029		 * So here we make sure we submit 2 bios, one for the first
1030		 * range and another one for the third range. Both will target
1031		 * the same physical extent from disk, but we can't currently
1032		 * make the compressed bio endio callback populate the folios
1033		 * for both ranges because each compressed bio is tightly
1034		 * coupled with a single extent map, and each range can have
1035		 * an extent map with a different offset value relative to the
1036		 * uncompressed data of our extent and different lengths. This
1037		 * is a corner case so we prioritize correctness over
1038		 * non-optimal behavior (submitting 2 bios for the same extent).
1039		 */
1040		if (compress_type != BTRFS_COMPRESS_NONE &&
1041		    prev_em_start && *prev_em_start != (u64)-1 &&
1042		    *prev_em_start != em->start)
1043			force_bio_submit = true;
1044
1045		if (prev_em_start)
1046			*prev_em_start = em->start;
1047
1048		free_extent_map(em);
1049		em = NULL;
1050
1051		/* we've found a hole, just zero and go on */
1052		if (block_start == EXTENT_MAP_HOLE) {
1053			folio_zero_range(folio, pg_offset, iosize);
 
1054
1055			end_folio_read(folio, true, cur, iosize);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1056			cur = cur + iosize;
1057			pg_offset += iosize;
1058			continue;
1059		}
1060		/* the get_extent function already copied into the folio */
 
 
1061		if (block_start == EXTENT_MAP_INLINE) {
1062			end_folio_read(folio, true, cur, iosize);
 
1063			cur = cur + iosize;
1064			pg_offset += iosize;
1065			continue;
1066		}
1067
1068		if (bio_ctrl->compress_type != compress_type) {
1069			submit_one_bio(bio_ctrl);
1070			bio_ctrl->compress_type = compress_type;
 
 
 
 
 
 
 
 
 
 
 
1071		}
1072
1073		if (force_bio_submit)
1074			submit_one_bio(bio_ctrl);
1075		submit_extent_folio(bio_ctrl, disk_bytenr, folio, iosize,
1076				    pg_offset);
1077		cur = cur + iosize;
1078		pg_offset += iosize;
1079	}
1080
 
 
 
 
 
1081	return 0;
1082}
1083
1084int btrfs_read_folio(struct file *file, struct folio *folio)
1085{
1086	struct btrfs_inode *inode = folio_to_inode(folio);
1087	const u64 start = folio_pos(folio);
1088	const u64 end = start + folio_size(folio) - 1;
1089	struct extent_state *cached_state = NULL;
1090	struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ };
1091	struct extent_map *em_cached = NULL;
1092	int ret;
 
 
 
1093
1094	btrfs_lock_and_flush_ordered_range(inode, start, end, &cached_state);
1095	ret = btrfs_do_readpage(folio, &em_cached, &bio_ctrl, NULL);
1096	unlock_extent(&inode->io_tree, start, end, &cached_state);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1097
1098	free_extent_map(em_cached);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1099
1100	/*
1101	 * If btrfs_do_readpage() failed we will want to submit the assembled
1102	 * bio to do the cleanup.
1103	 */
1104	submit_one_bio(&bio_ctrl);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1105	return ret;
1106}
1107
1108static void set_delalloc_bitmap(struct folio *folio, unsigned long *delalloc_bitmap,
1109				u64 start, u32 len)
1110{
1111	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
1112	const u64 folio_start = folio_pos(folio);
1113	unsigned int start_bit;
1114	unsigned int nbits;
1115
1116	ASSERT(start >= folio_start && start + len <= folio_start + PAGE_SIZE);
1117	start_bit = (start - folio_start) >> fs_info->sectorsize_bits;
1118	nbits = len >> fs_info->sectorsize_bits;
1119	ASSERT(bitmap_test_range_all_zero(delalloc_bitmap, start_bit, nbits));
1120	bitmap_set(delalloc_bitmap, start_bit, nbits);
1121}
1122
1123static bool find_next_delalloc_bitmap(struct folio *folio,
1124				      unsigned long *delalloc_bitmap, u64 start,
1125				      u64 *found_start, u32 *found_len)
1126{
1127	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
1128	const u64 folio_start = folio_pos(folio);
1129	const unsigned int bitmap_size = fs_info->sectors_per_page;
1130	unsigned int start_bit;
1131	unsigned int first_zero;
1132	unsigned int first_set;
1133
1134	ASSERT(start >= folio_start && start < folio_start + PAGE_SIZE);
1135
1136	start_bit = (start - folio_start) >> fs_info->sectorsize_bits;
1137	first_set = find_next_bit(delalloc_bitmap, bitmap_size, start_bit);
1138	if (first_set >= bitmap_size)
1139		return false;
1140
1141	*found_start = folio_start + (first_set << fs_info->sectorsize_bits);
1142	first_zero = find_next_zero_bit(delalloc_bitmap, bitmap_size, first_set);
1143	*found_len = (first_zero - first_set) << fs_info->sectorsize_bits;
1144	return true;
1145}
1146
1147/*
1148 * Do all of the delayed allocation setup.
1149 *
1150 * Return >0 if all the dirty blocks are submitted async (compression) or inlined.
1151 * The @folio should no longer be touched (treat it as already unlocked).
 
1152 *
1153 * Return 0 if there is still dirty block that needs to be submitted through
1154 * extent_writepage_io().
1155 * bio_ctrl->submit_bitmap will indicate which blocks of the folio should be
1156 * submitted, and @folio is still kept locked.
1157 *
1158 * Return <0 if there is any error hit.
1159 * Any allocated ordered extent range covering this folio will be marked
1160 * finished (IOERR), and @folio is still kept locked.
1161 */
1162static noinline_for_stack int writepage_delalloc(struct btrfs_inode *inode,
1163						 struct folio *folio,
1164						 struct btrfs_bio_ctrl *bio_ctrl)
1165{
1166	struct btrfs_fs_info *fs_info = inode_to_fs_info(&inode->vfs_inode);
1167	struct writeback_control *wbc = bio_ctrl->wbc;
1168	const bool is_subpage = btrfs_is_subpage(fs_info, folio->mapping);
1169	const u64 page_start = folio_pos(folio);
1170	const u64 page_end = page_start + folio_size(folio) - 1;
1171	unsigned long delalloc_bitmap = 0;
1172	/*
1173	 * Save the last found delalloc end. As the delalloc end can go beyond
1174	 * page boundary, thus we cannot rely on subpage bitmap to locate the
1175	 * last delalloc end.
1176	 */
1177	u64 last_delalloc_end = 0;
1178	/*
1179	 * The range end (exclusive) of the last successfully finished delalloc
1180	 * range.
1181	 * Any range covered by ordered extent must either be manually marked
1182	 * finished (error handling), or has IO submitted (and finish the
1183	 * ordered extent normally).
1184	 *
1185	 * This records the end of ordered extent cleanup if we hit an error.
1186	 */
1187	u64 last_finished_delalloc_end = page_start;
1188	u64 delalloc_start = page_start;
1189	u64 delalloc_end = page_end;
1190	u64 delalloc_to_write = 0;
1191	int ret = 0;
1192	int bit;
1193
1194	/* Save the dirty bitmap as our submission bitmap will be a subset of it. */
1195	if (btrfs_is_subpage(fs_info, inode->vfs_inode.i_mapping)) {
1196		ASSERT(fs_info->sectors_per_page > 1);
1197		btrfs_get_subpage_dirty_bitmap(fs_info, folio, &bio_ctrl->submit_bitmap);
1198	} else {
1199		bio_ctrl->submit_bitmap = 1;
1200	}
1201
1202	for_each_set_bit(bit, &bio_ctrl->submit_bitmap, fs_info->sectors_per_page) {
1203		u64 start = page_start + (bit << fs_info->sectorsize_bits);
1204
1205		btrfs_folio_set_lock(fs_info, folio, start, fs_info->sectorsize);
1206	}
1207
1208	/* Lock all (subpage) delalloc ranges inside the folio first. */
1209	while (delalloc_start < page_end) {
1210		delalloc_end = page_end;
1211		if (!find_lock_delalloc_range(&inode->vfs_inode, folio,
1212					      &delalloc_start, &delalloc_end)) {
 
 
1213			delalloc_start = delalloc_end + 1;
1214			continue;
1215		}
1216		set_delalloc_bitmap(folio, &delalloc_bitmap, delalloc_start,
1217				    min(delalloc_end, page_end) + 1 - delalloc_start);
1218		last_delalloc_end = delalloc_end;
1219		delalloc_start = delalloc_end + 1;
1220	}
1221	delalloc_start = page_start;
1222
1223	if (!last_delalloc_end)
1224		goto out;
1225
1226	/* Run the delalloc ranges for the above locked ranges. */
1227	while (delalloc_start < page_end) {
1228		u64 found_start;
1229		u32 found_len;
1230		bool found;
1231
1232		if (!is_subpage) {
1233			/*
1234			 * For non-subpage case, the found delalloc range must
1235			 * cover this folio and there must be only one locked
1236			 * delalloc range.
1237			 */
1238			found_start = page_start;
1239			found_len = last_delalloc_end + 1 - found_start;
1240			found = true;
1241		} else {
1242			found = find_next_delalloc_bitmap(folio, &delalloc_bitmap,
1243					delalloc_start, &found_start, &found_len);
1244		}
1245		if (!found)
1246			break;
1247		/*
1248		 * The subpage range covers the last sector, the delalloc range may
1249		 * end beyond the folio boundary, use the saved delalloc_end
1250		 * instead.
1251		 */
1252		if (found_start + found_len >= page_end)
1253			found_len = last_delalloc_end + 1 - found_start;
1254
1255		if (ret >= 0) {
1256			/*
1257			 * Some delalloc range may be created by previous folios.
1258			 * Thus we still need to clean up this range during error
1259			 * handling.
1260			 */
1261			last_finished_delalloc_end = found_start;
1262			/* No errors hit so far, run the current delalloc range. */
1263			ret = btrfs_run_delalloc_range(inode, folio,
1264						       found_start,
1265						       found_start + found_len - 1,
1266						       wbc);
1267			if (ret >= 0)
1268				last_finished_delalloc_end = found_start + found_len;
1269		} else {
1270			/*
1271			 * We've hit an error during previous delalloc range,
1272			 * have to cleanup the remaining locked ranges.
1273			 */
1274			unlock_extent(&inode->io_tree, found_start,
1275				      found_start + found_len - 1, NULL);
1276			__unlock_for_delalloc(&inode->vfs_inode, folio,
1277					      found_start,
1278					      found_start + found_len - 1);
1279		}
1280
1281		/*
1282		 * We have some ranges that's going to be submitted asynchronously
1283		 * (compression or inline).  These range have their own control
1284		 * on when to unlock the pages.  We should not touch them
1285		 * anymore, so clear the range from the submission bitmap.
1286		 */
1287		if (ret > 0) {
1288			unsigned int start_bit = (found_start - page_start) >>
1289						 fs_info->sectorsize_bits;
1290			unsigned int end_bit = (min(page_end + 1, found_start + found_len) -
1291						page_start) >> fs_info->sectorsize_bits;
1292			bitmap_clear(&bio_ctrl->submit_bitmap, start_bit, end_bit - start_bit);
1293		}
1294		/*
1295		 * Above btrfs_run_delalloc_range() may have unlocked the folio,
1296		 * thus for the last range, we cannot touch the folio anymore.
1297		 */
1298		if (found_start + found_len >= last_delalloc_end + 1)
1299			break;
1300
1301		delalloc_start = found_start + found_len;
1302	}
1303	/*
1304	 * It's possible we had some ordered extents created before we hit
1305	 * an error, cleanup non-async successfully created delalloc ranges.
1306	 */
1307	if (unlikely(ret < 0)) {
1308		unsigned int bitmap_size = min(
1309				(last_finished_delalloc_end - page_start) >>
1310				fs_info->sectorsize_bits,
1311				fs_info->sectors_per_page);
1312
1313		for_each_set_bit(bit, &bio_ctrl->submit_bitmap, bitmap_size)
1314			btrfs_mark_ordered_io_finished(inode, folio,
1315				page_start + (bit << fs_info->sectorsize_bits),
1316				fs_info->sectorsize, false);
1317		return ret;
1318	}
1319out:
1320	if (last_delalloc_end)
1321		delalloc_end = last_delalloc_end;
1322	else
1323		delalloc_end = page_end;
1324	/*
1325	 * delalloc_end is already one less than the total length, so
1326	 * we don't subtract one from PAGE_SIZE
1327	 */
1328	delalloc_to_write +=
1329		DIV_ROUND_UP(delalloc_end + 1 - page_start, PAGE_SIZE);
1330
1331	/*
1332	 * If all ranges are submitted asynchronously, we just need to account
1333	 * for them here.
1334	 */
1335	if (bitmap_empty(&bio_ctrl->submit_bitmap, fs_info->sectors_per_page)) {
1336		wbc->nr_to_write -= delalloc_to_write;
1337		return 1;
1338	}
1339
1340	if (wbc->nr_to_write < delalloc_to_write) {
1341		int thresh = 8192;
1342
1343		if (delalloc_to_write < thresh * 2)
1344			thresh = delalloc_to_write;
1345		wbc->nr_to_write = min_t(u64, delalloc_to_write,
1346					 thresh);
1347	}
1348
1349	return 0;
1350}
1351
1352/*
1353 * Return 0 if we have submitted or queued the sector for submission.
1354 * Return <0 for critical errors.
1355 *
1356 * Caller should make sure filepos < i_size and handle filepos >= i_size case.
1357 */
1358static int submit_one_sector(struct btrfs_inode *inode,
1359			     struct folio *folio,
1360			     u64 filepos, struct btrfs_bio_ctrl *bio_ctrl,
1361			     loff_t i_size)
1362{
1363	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1364	struct extent_map *em;
1365	u64 block_start;
1366	u64 disk_bytenr;
1367	u64 extent_offset;
1368	u64 em_end;
1369	const u32 sectorsize = fs_info->sectorsize;
1370
1371	ASSERT(IS_ALIGNED(filepos, sectorsize));
1372
1373	/* @filepos >= i_size case should be handled by the caller. */
1374	ASSERT(filepos < i_size);
1375
1376	em = btrfs_get_extent(inode, NULL, filepos, sectorsize);
1377	if (IS_ERR(em))
1378		return PTR_ERR_OR_ZERO(em);
1379
1380	extent_offset = filepos - em->start;
1381	em_end = extent_map_end(em);
1382	ASSERT(filepos <= em_end);
1383	ASSERT(IS_ALIGNED(em->start, sectorsize));
1384	ASSERT(IS_ALIGNED(em->len, sectorsize));
1385
1386	block_start = extent_map_block_start(em);
1387	disk_bytenr = extent_map_block_start(em) + extent_offset;
1388
1389	ASSERT(!extent_map_is_compressed(em));
1390	ASSERT(block_start != EXTENT_MAP_HOLE);
1391	ASSERT(block_start != EXTENT_MAP_INLINE);
1392
1393	free_extent_map(em);
1394	em = NULL;
1395
1396	/*
1397	 * Although the PageDirty bit is cleared before entering this
1398	 * function, subpage dirty bit is not cleared.
1399	 * So clear subpage dirty bit here so next time we won't submit
1400	 * a folio for a range already written to disk.
1401	 */
1402	btrfs_folio_clear_dirty(fs_info, folio, filepos, sectorsize);
1403	btrfs_folio_set_writeback(fs_info, folio, filepos, sectorsize);
1404	/*
1405	 * Above call should set the whole folio with writeback flag, even
1406	 * just for a single subpage sector.
1407	 * As long as the folio is properly locked and the range is correct,
1408	 * we should always get the folio with writeback flag.
1409	 */
1410	ASSERT(folio_test_writeback(folio));
 
 
 
 
 
 
 
 
1411
1412	submit_extent_folio(bio_ctrl, disk_bytenr, folio,
1413			    sectorsize, filepos - folio_pos(folio));
1414	return 0;
 
1415}
1416
1417/*
1418 * Helper for extent_writepage().  This calls the writepage start hooks,
1419 * and does the loop to map the page into extents and bios.
1420 *
1421 * We return 1 if the IO is started and the page is unlocked,
1422 * 0 if all went well (page still locked)
1423 * < 0 if there were errors (page still locked)
1424 */
1425static noinline_for_stack int extent_writepage_io(struct btrfs_inode *inode,
1426						  struct folio *folio,
1427						  u64 start, u32 len,
1428						  struct btrfs_bio_ctrl *bio_ctrl,
1429						  loff_t i_size)
1430{
1431	struct btrfs_fs_info *fs_info = inode->root->fs_info;
1432	unsigned long range_bitmap = 0;
1433	bool submitted_io = false;
1434	bool error = false;
1435	const u64 folio_start = folio_pos(folio);
1436	u64 cur;
1437	int bit;
 
 
 
 
 
 
 
 
 
1438	int ret = 0;
 
 
1439
1440	ASSERT(start >= folio_start &&
1441	       start + len <= folio_start + folio_size(folio));
 
 
 
 
 
 
 
1442
1443	ret = btrfs_writepage_cow_fixup(folio);
1444	if (ret) {
1445		/* Fixup worker will requeue */
1446		folio_redirty_for_writepage(bio_ctrl->wbc, folio);
1447		folio_unlock(folio);
1448		return 1;
1449	}
1450
1451	for (cur = start; cur < start + len; cur += fs_info->sectorsize)
1452		set_bit((cur - folio_start) >> fs_info->sectorsize_bits, &range_bitmap);
1453	bitmap_and(&bio_ctrl->submit_bitmap, &bio_ctrl->submit_bitmap, &range_bitmap,
1454		   fs_info->sectors_per_page);
 
1455
1456	bio_ctrl->end_io_func = end_bbio_data_write;
 
 
 
 
 
 
1457
1458	for_each_set_bit(bit, &bio_ctrl->submit_bitmap, fs_info->sectors_per_page) {
1459		cur = folio_pos(folio) + (bit << fs_info->sectorsize_bits);
1460
 
 
1461		if (cur >= i_size) {
1462			btrfs_mark_ordered_io_finished(inode, folio, cur,
1463						       start + len - cur, true);
1464			/*
1465			 * This range is beyond i_size, thus we don't need to
1466			 * bother writing back.
1467			 * But we still need to clear the dirty subpage bit, or
1468			 * the next time the folio gets dirtied, we will try to
1469			 * writeback the sectors with subpage dirty bits,
1470			 * causing writeback without ordered extent.
1471			 */
1472			btrfs_folio_clear_dirty(fs_info, folio, cur,
1473						start + len - cur);
1474			break;
1475		}
1476		ret = submit_one_sector(inode, folio, cur, bio_ctrl, i_size);
1477		if (unlikely(ret < 0)) {
1478			/*
1479			 * bio_ctrl may contain a bio crossing several folios.
1480			 * Submit it immediately so that the bio has a chance
1481			 * to finish normally, other than marked as error.
1482			 */
1483			submit_one_bio(bio_ctrl);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1484			/*
1485			 * Failed to grab the extent map which should be very rare.
1486			 * Since there is no bio submitted to finish the ordered
1487			 * extent, we have to manually finish this sector.
1488			 */
1489			btrfs_mark_ordered_io_finished(inode, folio, cur,
1490						       fs_info->sectorsize, false);
1491			error = true;
 
 
 
 
 
 
 
 
 
 
 
 
1492			continue;
1493		}
1494		submitted_io = true;
1495	}
1496
1497	/*
1498	 * If we didn't submitted any sector (>= i_size), folio dirty get
1499	 * cleared but PAGECACHE_TAG_DIRTY is not cleared (only cleared
1500	 * by folio_start_writeback() if the folio is not dirty).
1501	 *
1502	 * Here we set writeback and clear for the range. If the full folio
1503	 * is no longer dirty then we clear the PAGECACHE_TAG_DIRTY tag.
1504	 *
1505	 * If we hit any error, the corresponding sector will still be dirty
1506	 * thus no need to clear PAGECACHE_TAG_DIRTY.
1507	 */
1508	if (!submitted_io && !error) {
1509		btrfs_folio_set_writeback(fs_info, folio, start, len);
1510		btrfs_folio_clear_writeback(fs_info, folio, start, len);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1511	}
 
 
 
 
 
 
 
1512	return ret;
1513}
1514
1515/*
1516 * the writepage semantics are similar to regular writepage.  extent
1517 * records are inserted to lock ranges in the tree, and as dirty areas
1518 * are found, they are marked writeback.  Then the lock bits are removed
1519 * and the end_io handler clears the writeback ranges
1520 *
1521 * Return 0 if everything goes well.
1522 * Return <0 for error.
1523 */
1524static int extent_writepage(struct folio *folio, struct btrfs_bio_ctrl *bio_ctrl)
 
1525{
1526	struct btrfs_inode *inode = BTRFS_I(folio->mapping->host);
1527	struct btrfs_fs_info *fs_info = inode->root->fs_info;
 
 
1528	int ret;
1529	size_t pg_offset;
1530	loff_t i_size = i_size_read(&inode->vfs_inode);
 
1531	unsigned long end_index = i_size >> PAGE_SHIFT;
 
 
1532
1533	trace_extent_writepage(folio, &inode->vfs_inode, bio_ctrl->wbc);
 
 
 
1534
1535	WARN_ON(!folio_test_locked(folio));
1536
1537	pg_offset = offset_in_folio(folio, i_size);
1538	if (folio->index > end_index ||
1539	   (folio->index == end_index && !pg_offset)) {
1540		folio_invalidate(folio, 0, folio_size(folio));
1541		folio_unlock(folio);
 
 
 
 
1542		return 0;
1543	}
1544
1545	if (folio->index == end_index)
1546		folio_zero_range(folio, pg_offset, folio_size(folio) - pg_offset);
1547
1548	/*
1549	 * Default to unlock the whole folio.
1550	 * The proper bitmap can only be initialized until writepage_delalloc().
1551	 */
1552	bio_ctrl->submit_bitmap = (unsigned long)-1;
1553	ret = set_folio_extent_mapped(folio);
1554	if (ret < 0)
1555		goto done;
1556
1557	ret = writepage_delalloc(inode, folio, bio_ctrl);
 
 
1558	if (ret == 1)
1559		return 0;
1560	if (ret)
1561		goto done;
1562
1563	ret = extent_writepage_io(inode, folio, folio_pos(folio),
1564				  PAGE_SIZE, bio_ctrl, i_size);
1565	if (ret == 1)
1566		return 0;
1567
1568	bio_ctrl->wbc->nr_to_write--;
1569
1570done:
1571	if (ret < 0)
1572		mapping_set_error(folio->mapping, ret);
1573	/*
1574	 * Only unlock ranges that are submitted. As there can be some async
1575	 * submitted ranges inside the folio.
1576	 */
1577	btrfs_folio_end_lock_bitmap(fs_info, folio, bio_ctrl->submit_bitmap);
1578	ASSERT(ret <= 0);
 
 
1579	return ret;
 
 
 
1580}
1581
1582void wait_on_extent_buffer_writeback(struct extent_buffer *eb)
1583{
1584	wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_WRITEBACK,
1585		       TASK_UNINTERRUPTIBLE);
1586}
1587
1588/*
1589 * Lock extent buffer status and pages for writeback.
1590 *
1591 * Return %false if the extent buffer doesn't need to be submitted (e.g. the
1592 * extent buffer is not dirty)
1593 * Return %true is the extent buffer is submitted to bio.
1594 */
1595static noinline_for_stack bool lock_extent_buffer_for_io(struct extent_buffer *eb,
1596			  struct writeback_control *wbc)
1597{
1598	struct btrfs_fs_info *fs_info = eb->fs_info;
1599	bool ret = false;
 
1600
1601	btrfs_tree_lock(eb);
1602	while (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags)) {
1603		btrfs_tree_unlock(eb);
1604		if (wbc->sync_mode != WB_SYNC_ALL)
1605			return false;
1606		wait_on_extent_buffer_writeback(eb);
1607		btrfs_tree_lock(eb);
1608	}
1609
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1610	/*
1611	 * We need to do this to prevent races in people who check if the eb is
1612	 * under IO since we can end up having no IO bits set for a short period
1613	 * of time.
1614	 */
1615	spin_lock(&eb->refs_lock);
1616	if (test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
1617		set_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
1618		spin_unlock(&eb->refs_lock);
1619		btrfs_set_header_flag(eb, BTRFS_HEADER_FLAG_WRITTEN);
1620		percpu_counter_add_batch(&fs_info->dirty_metadata_bytes,
1621					 -eb->len,
1622					 fs_info->dirty_metadata_batch);
1623		ret = true;
1624	} else {
1625		spin_unlock(&eb->refs_lock);
1626	}
 
1627	btrfs_tree_unlock(eb);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1628	return ret;
1629}
1630
1631static void set_btree_ioerr(struct extent_buffer *eb)
1632{
1633	struct btrfs_fs_info *fs_info = eb->fs_info;
1634
1635	set_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
 
1636
1637	/*
1638	 * A read may stumble upon this buffer later, make sure that it gets an
1639	 * error and knows there was an error.
1640	 */
1641	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
1642
1643	/*
1644	 * We need to set the mapping with the io error as well because a write
1645	 * error will flip the file system readonly, and then syncfs() will
1646	 * return a 0 because we are readonly if we don't modify the err seq for
1647	 * the superblock.
1648	 */
1649	mapping_set_error(eb->fs_info->btree_inode->i_mapping, -EIO);
1650
1651	/*
1652	 * If writeback for a btree extent that doesn't belong to a log tree
1653	 * failed, increment the counter transaction->eb_write_errors.
1654	 * We do this because while the transaction is running and before it's
1655	 * committing (when we call filemap_fdata[write|wait]_range against
1656	 * the btree inode), we might have
1657	 * btree_inode->i_mapping->a_ops->writepages() called by the VM - if it
1658	 * returns an error or an error happens during writeback, when we're
1659	 * committing the transaction we wouldn't know about it, since the pages
1660	 * can be no longer dirty nor marked anymore for writeback (if a
1661	 * subsequent modification to the extent buffer didn't happen before the
1662	 * transaction commit), which makes filemap_fdata[write|wait]_range not
1663	 * able to find the pages which contain errors at transaction
1664	 * commit time. So if this happens we must abort the transaction,
1665	 * otherwise we commit a super block with btree roots that point to
1666	 * btree nodes/leafs whose content on disk is invalid - either garbage
1667	 * or the content of some node/leaf from a past generation that got
1668	 * cowed or deleted and is no longer valid.
1669	 *
1670	 * Note: setting AS_EIO/AS_ENOSPC in the btree inode's i_mapping would
1671	 * not be enough - we need to distinguish between log tree extents vs
1672	 * non-log tree extents, and the next filemap_fdatawait_range() call
1673	 * will catch and clear such errors in the mapping - and that call might
1674	 * be from a log sync and not from a transaction commit. Also, checking
1675	 * for the eb flag EXTENT_BUFFER_WRITE_ERR at transaction commit time is
1676	 * not done and would not be reliable - the eb might have been released
1677	 * from memory and reading it back again means that flag would not be
1678	 * set (since it's a runtime flag, not persisted on disk).
1679	 *
1680	 * Using the flags below in the btree inode also makes us achieve the
1681	 * goal of AS_EIO/AS_ENOSPC when writepages() returns success, started
1682	 * writeback for all dirty pages and before filemap_fdatawait_range()
1683	 * is called, the writeback for all dirty pages had already finished
1684	 * with errors - because we were not using AS_EIO/AS_ENOSPC,
1685	 * filemap_fdatawait_range() would return success, as it could not know
1686	 * that writeback errors happened (the pages were no longer tagged for
1687	 * writeback).
1688	 */
1689	switch (eb->log_index) {
1690	case -1:
1691		set_bit(BTRFS_FS_BTREE_ERR, &fs_info->flags);
1692		break;
1693	case 0:
1694		set_bit(BTRFS_FS_LOG1_ERR, &fs_info->flags);
1695		break;
1696	case 1:
1697		set_bit(BTRFS_FS_LOG2_ERR, &fs_info->flags);
1698		break;
1699	default:
1700		BUG(); /* unexpected, logic error */
1701	}
1702}
1703
1704/*
1705 * The endio specific version which won't touch any unsafe spinlock in endio
1706 * context.
1707 */
1708static struct extent_buffer *find_extent_buffer_nolock(
1709		const struct btrfs_fs_info *fs_info, u64 start)
1710{
 
1711	struct extent_buffer *eb;
 
1712
1713	rcu_read_lock();
1714	eb = radix_tree_lookup(&fs_info->buffer_radix,
1715			       start >> fs_info->sectorsize_bits);
1716	if (eb && atomic_inc_not_zero(&eb->refs)) {
1717		rcu_read_unlock();
1718		return eb;
1719	}
1720	rcu_read_unlock();
1721	return NULL;
1722}
1723
1724static void end_bbio_meta_write(struct btrfs_bio *bbio)
1725{
1726	struct extent_buffer *eb = bbio->private;
1727	struct btrfs_fs_info *fs_info = eb->fs_info;
1728	bool uptodate = !bbio->bio.bi_status;
1729	struct folio_iter fi;
1730	u32 bio_offset = 0;
1731
1732	if (!uptodate)
1733		set_btree_ioerr(eb);
1734
1735	bio_for_each_folio_all(fi, &bbio->bio) {
1736		u64 start = eb->start + bio_offset;
1737		struct folio *folio = fi.folio;
1738		u32 len = fi.length;
1739
1740		btrfs_folio_clear_writeback(fs_info, folio, start, len);
1741		bio_offset += len;
1742	}
1743
1744	clear_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags);
1745	smp_mb__after_atomic();
1746	wake_up_bit(&eb->bflags, EXTENT_BUFFER_WRITEBACK);
1747
1748	bio_put(&bbio->bio);
1749}
 
 
 
1750
1751static void prepare_eb_write(struct extent_buffer *eb)
1752{
1753	u32 nritems;
1754	unsigned long start;
1755	unsigned long end;
1756
1757	clear_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags);
 
1758
1759	/* Set btree blocks beyond nritems with 0 to avoid stale content */
1760	nritems = btrfs_header_nritems(eb);
1761	if (btrfs_header_level(eb) > 0) {
1762		end = btrfs_node_key_ptr_offset(eb, nritems);
1763		memzero_extent_buffer(eb, end, eb->len - end);
1764	} else {
1765		/*
1766		 * Leaf:
1767		 * header 0 1 2 .. N ... data_N .. data_2 data_1 data_0
1768		 */
1769		start = btrfs_item_nr_offset(eb, nritems);
1770		end = btrfs_item_nr_offset(eb, 0);
1771		if (nritems == 0)
1772			end += BTRFS_LEAF_DATA_SIZE(eb->fs_info);
1773		else
1774			end += btrfs_item_offset(eb, nritems - 1);
1775		memzero_extent_buffer(eb, start, end - start);
1776	}
1777}
1778
1779static noinline_for_stack void write_one_eb(struct extent_buffer *eb,
1780					    struct writeback_control *wbc)
1781{
1782	struct btrfs_fs_info *fs_info = eb->fs_info;
1783	struct btrfs_bio *bbio;
1784
1785	prepare_eb_write(eb);
1786
1787	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
1788			       REQ_OP_WRITE | REQ_META | wbc_to_write_flags(wbc),
1789			       eb->fs_info, end_bbio_meta_write, eb);
1790	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
1791	bio_set_dev(&bbio->bio, fs_info->fs_devices->latest_dev->bdev);
1792	wbc_init_bio(wbc, &bbio->bio);
1793	bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
1794	bbio->file_offset = eb->start;
1795	if (fs_info->nodesize < PAGE_SIZE) {
1796		struct folio *folio = eb->folios[0];
1797		bool ret;
1798
1799		folio_lock(folio);
1800		btrfs_subpage_set_writeback(fs_info, folio, eb->start, eb->len);
1801		if (btrfs_subpage_clear_and_test_dirty(fs_info, folio, eb->start,
1802						       eb->len)) {
1803			folio_clear_dirty_for_io(folio);
1804			wbc->nr_to_write--;
1805		}
1806		ret = bio_add_folio(&bbio->bio, folio, eb->len,
1807				    eb->start - folio_pos(folio));
1808		ASSERT(ret);
1809		wbc_account_cgroup_owner(wbc, folio, eb->len);
1810		folio_unlock(folio);
1811	} else {
1812		int num_folios = num_extent_folios(eb);
1813
1814		for (int i = 0; i < num_folios; i++) {
1815			struct folio *folio = eb->folios[i];
1816			bool ret;
1817
1818			folio_lock(folio);
1819			folio_clear_dirty_for_io(folio);
1820			folio_start_writeback(folio);
1821			ret = bio_add_folio(&bbio->bio, folio, eb->folio_size, 0);
1822			ASSERT(ret);
1823			wbc_account_cgroup_owner(wbc, folio, eb->folio_size);
1824			wbc->nr_to_write -= folio_nr_pages(folio);
1825			folio_unlock(folio);
1826		}
1827	}
1828	btrfs_submit_bbio(bbio, 0);
1829}
1830
1831/*
1832 * Submit one subpage btree page.
1833 *
1834 * The main difference to submit_eb_page() is:
1835 * - Page locking
1836 *   For subpage, we don't rely on page locking at all.
1837 *
1838 * - Flush write bio
1839 *   We only flush bio if we may be unable to fit current extent buffers into
1840 *   current bio.
1841 *
1842 * Return >=0 for the number of submitted extent buffers.
1843 * Return <0 for fatal error.
1844 */
1845static int submit_eb_subpage(struct folio *folio, struct writeback_control *wbc)
1846{
1847	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
1848	int submitted = 0;
1849	u64 folio_start = folio_pos(folio);
1850	int bit_start = 0;
1851	int sectors_per_node = fs_info->nodesize >> fs_info->sectorsize_bits;
1852
1853	/* Lock and write each dirty extent buffers in the range */
1854	while (bit_start < fs_info->sectors_per_page) {
1855		struct btrfs_subpage *subpage = folio_get_private(folio);
1856		struct extent_buffer *eb;
1857		unsigned long flags;
1858		u64 start;
1859
1860		/*
1861		 * Take private lock to ensure the subpage won't be detached
1862		 * in the meantime.
1863		 */
1864		spin_lock(&folio->mapping->i_private_lock);
1865		if (!folio_test_private(folio)) {
1866			spin_unlock(&folio->mapping->i_private_lock);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1867			break;
1868		}
1869		spin_lock_irqsave(&subpage->lock, flags);
1870		if (!test_bit(bit_start + btrfs_bitmap_nr_dirty * fs_info->sectors_per_page,
1871			      subpage->bitmaps)) {
1872			spin_unlock_irqrestore(&subpage->lock, flags);
1873			spin_unlock(&folio->mapping->i_private_lock);
1874			bit_start++;
1875			continue;
1876		}
1877
1878		start = folio_start + bit_start * fs_info->sectorsize;
1879		bit_start += sectors_per_node;
1880
1881		/*
1882		 * Here we just want to grab the eb without touching extra
1883		 * spin locks, so call find_extent_buffer_nolock().
1884		 */
1885		eb = find_extent_buffer_nolock(fs_info, start);
1886		spin_unlock_irqrestore(&subpage->lock, flags);
1887		spin_unlock(&folio->mapping->i_private_lock);
1888
1889		/*
1890		 * The eb has already reached 0 refs thus find_extent_buffer()
1891		 * doesn't return it. We don't need to write back such eb
1892		 * anyway.
1893		 */
1894		if (!eb)
1895			continue;
1896
1897		if (lock_extent_buffer_for_io(eb, wbc)) {
1898			write_one_eb(eb, wbc);
1899			submitted++;
1900		}
1901		free_extent_buffer(eb);
1902	}
1903	return submitted;
1904}
1905
1906/*
1907 * Submit all page(s) of one extent buffer.
1908 *
1909 * @page:	the page of one extent buffer
1910 * @eb_context:	to determine if we need to submit this page, if current page
1911 *		belongs to this eb, we don't need to submit
1912 *
1913 * The caller should pass each page in their bytenr order, and here we use
1914 * @eb_context to determine if we have submitted pages of one extent buffer.
1915 *
1916 * If we have, we just skip until we hit a new page that doesn't belong to
1917 * current @eb_context.
1918 *
1919 * If not, we submit all the page(s) of the extent buffer.
1920 *
1921 * Return >0 if we have submitted the extent buffer successfully.
1922 * Return 0 if we don't need to submit the page, as it's already submitted by
1923 * previous call.
1924 * Return <0 for fatal error.
1925 */
1926static int submit_eb_page(struct folio *folio, struct btrfs_eb_write_context *ctx)
1927{
1928	struct writeback_control *wbc = ctx->wbc;
1929	struct address_space *mapping = folio->mapping;
1930	struct extent_buffer *eb;
1931	int ret;
1932
1933	if (!folio_test_private(folio))
1934		return 0;
1935
1936	if (folio_to_fs_info(folio)->nodesize < PAGE_SIZE)
1937		return submit_eb_subpage(folio, wbc);
1938
1939	spin_lock(&mapping->i_private_lock);
1940	if (!folio_test_private(folio)) {
1941		spin_unlock(&mapping->i_private_lock);
1942		return 0;
1943	}
1944
1945	eb = folio_get_private(folio);
1946
1947	/*
1948	 * Shouldn't happen and normally this would be a BUG_ON but no point
1949	 * crashing the machine for something we can survive anyway.
1950	 */
1951	if (WARN_ON(!eb)) {
1952		spin_unlock(&mapping->i_private_lock);
1953		return 0;
1954	}
1955
1956	if (eb == ctx->eb) {
1957		spin_unlock(&mapping->i_private_lock);
1958		return 0;
1959	}
1960	ret = atomic_inc_not_zero(&eb->refs);
1961	spin_unlock(&mapping->i_private_lock);
1962	if (!ret)
1963		return 0;
1964
1965	ctx->eb = eb;
1966
1967	ret = btrfs_check_meta_write_pointer(eb->fs_info, ctx);
1968	if (ret) {
1969		if (ret == -EBUSY)
1970			ret = 0;
1971		free_extent_buffer(eb);
1972		return ret;
1973	}
1974
1975	if (!lock_extent_buffer_for_io(eb, wbc)) {
1976		free_extent_buffer(eb);
1977		return 0;
1978	}
1979	/* Implies write in zoned mode. */
1980	if (ctx->zoned_bg) {
1981		/* Mark the last eb in the block group. */
1982		btrfs_schedule_zone_finish_bg(ctx->zoned_bg, eb);
1983		ctx->zoned_bg->meta_write_pointer += eb->len;
1984	}
1985	write_one_eb(eb, wbc);
1986	free_extent_buffer(eb);
1987	return 1;
1988}
1989
1990int btree_write_cache_pages(struct address_space *mapping,
1991				   struct writeback_control *wbc)
1992{
1993	struct btrfs_eb_write_context ctx = { .wbc = wbc };
1994	struct btrfs_fs_info *fs_info = inode_to_fs_info(mapping->host);
 
 
 
 
 
 
 
 
1995	int ret = 0;
1996	int done = 0;
1997	int nr_to_write_done = 0;
1998	struct folio_batch fbatch;
1999	unsigned int nr_folios;
2000	pgoff_t index;
2001	pgoff_t end;		/* Inclusive */
2002	int scanned = 0;
2003	xa_mark_t tag;
2004
2005	folio_batch_init(&fbatch);
2006	if (wbc->range_cyclic) {
2007		index = mapping->writeback_index; /* Start from prev offset */
2008		end = -1;
2009		/*
2010		 * Start from the beginning does not need to cycle over the
2011		 * range, mark it as scanned.
2012		 */
2013		scanned = (index == 0);
2014	} else {
2015		index = wbc->range_start >> PAGE_SHIFT;
2016		end = wbc->range_end >> PAGE_SHIFT;
2017		scanned = 1;
2018	}
2019	if (wbc->sync_mode == WB_SYNC_ALL)
2020		tag = PAGECACHE_TAG_TOWRITE;
2021	else
2022		tag = PAGECACHE_TAG_DIRTY;
2023	btrfs_zoned_meta_io_lock(fs_info);
2024retry:
2025	if (wbc->sync_mode == WB_SYNC_ALL)
2026		tag_pages_for_writeback(mapping, index, end);
2027	while (!done && !nr_to_write_done && (index <= end) &&
2028	       (nr_folios = filemap_get_folios_tag(mapping, &index, end,
2029					    tag, &fbatch))) {
2030		unsigned i;
2031
2032		for (i = 0; i < nr_folios; i++) {
2033			struct folio *folio = fbatch.folios[i];
 
2034
2035			ret = submit_eb_page(folio, &ctx);
2036			if (ret == 0)
2037				continue;
2038			if (ret < 0) {
 
2039				done = 1;
2040				break;
2041			}
2042
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2043			/*
2044			 * the filesystem may choose to bump up nr_to_write.
2045			 * We have to make sure to honor the new nr_to_write
2046			 * at any time
2047			 */
2048			nr_to_write_done = wbc->nr_to_write <= 0;
2049		}
2050		folio_batch_release(&fbatch);
2051		cond_resched();
2052	}
2053	if (!scanned && !done) {
2054		/*
2055		 * We hit the last page and there is more work to be done: wrap
2056		 * back to the start of the file
2057		 */
2058		scanned = 1;
2059		index = 0;
2060		goto retry;
2061	}
2062	/*
2063	 * If something went wrong, don't allow any metadata write bio to be
2064	 * submitted.
2065	 *
2066	 * This would prevent use-after-free if we had dirty pages not
2067	 * cleaned up, which can still happen by fuzzed images.
2068	 *
2069	 * - Bad extent tree
2070	 *   Allowing existing tree block to be allocated for other trees.
2071	 *
2072	 * - Log tree operations
2073	 *   Exiting tree blocks get allocated to log tree, bumps its
2074	 *   generation, then get cleaned in tree re-balance.
2075	 *   Such tree block will not be written back, since it's clean,
2076	 *   thus no WRITTEN flag set.
2077	 *   And after log writes back, this tree block is not traced by
2078	 *   any dirty extent_io_tree.
2079	 *
2080	 * - Offending tree block gets re-dirtied from its original owner
2081	 *   Since it has bumped generation, no WRITTEN flag, it can be
2082	 *   reused without COWing. This tree block will not be traced
2083	 *   by btrfs_transaction::dirty_pages.
2084	 *
2085	 *   Now such dirty tree block will not be cleaned by any dirty
2086	 *   extent io tree. Thus we don't want to submit such wild eb
2087	 *   if the fs already has error.
2088	 *
2089	 * We can get ret > 0 from submit_extent_folio() indicating how many ebs
2090	 * were submitted. Reset it to 0 to avoid false alerts for the caller.
2091	 */
2092	if (ret > 0)
2093		ret = 0;
2094	if (!ret && BTRFS_FS_ERROR(fs_info))
2095		ret = -EROFS;
2096
2097	if (ctx.zoned_bg)
2098		btrfs_put_block_group(ctx.zoned_bg);
2099	btrfs_zoned_meta_io_unlock(fs_info);
2100	return ret;
2101}
2102
2103/*
2104 * Walk the list of dirty pages of the given address space and write all of them.
2105 *
2106 * @mapping:   address space structure to write
2107 * @wbc:       subtract the number of written pages from *@wbc->nr_to_write
2108 * @bio_ctrl:  holds context for the write, namely the bio
2109 *
2110 * If a page is already under I/O, write_cache_pages() skips it, even
2111 * if it's dirty.  This is desirable behaviour for memory-cleaning writeback,
2112 * but it is INCORRECT for data-integrity system calls such as fsync().  fsync()
2113 * and msync() need to guarantee that all the data which was dirty at the time
2114 * the call was made get new I/O started against them.  If wbc->sync_mode is
2115 * WB_SYNC_ALL then we were called for data integrity and we must wait for
2116 * existing IO to complete.
2117 */
2118static int extent_write_cache_pages(struct address_space *mapping,
2119			     struct btrfs_bio_ctrl *bio_ctrl)
 
 
 
2120{
2121	struct writeback_control *wbc = bio_ctrl->wbc;
2122	struct inode *inode = mapping->host;
2123	int ret = 0;
2124	int done = 0;
 
2125	int nr_to_write_done = 0;
2126	struct folio_batch fbatch;
2127	unsigned int nr_folios;
2128	pgoff_t index;
2129	pgoff_t end;		/* Inclusive */
2130	pgoff_t done_index;
2131	int range_whole = 0;
2132	int scanned = 0;
2133	xa_mark_t tag;
2134
2135	/*
2136	 * We have to hold onto the inode so that ordered extents can do their
2137	 * work when the IO finishes.  The alternative to this is failing to add
2138	 * an ordered extent if the igrab() fails there and that is a huge pain
2139	 * to deal with, so instead just hold onto the inode throughout the
2140	 * writepages operation.  If it fails here we are freeing up the inode
2141	 * anyway and we'd rather not waste our time writing out stuff that is
2142	 * going to be truncated anyway.
2143	 */
2144	if (!igrab(inode))
2145		return 0;
2146
2147	folio_batch_init(&fbatch);
2148	if (wbc->range_cyclic) {
2149		index = mapping->writeback_index; /* Start from prev offset */
2150		end = -1;
2151		/*
2152		 * Start from the beginning does not need to cycle over the
2153		 * range, mark it as scanned.
2154		 */
2155		scanned = (index == 0);
2156	} else {
2157		index = wbc->range_start >> PAGE_SHIFT;
2158		end = wbc->range_end >> PAGE_SHIFT;
2159		if (wbc->range_start == 0 && wbc->range_end == LLONG_MAX)
2160			range_whole = 1;
2161		scanned = 1;
2162	}
2163
2164	/*
2165	 * We do the tagged writepage as long as the snapshot flush bit is set
2166	 * and we are the first one who do the filemap_flush() on this inode.
2167	 *
2168	 * The nr_to_write == LONG_MAX is needed to make sure other flushers do
2169	 * not race in and drop the bit.
2170	 */
2171	if (range_whole && wbc->nr_to_write == LONG_MAX &&
2172	    test_and_clear_bit(BTRFS_INODE_SNAPSHOT_FLUSH,
2173			       &BTRFS_I(inode)->runtime_flags))
2174		wbc->tagged_writepages = 1;
2175
2176	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2177		tag = PAGECACHE_TAG_TOWRITE;
2178	else
2179		tag = PAGECACHE_TAG_DIRTY;
2180retry:
2181	if (wbc->sync_mode == WB_SYNC_ALL || wbc->tagged_writepages)
2182		tag_pages_for_writeback(mapping, index, end);
2183	done_index = index;
2184	while (!done && !nr_to_write_done && (index <= end) &&
2185			(nr_folios = filemap_get_folios_tag(mapping, &index,
2186							end, tag, &fbatch))) {
2187		unsigned i;
2188
2189		for (i = 0; i < nr_folios; i++) {
2190			struct folio *folio = fbatch.folios[i];
 
2191
2192			done_index = folio_next_index(folio);
2193			/*
2194			 * At this point we hold neither the i_pages lock nor
2195			 * the page lock: the page may be truncated or
2196			 * invalidated (changing page->mapping to NULL),
2197			 * or even swizzled back from swapper_space to
2198			 * tmpfs file mapping
2199			 */
2200			if (!folio_trylock(folio)) {
2201				submit_write_bio(bio_ctrl, 0);
2202				folio_lock(folio);
2203			}
2204
2205			if (unlikely(folio->mapping != mapping)) {
2206				folio_unlock(folio);
2207				continue;
2208			}
2209
2210			if (!folio_test_dirty(folio)) {
2211				/* Someone wrote it for us. */
2212				folio_unlock(folio);
2213				continue;
2214			}
2215
2216			/*
2217			 * For subpage case, compression can lead to mixed
2218			 * writeback and dirty flags, e.g:
2219			 * 0     32K    64K    96K    128K
2220			 * |     |//////||/////|   |//|
2221			 *
2222			 * In above case, [32K, 96K) is asynchronously submitted
2223			 * for compression, and [124K, 128K) needs to be written back.
2224			 *
2225			 * If we didn't wait wrtiteback for page 64K, [128K, 128K)
2226			 * won't be submitted as the page still has writeback flag
2227			 * and will be skipped in the next check.
2228			 *
2229			 * This mixed writeback and dirty case is only possible for
2230			 * subpage case.
2231			 *
2232			 * TODO: Remove this check after migrating compression to
2233			 * regular submission.
2234			 */
2235			if (wbc->sync_mode != WB_SYNC_NONE ||
2236			    btrfs_is_subpage(inode_to_fs_info(inode), mapping)) {
2237				if (folio_test_writeback(folio))
2238					submit_write_bio(bio_ctrl, 0);
2239				folio_wait_writeback(folio);
2240			}
2241
2242			if (folio_test_writeback(folio) ||
2243			    !folio_clear_dirty_for_io(folio)) {
2244				folio_unlock(folio);
2245				continue;
2246			}
2247
2248			ret = extent_writepage(folio, bio_ctrl);
2249			if (ret < 0) {
2250				done = 1;
2251				break;
 
2252			}
 
 
2253
2254			/*
2255			 * The filesystem may choose to bump up nr_to_write.
2256			 * We have to make sure to honor the new nr_to_write
2257			 * at any time.
2258			 */
2259			nr_to_write_done = (wbc->sync_mode == WB_SYNC_NONE &&
2260					    wbc->nr_to_write <= 0);
2261		}
2262		folio_batch_release(&fbatch);
2263		cond_resched();
2264	}
2265	if (!scanned && !done) {
2266		/*
2267		 * We hit the last page and there is more work to be done: wrap
2268		 * back to the start of the file
2269		 */
2270		scanned = 1;
2271		index = 0;
2272
2273		/*
2274		 * If we're looping we could run into a page that is locked by a
2275		 * writer and that writer could be waiting on writeback for a
2276		 * page in our current bio, and thus deadlock, so flush the
2277		 * write bio here.
2278		 */
2279		submit_write_bio(bio_ctrl, 0);
2280		goto retry;
2281	}
2282
2283	if (wbc->range_cyclic || (wbc->nr_to_write > 0 && range_whole))
2284		mapping->writeback_index = done_index;
2285
2286	btrfs_add_delayed_iput(BTRFS_I(inode));
2287	return ret;
2288}
2289
2290/*
2291 * Submit the pages in the range to bio for call sites which delalloc range has
2292 * already been ran (aka, ordered extent inserted) and all pages are still
2293 * locked.
2294 */
2295void extent_write_locked_range(struct inode *inode, const struct folio *locked_folio,
2296			       u64 start, u64 end, struct writeback_control *wbc,
2297			       bool pages_dirty)
2298{
2299	bool found_error = false;
2300	int ret = 0;
2301	struct address_space *mapping = inode->i_mapping;
2302	struct btrfs_fs_info *fs_info = inode_to_fs_info(inode);
2303	const u32 sectorsize = fs_info->sectorsize;
2304	loff_t i_size = i_size_read(inode);
2305	u64 cur = start;
2306	struct btrfs_bio_ctrl bio_ctrl = {
2307		.wbc = wbc,
2308		.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
2309	};
2310
2311	if (wbc->no_cgroup_owner)
2312		bio_ctrl.opf |= REQ_BTRFS_CGROUP_PUNT;
2313
2314	ASSERT(IS_ALIGNED(start, sectorsize) && IS_ALIGNED(end + 1, sectorsize));
 
 
 
 
2315
2316	while (cur <= end) {
2317		u64 cur_end = min(round_down(cur, PAGE_SIZE) + PAGE_SIZE - 1, end);
2318		u32 cur_len = cur_end + 1 - cur;
2319		struct folio *folio;
 
2320
2321		folio = filemap_get_folio(mapping, cur >> PAGE_SHIFT);
 
 
 
 
 
 
 
 
 
 
 
 
2322
2323		/*
2324		 * This shouldn't happen, the pages are pinned and locked, this
2325		 * code is just in case, but shouldn't actually be run.
2326		 */
2327		if (IS_ERR(folio)) {
2328			btrfs_mark_ordered_io_finished(BTRFS_I(inode), NULL,
2329						       cur, cur_len, false);
2330			mapping_set_error(mapping, PTR_ERR(folio));
2331			cur = cur_end + 1;
2332			continue;
2333		}
2334
2335		ASSERT(folio_test_locked(folio));
2336		if (pages_dirty && folio != locked_folio)
2337			ASSERT(folio_test_dirty(folio));
2338
2339		/*
2340		 * Set the submission bitmap to submit all sectors.
2341		 * extent_writepage_io() will do the truncation correctly.
2342		 */
2343		bio_ctrl.submit_bitmap = (unsigned long)-1;
2344		ret = extent_writepage_io(BTRFS_I(inode), folio, cur, cur_len,
2345					  &bio_ctrl, i_size);
2346		if (ret == 1)
2347			goto next_page;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2348
2349		if (ret)
2350			mapping_set_error(mapping, ret);
2351		btrfs_folio_end_lock(fs_info, folio, cur, cur_len);
2352		if (ret < 0)
2353			found_error = true;
2354next_page:
2355		folio_put(folio);
2356		cur = cur_end + 1;
 
 
 
 
 
2357	}
2358
2359	submit_write_bio(&bio_ctrl, found_error ? ret : 0);
 
2360}
2361
2362int btrfs_writepages(struct address_space *mapping, struct writeback_control *wbc)
 
 
 
2363{
2364	struct inode *inode = mapping->host;
2365	int ret = 0;
2366	struct btrfs_bio_ctrl bio_ctrl = {
2367		.wbc = wbc,
2368		.opf = REQ_OP_WRITE | wbc_to_write_flags(wbc),
 
 
 
 
2369	};
2370
2371	/*
2372	 * Allow only a single thread to do the reloc work in zoned mode to
2373	 * protect the write pointer updates.
2374	 */
2375	btrfs_zoned_data_reloc_lock(BTRFS_I(inode));
2376	ret = extent_write_cache_pages(mapping, &bio_ctrl);
2377	submit_write_bio(&bio_ctrl, ret);
2378	btrfs_zoned_data_reloc_unlock(BTRFS_I(inode));
2379	return ret;
2380}
2381
2382void btrfs_readahead(struct readahead_control *rac)
2383{
2384	struct btrfs_bio_ctrl bio_ctrl = { .opf = REQ_OP_READ | REQ_RAHEAD };
2385	struct folio *folio;
2386	struct btrfs_inode *inode = BTRFS_I(rac->mapping->host);
2387	const u64 start = readahead_pos(rac);
2388	const u64 end = start + readahead_length(rac) - 1;
2389	struct extent_state *cached_state = NULL;
 
 
2390	struct extent_map *em_cached = NULL;
 
2391	u64 prev_em_start = (u64)-1;
2392
2393	btrfs_lock_and_flush_ordered_range(inode, start, end, &cached_state);
 
2394
2395	while ((folio = readahead_folio(rac)) != NULL)
2396		btrfs_do_readpage(folio, &em_cached, &bio_ctrl, &prev_em_start);
 
 
 
 
 
2397
2398	unlock_extent(&inode->io_tree, start, end, &cached_state);
 
 
 
 
 
 
 
 
 
2399
2400	if (em_cached)
2401		free_extent_map(em_cached);
2402	submit_one_bio(&bio_ctrl);
 
 
 
 
2403}
2404
2405/*
2406 * basic invalidate_folio code, this waits on any locked or writeback
2407 * ranges corresponding to the folio, and then deletes any extent state
2408 * records from the tree
2409 */
2410int extent_invalidate_folio(struct extent_io_tree *tree,
2411			  struct folio *folio, size_t offset)
2412{
2413	struct extent_state *cached_state = NULL;
2414	u64 start = folio_pos(folio);
2415	u64 end = start + folio_size(folio) - 1;
2416	size_t blocksize = folio_to_fs_info(folio)->sectorsize;
2417
2418	/* This function is only called for the btree inode */
2419	ASSERT(tree->owner == IO_TREE_BTREE_INODE_IO);
2420
2421	start += ALIGN(offset, blocksize);
2422	if (start > end)
2423		return 0;
2424
2425	lock_extent(tree, start, end, &cached_state);
2426	folio_wait_writeback(folio);
2427
2428	/*
2429	 * Currently for btree io tree, only EXTENT_LOCKED is utilized,
2430	 * so here we only need to unlock the extent range to free any
2431	 * existing extent state.
2432	 */
2433	unlock_extent(tree, start, end, &cached_state);
2434	return 0;
2435}
2436
2437/*
2438 * a helper for release_folio, this tests for areas of the page that
2439 * are locked or under IO and drops the related state bits if it is safe
2440 * to drop the page.
2441 */
2442static bool try_release_extent_state(struct extent_io_tree *tree,
2443				     struct folio *folio)
 
2444{
2445	u64 start = folio_pos(folio);
2446	u64 end = start + PAGE_SIZE - 1;
2447	bool ret;
2448
2449	if (test_range_bit_exists(tree, start, end, EXTENT_LOCKED)) {
2450		ret = false;
2451	} else {
2452		u32 clear_bits = ~(EXTENT_LOCKED | EXTENT_NODATASUM |
2453				   EXTENT_DELALLOC_NEW | EXTENT_CTLBITS |
2454				   EXTENT_QGROUP_RESERVED);
2455		int ret2;
2456
 
 
 
 
 
 
2457		/*
2458		 * At this point we can safely clear everything except the
2459		 * locked bit, the nodatasum bit and the delalloc new bit.
2460		 * The delalloc new bit will be cleared by ordered extent
2461		 * completion.
2462		 */
2463		ret2 = __clear_extent_bit(tree, start, end, clear_bits, NULL, NULL);
 
 
2464
2465		/* if clear_extent_bit failed for enomem reasons,
2466		 * we can't allow the release to continue.
2467		 */
2468		if (ret2 < 0)
2469			ret = false;
2470		else
2471			ret = true;
2472	}
2473	return ret;
2474}
2475
2476/*
2477 * a helper for release_folio.  As long as there are no locked extents
2478 * in the range corresponding to the page, both state records and extent
2479 * map records are removed
2480 */
2481bool try_release_extent_mapping(struct folio *folio, gfp_t mask)
 
 
2482{
2483	u64 start = folio_pos(folio);
 
2484	u64 end = start + PAGE_SIZE - 1;
2485	struct btrfs_inode *inode = folio_to_inode(folio);
2486	struct extent_io_tree *io_tree = &inode->io_tree;
2487
2488	while (start <= end) {
2489		const u64 cur_gen = btrfs_get_fs_generation(inode->root->fs_info);
2490		const u64 len = end - start + 1;
2491		struct extent_map_tree *extent_tree = &inode->extent_tree;
2492		struct extent_map *em;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2493
2494		write_lock(&extent_tree->lock);
2495		em = lookup_extent_mapping(extent_tree, start, len);
2496		if (!em) {
2497			write_unlock(&extent_tree->lock);
2498			break;
2499		}
2500		if ((em->flags & EXTENT_FLAG_PINNED) || em->start != start) {
2501			write_unlock(&extent_tree->lock);
2502			free_extent_map(em);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2503			break;
 
 
 
 
 
 
 
 
 
2504		}
2505		if (test_range_bit_exists(io_tree, em->start,
2506					  extent_map_end(em) - 1, EXTENT_LOCKED))
2507			goto next;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2508		/*
2509		 * If it's not in the list of modified extents, used by a fast
2510		 * fsync, we can remove it. If it's being logged we can safely
2511		 * remove it since fsync took an extra reference on the em.
 
2512		 */
2513		if (list_empty(&em->list) || (em->flags & EXTENT_FLAG_LOGGING))
2514			goto remove_em;
2515		/*
2516		 * If it's in the list of modified extents, remove it only if
2517		 * its generation is older then the current one, in which case
2518		 * we don't need it for a fast fsync. Otherwise don't remove it,
2519		 * we could be racing with an ongoing fast fsync that could miss
2520		 * the new extent.
2521		 */
2522		if (em->generation >= cur_gen)
2523			goto next;
2524remove_em:
 
 
 
 
2525		/*
2526		 * We only remove extent maps that are not in the list of
2527		 * modified extents or that are in the list but with a
2528		 * generation lower then the current generation, so there is no
2529		 * need to set the full fsync flag on the inode (it hurts the
2530		 * fsync performance for workloads with a data size that exceeds
2531		 * or is close to the system's memory).
2532		 */
2533		remove_extent_mapping(inode, em);
2534		/* Once for the inode's extent map tree. */
2535		free_extent_map(em);
2536next:
2537		start = extent_map_end(em);
2538		write_unlock(&extent_tree->lock);
 
 
 
 
 
 
 
 
 
 
2539
2540		/* Once for us, for the lookup_extent_mapping() reference. */
2541		free_extent_map(em);
2542
2543		if (need_resched()) {
2544			/*
2545			 * If we need to resched but we can't block just exit
2546			 * and leave any remaining extent maps.
 
 
 
2547			 */
2548			if (!gfpflags_allow_blocking(mask))
2549				break;
 
 
 
 
 
 
 
 
 
 
 
2550
2551			cond_resched();
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2552		}
2553	}
2554	return try_release_extent_state(io_tree, folio);
 
 
 
 
 
 
2555}
2556
2557static void __free_extent_buffer(struct extent_buffer *eb)
2558{
 
2559	kmem_cache_free(extent_buffer_cache, eb);
2560}
2561
2562static int extent_buffer_under_io(const struct extent_buffer *eb)
2563{
2564	return (test_bit(EXTENT_BUFFER_WRITEBACK, &eb->bflags) ||
 
2565		test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
2566}
2567
2568static bool folio_range_has_eb(struct folio *folio)
2569{
2570	struct btrfs_subpage *subpage;
2571
2572	lockdep_assert_held(&folio->mapping->i_private_lock);
2573
2574	if (folio_test_private(folio)) {
2575		subpage = folio_get_private(folio);
2576		if (atomic_read(&subpage->eb_refs))
2577			return true;
2578	}
2579	return false;
2580}
2581
2582static void detach_extent_buffer_folio(const struct extent_buffer *eb, struct folio *folio)
2583{
2584	struct btrfs_fs_info *fs_info = eb->fs_info;
2585	const bool mapped = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
 
2586
2587	/*
2588	 * For mapped eb, we're going to change the folio private, which should
2589	 * be done under the i_private_lock.
2590	 */
2591	if (mapped)
2592		spin_lock(&folio->mapping->i_private_lock);
2593
2594	if (!folio_test_private(folio)) {
2595		if (mapped)
2596			spin_unlock(&folio->mapping->i_private_lock);
2597		return;
2598	}
2599
2600	if (fs_info->nodesize >= PAGE_SIZE) {
 
 
 
 
 
 
2601		/*
2602		 * We do this since we'll remove the pages after we've
2603		 * removed the eb from the radix tree, so we could race
2604		 * and have this page now attached to the new eb.  So
2605		 * only clear folio if it's still connected to
2606		 * this eb.
2607		 */
2608		if (folio_test_private(folio) && folio_get_private(folio) == eb) {
 
2609			BUG_ON(test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags));
2610			BUG_ON(folio_test_dirty(folio));
2611			BUG_ON(folio_test_writeback(folio));
2612			/* We need to make sure we haven't be attached to a new eb. */
2613			folio_detach_private(folio);
 
 
 
 
 
 
2614		}
2615		if (mapped)
2616			spin_unlock(&folio->mapping->i_private_lock);
2617		return;
2618	}
2619
2620	/*
2621	 * For subpage, we can have dummy eb with folio private attached.  In
2622	 * this case, we can directly detach the private as such folio is only
2623	 * attached to one dummy eb, no sharing.
2624	 */
2625	if (!mapped) {
2626		btrfs_detach_subpage(fs_info, folio);
2627		return;
2628	}
2629
2630	btrfs_folio_dec_eb_refs(fs_info, folio);
2631
2632	/*
2633	 * We can only detach the folio private if there are no other ebs in the
2634	 * page range and no unfinished IO.
2635	 */
2636	if (!folio_range_has_eb(folio))
2637		btrfs_detach_subpage(fs_info, folio);
2638
2639	spin_unlock(&folio->mapping->i_private_lock);
2640}
2641
2642/* Release all pages attached to the extent buffer */
2643static void btrfs_release_extent_buffer_pages(const struct extent_buffer *eb)
2644{
2645	ASSERT(!extent_buffer_under_io(eb));
2646
2647	for (int i = 0; i < INLINE_EXTENT_BUFFER_PAGES; i++) {
2648		struct folio *folio = eb->folios[i];
2649
2650		if (!folio)
2651			continue;
2652
2653		detach_extent_buffer_folio(eb, folio);
2654
2655		/* One for when we allocated the folio. */
2656		folio_put(folio);
2657	}
2658}
2659
2660/*
2661 * Helper for releasing the extent buffer.
2662 */
2663static inline void btrfs_release_extent_buffer(struct extent_buffer *eb)
2664{
2665	btrfs_release_extent_buffer_pages(eb);
2666	btrfs_leak_debug_del_eb(eb);
2667	__free_extent_buffer(eb);
2668}
2669
2670static struct extent_buffer *
2671__alloc_extent_buffer(struct btrfs_fs_info *fs_info, u64 start,
2672		      unsigned long len)
2673{
2674	struct extent_buffer *eb = NULL;
2675
2676	eb = kmem_cache_zalloc(extent_buffer_cache, GFP_NOFS|__GFP_NOFAIL);
2677	eb->start = start;
2678	eb->len = len;
2679	eb->fs_info = fs_info;
2680	init_rwsem(&eb->lock);
 
 
 
 
 
 
 
 
 
 
2681
2682	btrfs_leak_debug_add_eb(eb);
2683
2684	spin_lock_init(&eb->refs_lock);
2685	atomic_set(&eb->refs, 1);
 
2686
2687	ASSERT(len <= BTRFS_MAX_METADATA_BLOCKSIZE);
 
 
 
 
 
2688
2689	return eb;
2690}
2691
2692struct extent_buffer *btrfs_clone_extent_buffer(const struct extent_buffer *src)
2693{
 
 
2694	struct extent_buffer *new;
2695	int num_folios = num_extent_folios(src);
2696	int ret;
2697
2698	new = __alloc_extent_buffer(src->fs_info, src->start, src->len);
2699	if (new == NULL)
2700		return NULL;
2701
2702	/*
2703	 * Set UNMAPPED before calling btrfs_release_extent_buffer(), as
2704	 * btrfs_release_extent_buffer() have different behavior for
2705	 * UNMAPPED subpage extent buffer.
2706	 */
2707	set_bit(EXTENT_BUFFER_UNMAPPED, &new->bflags);
2708
2709	ret = alloc_eb_folio_array(new, false);
2710	if (ret) {
2711		btrfs_release_extent_buffer(new);
2712		return NULL;
2713	}
2714
2715	for (int i = 0; i < num_folios; i++) {
2716		struct folio *folio = new->folios[i];
2717
2718		ret = attach_extent_buffer_folio(new, folio, NULL);
2719		if (ret < 0) {
2720			btrfs_release_extent_buffer(new);
2721			return NULL;
2722		}
2723		WARN_ON(folio_test_dirty(folio));
 
 
 
2724	}
2725	copy_extent_buffer_full(new, src);
2726	set_extent_buffer_uptodate(new);
 
 
2727
2728	return new;
2729}
2730
2731struct extent_buffer *__alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
2732						  u64 start, unsigned long len)
2733{
2734	struct extent_buffer *eb;
2735	int num_folios = 0;
2736	int ret;
 
 
2737
2738	eb = __alloc_extent_buffer(fs_info, start, len);
2739	if (!eb)
2740		return NULL;
2741
2742	ret = alloc_eb_folio_array(eb, false);
2743	if (ret)
2744		goto err;
2745
2746	num_folios = num_extent_folios(eb);
2747	for (int i = 0; i < num_folios; i++) {
2748		ret = attach_extent_buffer_folio(eb, eb->folios[i], NULL);
2749		if (ret < 0)
2750			goto err;
2751	}
2752
2753	set_extent_buffer_uptodate(eb);
2754	btrfs_set_header_nritems(eb, 0);
2755	set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
2756
2757	return eb;
2758err:
2759	for (int i = 0; i < num_folios; i++) {
2760		if (eb->folios[i]) {
2761			detach_extent_buffer_folio(eb, eb->folios[i]);
2762			folio_put(eb->folios[i]);
2763		}
2764	}
2765	__free_extent_buffer(eb);
2766	return NULL;
2767}
2768
2769struct extent_buffer *alloc_dummy_extent_buffer(struct btrfs_fs_info *fs_info,
2770						u64 start)
2771{
2772	return __alloc_dummy_extent_buffer(fs_info, start, fs_info->nodesize);
 
 
 
 
 
 
 
 
 
 
 
 
2773}
2774
2775static void check_buffer_tree_ref(struct extent_buffer *eb)
2776{
2777	int refs;
2778	/*
2779	 * The TREE_REF bit is first set when the extent_buffer is added
2780	 * to the radix tree. It is also reset, if unset, when a new reference
2781	 * is created by find_extent_buffer.
2782	 *
2783	 * It is only cleared in two cases: freeing the last non-tree
2784	 * reference to the extent_buffer when its STALE bit is set or
2785	 * calling release_folio when the tree reference is the only reference.
 
2786	 *
2787	 * In both cases, care is taken to ensure that the extent_buffer's
2788	 * pages are not under io. However, release_folio can be concurrently
2789	 * called with creating new references, which is prone to race
2790	 * conditions between the calls to check_buffer_tree_ref in those
2791	 * codepaths and clearing TREE_REF in try_release_extent_buffer.
 
2792	 *
2793	 * The actual lifetime of the extent_buffer in the radix tree is
2794	 * adequately protected by the refcount, but the TREE_REF bit and
2795	 * its corresponding reference are not. To protect against this
2796	 * class of races, we call check_buffer_tree_ref from the codepaths
2797	 * which trigger io. Note that once io is initiated, TREE_REF can no
2798	 * longer be cleared, so that is the moment at which any such race is
2799	 * best fixed.
2800	 */
2801	refs = atomic_read(&eb->refs);
2802	if (refs >= 2 && test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
2803		return;
2804
2805	spin_lock(&eb->refs_lock);
2806	if (!test_and_set_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
2807		atomic_inc(&eb->refs);
2808	spin_unlock(&eb->refs_lock);
2809}
2810
2811static void mark_extent_buffer_accessed(struct extent_buffer *eb)
 
2812{
2813	int num_folios= num_extent_folios(eb);
2814
2815	check_buffer_tree_ref(eb);
2816
2817	for (int i = 0; i < num_folios; i++)
2818		folio_mark_accessed(eb->folios[i]);
 
 
 
 
 
2819}
2820
2821struct extent_buffer *find_extent_buffer(struct btrfs_fs_info *fs_info,
2822					 u64 start)
2823{
2824	struct extent_buffer *eb;
2825
2826	eb = find_extent_buffer_nolock(fs_info, start);
2827	if (!eb)
2828		return NULL;
2829	/*
2830	 * Lock our eb's refs_lock to avoid races with free_extent_buffer().
2831	 * When we get our eb it might be flagged with EXTENT_BUFFER_STALE and
2832	 * another task running free_extent_buffer() might have seen that flag
2833	 * set, eb->refs == 2, that the buffer isn't under IO (dirty and
2834	 * writeback flags not set) and it's still in the tree (flag
2835	 * EXTENT_BUFFER_TREE_REF set), therefore being in the process of
2836	 * decrementing the extent buffer's reference count twice.  So here we
2837	 * could race and increment the eb's reference count, clear its stale
2838	 * flag, mark it as dirty and drop our reference before the other task
2839	 * finishes executing free_extent_buffer, which would later result in
2840	 * an attempt to free an extent buffer that is dirty.
2841	 */
2842	if (test_bit(EXTENT_BUFFER_STALE, &eb->bflags)) {
2843		spin_lock(&eb->refs_lock);
2844		spin_unlock(&eb->refs_lock);
 
 
 
 
 
 
 
2845	}
2846	mark_extent_buffer_accessed(eb);
2847	return eb;
 
2848}
2849
2850#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
2851struct extent_buffer *alloc_test_extent_buffer(struct btrfs_fs_info *fs_info,
2852					u64 start)
2853{
2854	struct extent_buffer *eb, *exists = NULL;
2855	int ret;
2856
2857	eb = find_extent_buffer(fs_info, start);
2858	if (eb)
2859		return eb;
2860	eb = alloc_dummy_extent_buffer(fs_info, start);
2861	if (!eb)
2862		return ERR_PTR(-ENOMEM);
2863	eb->fs_info = fs_info;
2864again:
2865	ret = radix_tree_preload(GFP_NOFS);
2866	if (ret) {
2867		exists = ERR_PTR(ret);
2868		goto free_eb;
2869	}
2870	spin_lock(&fs_info->buffer_lock);
2871	ret = radix_tree_insert(&fs_info->buffer_radix,
2872				start >> fs_info->sectorsize_bits, eb);
2873	spin_unlock(&fs_info->buffer_lock);
2874	radix_tree_preload_end();
2875	if (ret == -EEXIST) {
2876		exists = find_extent_buffer(fs_info, start);
2877		if (exists)
2878			goto free_eb;
2879		else
2880			goto again;
2881	}
2882	check_buffer_tree_ref(eb);
2883	set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
2884
 
 
 
 
 
 
 
2885	return eb;
2886free_eb:
2887	btrfs_release_extent_buffer(eb);
2888	return exists;
2889}
2890#endif
2891
2892static struct extent_buffer *grab_extent_buffer(
2893		struct btrfs_fs_info *fs_info, struct page *page)
2894{
2895	struct folio *folio = page_folio(page);
2896	struct extent_buffer *exists;
2897
2898	lockdep_assert_held(&page->mapping->i_private_lock);
2899
2900	/*
2901	 * For subpage case, we completely rely on radix tree to ensure we
2902	 * don't try to insert two ebs for the same bytenr.  So here we always
2903	 * return NULL and just continue.
2904	 */
2905	if (fs_info->nodesize < PAGE_SIZE)
2906		return NULL;
2907
2908	/* Page not yet attached to an extent buffer */
2909	if (!folio_test_private(folio))
2910		return NULL;
2911
2912	/*
2913	 * We could have already allocated an eb for this page and attached one
2914	 * so lets see if we can get a ref on the existing eb, and if we can we
2915	 * know it's good and we can just return that one, else we know we can
2916	 * just overwrite folio private.
2917	 */
2918	exists = folio_get_private(folio);
2919	if (atomic_inc_not_zero(&exists->refs))
2920		return exists;
2921
2922	WARN_ON(PageDirty(page));
2923	folio_detach_private(folio);
2924	return NULL;
2925}
2926
2927static int check_eb_alignment(struct btrfs_fs_info *fs_info, u64 start)
2928{
2929	if (!IS_ALIGNED(start, fs_info->sectorsize)) {
2930		btrfs_err(fs_info, "bad tree block start %llu", start);
2931		return -EINVAL;
2932	}
2933
2934	if (fs_info->nodesize < PAGE_SIZE &&
2935	    offset_in_page(start) + fs_info->nodesize > PAGE_SIZE) {
2936		btrfs_err(fs_info,
2937		"tree block crosses page boundary, start %llu nodesize %u",
2938			  start, fs_info->nodesize);
2939		return -EINVAL;
2940	}
2941	if (fs_info->nodesize >= PAGE_SIZE &&
2942	    !PAGE_ALIGNED(start)) {
2943		btrfs_err(fs_info,
2944		"tree block is not page aligned, start %llu nodesize %u",
2945			  start, fs_info->nodesize);
2946		return -EINVAL;
2947	}
2948	if (!IS_ALIGNED(start, fs_info->nodesize) &&
2949	    !test_and_set_bit(BTRFS_FS_UNALIGNED_TREE_BLOCK, &fs_info->flags)) {
2950		btrfs_warn(fs_info,
2951"tree block not nodesize aligned, start %llu nodesize %u, can be resolved by a full metadata balance",
2952			      start, fs_info->nodesize);
2953	}
2954	return 0;
2955}
2956
2957
2958/*
2959 * Return 0 if eb->folios[i] is attached to btree inode successfully.
2960 * Return >0 if there is already another extent buffer for the range,
2961 * and @found_eb_ret would be updated.
2962 * Return -EAGAIN if the filemap has an existing folio but with different size
2963 * than @eb.
2964 * The caller needs to free the existing folios and retry using the same order.
2965 */
2966static int attach_eb_folio_to_filemap(struct extent_buffer *eb, int i,
2967				      struct btrfs_subpage *prealloc,
2968				      struct extent_buffer **found_eb_ret)
2969{
2970
2971	struct btrfs_fs_info *fs_info = eb->fs_info;
2972	struct address_space *mapping = fs_info->btree_inode->i_mapping;
2973	const unsigned long index = eb->start >> PAGE_SHIFT;
2974	struct folio *existing_folio = NULL;
2975	int ret;
2976
2977	ASSERT(found_eb_ret);
2978
2979	/* Caller should ensure the folio exists. */
2980	ASSERT(eb->folios[i]);
2981
2982retry:
2983	ret = filemap_add_folio(mapping, eb->folios[i], index + i,
2984				GFP_NOFS | __GFP_NOFAIL);
2985	if (!ret)
2986		goto finish;
2987
2988	existing_folio = filemap_lock_folio(mapping, index + i);
2989	/* The page cache only exists for a very short time, just retry. */
2990	if (IS_ERR(existing_folio)) {
2991		existing_folio = NULL;
2992		goto retry;
2993	}
2994
2995	/* For now, we should only have single-page folios for btree inode. */
2996	ASSERT(folio_nr_pages(existing_folio) == 1);
2997
2998	if (folio_size(existing_folio) != eb->folio_size) {
2999		folio_unlock(existing_folio);
3000		folio_put(existing_folio);
3001		return -EAGAIN;
3002	}
3003
3004finish:
3005	spin_lock(&mapping->i_private_lock);
3006	if (existing_folio && fs_info->nodesize < PAGE_SIZE) {
3007		/* We're going to reuse the existing page, can drop our folio now. */
3008		__free_page(folio_page(eb->folios[i], 0));
3009		eb->folios[i] = existing_folio;
3010	} else if (existing_folio) {
3011		struct extent_buffer *existing_eb;
3012
3013		existing_eb = grab_extent_buffer(fs_info,
3014						 folio_page(existing_folio, 0));
3015		if (existing_eb) {
3016			/* The extent buffer still exists, we can use it directly. */
3017			*found_eb_ret = existing_eb;
3018			spin_unlock(&mapping->i_private_lock);
3019			folio_unlock(existing_folio);
3020			folio_put(existing_folio);
3021			return 1;
3022		}
3023		/* The extent buffer no longer exists, we can reuse the folio. */
3024		__free_page(folio_page(eb->folios[i], 0));
3025		eb->folios[i] = existing_folio;
3026	}
3027	eb->folio_size = folio_size(eb->folios[i]);
3028	eb->folio_shift = folio_shift(eb->folios[i]);
3029	/* Should not fail, as we have preallocated the memory. */
3030	ret = attach_extent_buffer_folio(eb, eb->folios[i], prealloc);
3031	ASSERT(!ret);
3032	/*
3033	 * To inform we have an extra eb under allocation, so that
3034	 * detach_extent_buffer_page() won't release the folio private when the
3035	 * eb hasn't been inserted into radix tree yet.
3036	 *
3037	 * The ref will be decreased when the eb releases the page, in
3038	 * detach_extent_buffer_page().  Thus needs no special handling in the
3039	 * error path.
3040	 */
3041	btrfs_folio_inc_eb_refs(fs_info, eb->folios[i]);
3042	spin_unlock(&mapping->i_private_lock);
3043	return 0;
3044}
3045
3046struct extent_buffer *alloc_extent_buffer(struct btrfs_fs_info *fs_info,
3047					  u64 start, u64 owner_root, int level)
3048{
3049	unsigned long len = fs_info->nodesize;
3050	int num_folios;
3051	int attached = 0;
 
3052	struct extent_buffer *eb;
3053	struct extent_buffer *existing_eb = NULL;
3054	struct btrfs_subpage *prealloc = NULL;
3055	u64 lockdep_owner = owner_root;
3056	bool page_contig = true;
3057	int uptodate = 1;
3058	int ret;
3059
3060	if (check_eb_alignment(fs_info, start))
3061		return ERR_PTR(-EINVAL);
3062
3063#if BITS_PER_LONG == 32
3064	if (start >= MAX_LFS_FILESIZE) {
3065		btrfs_err_rl(fs_info,
3066		"extent buffer %llu is beyond 32bit page cache limit", start);
3067		btrfs_err_32bit_limit(fs_info);
3068		return ERR_PTR(-EOVERFLOW);
3069	}
3070	if (start >= BTRFS_32BIT_EARLY_WARN_THRESHOLD)
3071		btrfs_warn_32bit_limit(fs_info);
3072#endif
3073
3074	eb = find_extent_buffer(fs_info, start);
3075	if (eb)
3076		return eb;
3077
3078	eb = __alloc_extent_buffer(fs_info, start, len);
3079	if (!eb)
3080		return ERR_PTR(-ENOMEM);
3081
3082	/*
3083	 * The reloc trees are just snapshots, so we need them to appear to be
3084	 * just like any other fs tree WRT lockdep.
3085	 */
3086	if (lockdep_owner == BTRFS_TREE_RELOC_OBJECTID)
3087		lockdep_owner = BTRFS_FS_TREE_OBJECTID;
3088
3089	btrfs_set_buffer_lockdep_class(lockdep_owner, eb, level);
3090
3091	/*
3092	 * Preallocate folio private for subpage case, so that we won't
3093	 * allocate memory with i_private_lock nor page lock hold.
3094	 *
3095	 * The memory will be freed by attach_extent_buffer_page() or freed
3096	 * manually if we exit earlier.
3097	 */
3098	if (fs_info->nodesize < PAGE_SIZE) {
3099		prealloc = btrfs_alloc_subpage(fs_info, BTRFS_SUBPAGE_METADATA);
3100		if (IS_ERR(prealloc)) {
3101			ret = PTR_ERR(prealloc);
3102			goto out;
3103		}
3104	}
3105
3106reallocate:
3107	/* Allocate all pages first. */
3108	ret = alloc_eb_folio_array(eb, true);
3109	if (ret < 0) {
3110		btrfs_free_subpage(prealloc);
3111		goto out;
3112	}
3113
3114	num_folios = num_extent_folios(eb);
3115	/* Attach all pages to the filemap. */
3116	for (int i = 0; i < num_folios; i++) {
3117		struct folio *folio;
3118
3119		ret = attach_eb_folio_to_filemap(eb, i, prealloc, &existing_eb);
3120		if (ret > 0) {
3121			ASSERT(existing_eb);
3122			goto out;
3123		}
3124
3125		/*
3126		 * TODO: Special handling for a corner case where the order of
3127		 * folios mismatch between the new eb and filemap.
3128		 *
3129		 * This happens when:
3130		 *
3131		 * - the new eb is using higher order folio
3132		 *
3133		 * - the filemap is still using 0-order folios for the range
3134		 *   This can happen at the previous eb allocation, and we don't
3135		 *   have higher order folio for the call.
3136		 *
3137		 * - the existing eb has already been freed
3138		 *
3139		 * In this case, we have to free the existing folios first, and
3140		 * re-allocate using the same order.
3141		 * Thankfully this is not going to happen yet, as we're still
3142		 * using 0-order folios.
3143		 */
3144		if (unlikely(ret == -EAGAIN)) {
3145			ASSERT(0);
3146			goto reallocate;
3147		}
3148		attached++;
3149
3150		/*
3151		 * Only after attach_eb_folio_to_filemap(), eb->folios[] is
3152		 * reliable, as we may choose to reuse the existing page cache
3153		 * and free the allocated page.
3154		 */
3155		folio = eb->folios[i];
3156		WARN_ON(btrfs_folio_test_dirty(fs_info, folio, eb->start, eb->len));
3157
3158		/*
3159		 * Check if the current page is physically contiguous with previous eb
3160		 * page.
3161		 * At this stage, either we allocated a large folio, thus @i
3162		 * would only be 0, or we fall back to per-page allocation.
3163		 */
3164		if (i && folio_page(eb->folios[i - 1], 0) + 1 != folio_page(folio, 0))
3165			page_contig = false;
 
 
 
 
 
 
 
 
 
 
3166
3167		if (!btrfs_folio_test_uptodate(fs_info, folio, eb->start, eb->len))
 
 
 
 
 
 
 
 
 
 
 
 
3168			uptodate = 0;
3169
3170		/*
3171		 * We can't unlock the pages just yet since the extent buffer
3172		 * hasn't been properly inserted in the radix tree, this
3173		 * opens a race with btree_release_folio which can free a page
3174		 * while we are still filling in all pages for the buffer and
3175		 * we could crash.
3176		 */
3177	}
3178	if (uptodate)
3179		set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3180	/* All pages are physically contiguous, can skip cross page handling. */
3181	if (page_contig)
3182		eb->addr = folio_address(eb->folios[0]) + offset_in_page(eb->start);
3183again:
3184	ret = radix_tree_preload(GFP_NOFS);
3185	if (ret)
3186		goto out;
3187
3188	spin_lock(&fs_info->buffer_lock);
3189	ret = radix_tree_insert(&fs_info->buffer_radix,
3190				start >> fs_info->sectorsize_bits, eb);
3191	spin_unlock(&fs_info->buffer_lock);
3192	radix_tree_preload_end();
3193	if (ret == -EEXIST) {
3194		ret = 0;
3195		existing_eb = find_extent_buffer(fs_info, start);
3196		if (existing_eb)
3197			goto out;
3198		else
3199			goto again;
3200	}
3201	/* add one reference for the tree */
3202	check_buffer_tree_ref(eb);
3203	set_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags);
3204
3205	/*
3206	 * Now it's safe to unlock the pages because any calls to
3207	 * btree_release_folio will correctly detect that a page belongs to a
3208	 * live buffer and won't free them prematurely.
3209	 */
3210	for (int i = 0; i < num_folios; i++)
3211		unlock_page(folio_page(eb->folios[i], 0));
 
 
 
 
 
 
 
 
 
3212	return eb;
3213
3214out:
3215	WARN_ON(!atomic_dec_and_test(&eb->refs));
3216
3217	/*
3218	 * Any attached folios need to be detached before we unlock them.  This
3219	 * is because when we're inserting our new folios into the mapping, and
3220	 * then attaching our eb to that folio.  If we fail to insert our folio
3221	 * we'll lookup the folio for that index, and grab that EB.  We do not
3222	 * want that to grab this eb, as we're getting ready to free it.  So we
3223	 * have to detach it first and then unlock it.
3224	 *
3225	 * We have to drop our reference and NULL it out here because in the
3226	 * subpage case detaching does a btrfs_folio_dec_eb_refs() for our eb.
3227	 * Below when we call btrfs_release_extent_buffer() we will call
3228	 * detach_extent_buffer_folio() on our remaining pages in the !subpage
3229	 * case.  If we left eb->folios[i] populated in the subpage case we'd
3230	 * double put our reference and be super sad.
3231	 */
3232	for (int i = 0; i < attached; i++) {
3233		ASSERT(eb->folios[i]);
3234		detach_extent_buffer_folio(eb, eb->folios[i]);
3235		unlock_page(folio_page(eb->folios[i], 0));
3236		folio_put(eb->folios[i]);
3237		eb->folios[i] = NULL;
3238	}
3239	/*
3240	 * Now all pages of that extent buffer is unmapped, set UNMAPPED flag,
3241	 * so it can be cleaned up without utilizing page->mapping.
3242	 */
3243	set_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3244
3245	btrfs_release_extent_buffer(eb);
3246	if (ret < 0)
3247		return ERR_PTR(ret);
3248	ASSERT(existing_eb);
3249	return existing_eb;
3250}
3251
3252static inline void btrfs_release_extent_buffer_rcu(struct rcu_head *head)
3253{
3254	struct extent_buffer *eb =
3255			container_of(head, struct extent_buffer, rcu_head);
3256
3257	__free_extent_buffer(eb);
3258}
3259
 
3260static int release_extent_buffer(struct extent_buffer *eb)
3261	__releases(&eb->refs_lock)
3262{
3263	lockdep_assert_held(&eb->refs_lock);
3264
3265	WARN_ON(atomic_read(&eb->refs) == 0);
3266	if (atomic_dec_and_test(&eb->refs)) {
3267		if (test_and_clear_bit(EXTENT_BUFFER_IN_TREE, &eb->bflags)) {
3268			struct btrfs_fs_info *fs_info = eb->fs_info;
3269
3270			spin_unlock(&eb->refs_lock);
3271
3272			spin_lock(&fs_info->buffer_lock);
3273			radix_tree_delete(&fs_info->buffer_radix,
3274					  eb->start >> fs_info->sectorsize_bits);
3275			spin_unlock(&fs_info->buffer_lock);
3276		} else {
3277			spin_unlock(&eb->refs_lock);
3278		}
3279
3280		btrfs_leak_debug_del_eb(eb);
3281		/* Should be safe to release our pages at this point */
3282		btrfs_release_extent_buffer_pages(eb);
3283#ifdef CONFIG_BTRFS_FS_RUN_SANITY_TESTS
3284		if (unlikely(test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags))) {
3285			__free_extent_buffer(eb);
3286			return 1;
3287		}
3288#endif
3289		call_rcu(&eb->rcu_head, btrfs_release_extent_buffer_rcu);
3290		return 1;
3291	}
3292	spin_unlock(&eb->refs_lock);
3293
3294	return 0;
3295}
3296
3297void free_extent_buffer(struct extent_buffer *eb)
3298{
3299	int refs;
 
3300	if (!eb)
3301		return;
3302
3303	refs = atomic_read(&eb->refs);
3304	while (1) {
3305		if ((!test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) && refs <= 3)
3306		    || (test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags) &&
3307			refs == 1))
3308			break;
3309		if (atomic_try_cmpxchg(&eb->refs, &refs, refs - 1))
 
3310			return;
3311	}
3312
3313	spin_lock(&eb->refs_lock);
3314	if (atomic_read(&eb->refs) == 2 &&
 
 
 
 
3315	    test_bit(EXTENT_BUFFER_STALE, &eb->bflags) &&
3316	    !extent_buffer_under_io(eb) &&
3317	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3318		atomic_dec(&eb->refs);
3319
3320	/*
3321	 * I know this is terrible, but it's temporary until we stop tracking
3322	 * the uptodate bits and such for the extent buffers.
3323	 */
3324	release_extent_buffer(eb);
3325}
3326
3327void free_extent_buffer_stale(struct extent_buffer *eb)
3328{
3329	if (!eb)
3330		return;
3331
3332	spin_lock(&eb->refs_lock);
3333	set_bit(EXTENT_BUFFER_STALE, &eb->bflags);
3334
3335	if (atomic_read(&eb->refs) == 2 && !extent_buffer_under_io(eb) &&
3336	    test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags))
3337		atomic_dec(&eb->refs);
3338	release_extent_buffer(eb);
3339}
3340
3341static void btree_clear_folio_dirty(struct folio *folio)
3342{
3343	ASSERT(folio_test_dirty(folio));
3344	ASSERT(folio_test_locked(folio));
3345	folio_clear_dirty_for_io(folio);
3346	xa_lock_irq(&folio->mapping->i_pages);
3347	if (!folio_test_dirty(folio))
3348		__xa_clear_mark(&folio->mapping->i_pages,
3349				folio_index(folio), PAGECACHE_TAG_DIRTY);
3350	xa_unlock_irq(&folio->mapping->i_pages);
3351}
3352
3353static void clear_subpage_extent_buffer_dirty(const struct extent_buffer *eb)
3354{
3355	struct btrfs_fs_info *fs_info = eb->fs_info;
3356	struct folio *folio = eb->folios[0];
3357	bool last;
3358
3359	/* btree_clear_folio_dirty() needs page locked. */
3360	folio_lock(folio);
3361	last = btrfs_subpage_clear_and_test_dirty(fs_info, folio, eb->start, eb->len);
3362	if (last)
3363		btree_clear_folio_dirty(folio);
3364	folio_unlock(folio);
3365	WARN_ON(atomic_read(&eb->refs) == 0);
3366}
3367
3368void btrfs_clear_buffer_dirty(struct btrfs_trans_handle *trans,
3369			      struct extent_buffer *eb)
3370{
3371	struct btrfs_fs_info *fs_info = eb->fs_info;
3372	int num_folios;
3373
3374	btrfs_assert_tree_write_locked(eb);
3375
3376	if (trans && btrfs_header_generation(eb) != trans->transid)
3377		return;
3378
3379	/*
3380	 * Instead of clearing the dirty flag off of the buffer, mark it as
3381	 * EXTENT_BUFFER_ZONED_ZEROOUT. This allows us to preserve
3382	 * write-ordering in zoned mode, without the need to later re-dirty
3383	 * the extent_buffer.
3384	 *
3385	 * The actual zeroout of the buffer will happen later in
3386	 * btree_csum_one_bio.
3387	 */
3388	if (btrfs_is_zoned(fs_info) && test_bit(EXTENT_BUFFER_DIRTY, &eb->bflags)) {
3389		set_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags);
3390		return;
3391	}
3392
3393	if (!test_and_clear_bit(EXTENT_BUFFER_DIRTY, &eb->bflags))
3394		return;
3395
3396	percpu_counter_add_batch(&fs_info->dirty_metadata_bytes, -eb->len,
3397				 fs_info->dirty_metadata_batch);
3398
3399	if (eb->fs_info->nodesize < PAGE_SIZE)
3400		return clear_subpage_extent_buffer_dirty(eb);
 
 
3401
3402	num_folios = num_extent_folios(eb);
3403	for (int i = 0; i < num_folios; i++) {
3404		struct folio *folio = eb->folios[i];
3405
3406		if (!folio_test_dirty(folio))
3407			continue;
3408		folio_lock(folio);
3409		btree_clear_folio_dirty(folio);
3410		folio_unlock(folio);
 
 
 
 
 
3411	}
3412	WARN_ON(atomic_read(&eb->refs) == 0);
3413}
3414
3415void set_extent_buffer_dirty(struct extent_buffer *eb)
3416{
3417	int num_folios;
3418	bool was_dirty;
 
3419
3420	check_buffer_tree_ref(eb);
3421
3422	was_dirty = test_and_set_bit(EXTENT_BUFFER_DIRTY, &eb->bflags);
3423
3424	num_folios = num_extent_folios(eb);
3425	WARN_ON(atomic_read(&eb->refs) == 0);
3426	WARN_ON(!test_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags));
3427	WARN_ON(test_bit(EXTENT_BUFFER_ZONED_ZEROOUT, &eb->bflags));
3428
3429	if (!was_dirty) {
3430		bool subpage = eb->fs_info->nodesize < PAGE_SIZE;
3431
3432		/*
3433		 * For subpage case, we can have other extent buffers in the
3434		 * same page, and in clear_subpage_extent_buffer_dirty() we
3435		 * have to clear page dirty without subpage lock held.
3436		 * This can cause race where our page gets dirty cleared after
3437		 * we just set it.
3438		 *
3439		 * Thankfully, clear_subpage_extent_buffer_dirty() has locked
3440		 * its page for other reasons, we can use page lock to prevent
3441		 * the above race.
3442		 */
3443		if (subpage)
3444			lock_page(folio_page(eb->folios[0], 0));
3445		for (int i = 0; i < num_folios; i++)
3446			btrfs_folio_set_dirty(eb->fs_info, eb->folios[i],
3447					      eb->start, eb->len);
3448		if (subpage)
3449			unlock_page(folio_page(eb->folios[0], 0));
3450		percpu_counter_add_batch(&eb->fs_info->dirty_metadata_bytes,
3451					 eb->len,
3452					 eb->fs_info->dirty_metadata_batch);
3453	}
3454#ifdef CONFIG_BTRFS_DEBUG
3455	for (int i = 0; i < num_folios; i++)
3456		ASSERT(folio_test_dirty(eb->folios[i]));
3457#endif
3458}
3459
3460void clear_extent_buffer_uptodate(struct extent_buffer *eb)
3461{
3462	struct btrfs_fs_info *fs_info = eb->fs_info;
3463	int num_folios = num_extent_folios(eb);
 
3464
3465	clear_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3466	for (int i = 0; i < num_folios; i++) {
3467		struct folio *folio = eb->folios[i];
3468
3469		if (!folio)
3470			continue;
3471
3472		/*
3473		 * This is special handling for metadata subpage, as regular
3474		 * btrfs_is_subpage() can not handle cloned/dummy metadata.
3475		 */
3476		if (fs_info->nodesize >= PAGE_SIZE)
3477			folio_clear_uptodate(folio);
3478		else
3479			btrfs_subpage_clear_uptodate(fs_info, folio,
3480						     eb->start, eb->len);
3481	}
3482}
3483
3484void set_extent_buffer_uptodate(struct extent_buffer *eb)
3485{
3486	struct btrfs_fs_info *fs_info = eb->fs_info;
3487	int num_folios = num_extent_folios(eb);
 
3488
3489	set_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags);
3490	for (int i = 0; i < num_folios; i++) {
3491		struct folio *folio = eb->folios[i];
3492
3493		/*
3494		 * This is special handling for metadata subpage, as regular
3495		 * btrfs_is_subpage() can not handle cloned/dummy metadata.
3496		 */
3497		if (fs_info->nodesize >= PAGE_SIZE)
3498			folio_mark_uptodate(folio);
3499		else
3500			btrfs_subpage_set_uptodate(fs_info, folio,
3501						   eb->start, eb->len);
3502	}
3503}
3504
3505static void clear_extent_buffer_reading(struct extent_buffer *eb)
3506{
3507	clear_bit(EXTENT_BUFFER_READING, &eb->bflags);
3508	smp_mb__after_atomic();
3509	wake_up_bit(&eb->bflags, EXTENT_BUFFER_READING);
3510}
3511
3512static void end_bbio_meta_read(struct btrfs_bio *bbio)
 
 
3513{
3514	struct extent_buffer *eb = bbio->private;
3515	struct btrfs_fs_info *fs_info = eb->fs_info;
3516	bool uptodate = !bbio->bio.bi_status;
3517	struct folio_iter fi;
3518	u32 bio_offset = 0;
3519
3520	/*
3521	 * If the extent buffer is marked UPTODATE before the read operation
3522	 * completes, other calls to read_extent_buffer_pages() will return
3523	 * early without waiting for the read to finish, causing data races.
3524	 */
3525	WARN_ON(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags));
3526
3527	eb->read_mirror = bbio->mirror_num;
3528
3529	if (uptodate &&
3530	    btrfs_validate_extent_buffer(eb, &bbio->parent_check) < 0)
3531		uptodate = false;
3532
3533	if (uptodate) {
3534		set_extent_buffer_uptodate(eb);
 
 
3535	} else {
3536		clear_extent_buffer_uptodate(eb);
3537		set_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
3538	}
3539
3540	bio_for_each_folio_all(fi, &bbio->bio) {
3541		struct folio *folio = fi.folio;
3542		u64 start = eb->start + bio_offset;
3543		u32 len = fi.length;
3544
3545		if (uptodate)
3546			btrfs_folio_set_uptodate(fs_info, folio, start, len);
3547		else
3548			btrfs_folio_clear_uptodate(fs_info, folio, start, len);
3549
3550		bio_offset += len;
 
 
 
3551	}
3552
3553	clear_extent_buffer_reading(eb);
3554	free_extent_buffer(eb);
3555
3556	bio_put(&bbio->bio);
3557}
3558
3559int read_extent_buffer_pages(struct extent_buffer *eb, int wait, int mirror_num,
3560			     const struct btrfs_tree_parent_check *check)
3561{
3562	struct btrfs_bio *bbio;
3563	bool ret;
3564
3565	if (test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
3566		return 0;
3567
3568	/*
3569	 * We could have had EXTENT_BUFFER_UPTODATE cleared by the write
3570	 * operation, which could potentially still be in flight.  In this case
3571	 * we simply want to return an error.
3572	 */
3573	if (unlikely(test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags)))
3574		return -EIO;
3575
3576	/* Someone else is already reading the buffer, just wait for it. */
3577	if (test_and_set_bit(EXTENT_BUFFER_READING, &eb->bflags))
3578		goto done;
3579
3580	/*
3581	 * Between the initial test_bit(EXTENT_BUFFER_UPTODATE) and the above
3582	 * test_and_set_bit(EXTENT_BUFFER_READING), someone else could have
3583	 * started and finished reading the same eb.  In this case, UPTODATE
3584	 * will now be set, and we shouldn't read it in again.
3585	 */
3586	if (unlikely(test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))) {
3587		clear_extent_buffer_reading(eb);
3588		return 0;
3589	}
3590
3591	clear_bit(EXTENT_BUFFER_READ_ERR, &eb->bflags);
3592	eb->read_mirror = 0;
3593	check_buffer_tree_ref(eb);
3594	atomic_inc(&eb->refs);
3595
3596	bbio = btrfs_bio_alloc(INLINE_EXTENT_BUFFER_PAGES,
3597			       REQ_OP_READ | REQ_META, eb->fs_info,
3598			       end_bbio_meta_read, eb);
3599	bbio->bio.bi_iter.bi_sector = eb->start >> SECTOR_SHIFT;
3600	bbio->inode = BTRFS_I(eb->fs_info->btree_inode);
3601	bbio->file_offset = eb->start;
3602	memcpy(&bbio->parent_check, check, sizeof(*check));
3603	if (eb->fs_info->nodesize < PAGE_SIZE) {
3604		ret = bio_add_folio(&bbio->bio, eb->folios[0], eb->len,
3605				    eb->start - folio_pos(eb->folios[0]));
3606		ASSERT(ret);
3607	} else {
3608		int num_folios = num_extent_folios(eb);
3609
3610		for (int i = 0; i < num_folios; i++) {
3611			struct folio *folio = eb->folios[i];
3612
3613			ret = bio_add_folio(&bbio->bio, folio, eb->folio_size, 0);
3614			ASSERT(ret);
3615		}
3616	}
3617	btrfs_submit_bbio(bbio, mirror_num);
3618
3619done:
3620	if (wait == WAIT_COMPLETE) {
3621		wait_on_bit_io(&eb->bflags, EXTENT_BUFFER_READING, TASK_UNINTERRUPTIBLE);
3622		if (!test_bit(EXTENT_BUFFER_UPTODATE, &eb->bflags))
3623			return -EIO;
3624	}
3625
3626	return 0;
3627}
3628
3629static bool report_eb_range(const struct extent_buffer *eb, unsigned long start,
3630			    unsigned long len)
3631{
3632	btrfs_warn(eb->fs_info,
3633		"access to eb bytenr %llu len %u out of range start %lu len %lu",
3634		eb->start, eb->len, start, len);
3635	WARN_ON(IS_ENABLED(CONFIG_BTRFS_DEBUG));
3636
3637	return true;
3638}
3639
3640/*
3641 * Check if the [start, start + len) range is valid before reading/writing
3642 * the eb.
3643 * NOTE: @start and @len are offset inside the eb, not logical address.
3644 *
3645 * Caller should not touch the dst/src memory if this function returns error.
3646 */
3647static inline int check_eb_range(const struct extent_buffer *eb,
3648				 unsigned long start, unsigned long len)
3649{
3650	unsigned long offset;
3651
3652	/* start, start + len should not go beyond eb->len nor overflow */
3653	if (unlikely(check_add_overflow(start, len, &offset) || offset > eb->len))
3654		return report_eb_range(eb, start, len);
3655
3656	return false;
 
 
 
 
 
 
 
 
3657}
3658
3659void read_extent_buffer(const struct extent_buffer *eb, void *dstv,
3660			unsigned long start, unsigned long len)
 
3661{
3662	const int unit_size = eb->folio_size;
3663	size_t cur;
3664	size_t offset;
 
 
3665	char *dst = (char *)dstv;
3666	unsigned long i = get_eb_folio_index(eb, start);
 
3667
3668	if (check_eb_range(eb, start, len)) {
3669		/*
3670		 * Invalid range hit, reset the memory, so callers won't get
3671		 * some random garbage for their uninitialized memory.
3672		 */
3673		memset(dstv, 0, len);
3674		return;
3675	}
3676
3677	if (eb->addr) {
3678		memcpy(dstv, eb->addr + start, len);
3679		return;
3680	}
3681
3682	offset = get_eb_offset_in_folio(eb, start);
3683
3684	while (len > 0) {
3685		char *kaddr;
3686
3687		cur = min(len, unit_size - offset);
3688		kaddr = folio_address(eb->folios[i]);
3689		memcpy(dst, kaddr + offset, cur);
3690
3691		dst += cur;
3692		len -= cur;
3693		offset = 0;
3694		i++;
3695	}
3696}
3697
3698int read_extent_buffer_to_user_nofault(const struct extent_buffer *eb,
3699				       void __user *dstv,
3700				       unsigned long start, unsigned long len)
3701{
3702	const int unit_size = eb->folio_size;
3703	size_t cur;
3704	size_t offset;
 
 
3705	char __user *dst = (char __user *)dstv;
3706	unsigned long i = get_eb_folio_index(eb, start);
 
3707	int ret = 0;
3708
3709	WARN_ON(start > eb->len);
3710	WARN_ON(start + len > eb->start + eb->len);
3711
3712	if (eb->addr) {
3713		if (copy_to_user_nofault(dstv, eb->addr + start, len))
3714			ret = -EFAULT;
3715		return ret;
3716	}
3717
3718	offset = get_eb_offset_in_folio(eb, start);
3719
3720	while (len > 0) {
3721		char *kaddr;
3722
3723		cur = min(len, unit_size - offset);
3724		kaddr = folio_address(eb->folios[i]);
3725		if (copy_to_user_nofault(dst, kaddr + offset, cur)) {
3726			ret = -EFAULT;
3727			break;
3728		}
3729
3730		dst += cur;
3731		len -= cur;
3732		offset = 0;
3733		i++;
3734	}
3735
3736	return ret;
3737}
3738
3739int memcmp_extent_buffer(const struct extent_buffer *eb, const void *ptrv,
3740			 unsigned long start, unsigned long len)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
3741{
3742	const int unit_size = eb->folio_size;
3743	size_t cur;
3744	size_t offset;
 
3745	char *kaddr;
3746	char *ptr = (char *)ptrv;
3747	unsigned long i = get_eb_folio_index(eb, start);
 
3748	int ret = 0;
3749
3750	if (check_eb_range(eb, start, len))
3751		return -EINVAL;
3752
3753	if (eb->addr)
3754		return memcmp(ptrv, eb->addr + start, len);
3755
3756	offset = get_eb_offset_in_folio(eb, start);
3757
3758	while (len > 0) {
3759		cur = min(len, unit_size - offset);
3760		kaddr = folio_address(eb->folios[i]);
 
 
 
3761		ret = memcmp(ptr, kaddr + offset, cur);
3762		if (ret)
3763			break;
3764
3765		ptr += cur;
3766		len -= cur;
3767		offset = 0;
3768		i++;
3769	}
3770	return ret;
3771}
3772
3773/*
3774 * Check that the extent buffer is uptodate.
3775 *
3776 * For regular sector size == PAGE_SIZE case, check if @page is uptodate.
3777 * For subpage case, check if the range covered by the eb has EXTENT_UPTODATE.
3778 */
3779static void assert_eb_folio_uptodate(const struct extent_buffer *eb, int i)
3780{
3781	struct btrfs_fs_info *fs_info = eb->fs_info;
3782	struct folio *folio = eb->folios[i];
3783
3784	ASSERT(folio);
3785
3786	/*
3787	 * If we are using the commit root we could potentially clear a page
3788	 * Uptodate while we're using the extent buffer that we've previously
3789	 * looked up.  We don't want to complain in this case, as the page was
3790	 * valid before, we just didn't write it out.  Instead we want to catch
3791	 * the case where we didn't actually read the block properly, which
3792	 * would have !PageUptodate and !EXTENT_BUFFER_WRITE_ERR.
3793	 */
3794	if (test_bit(EXTENT_BUFFER_WRITE_ERR, &eb->bflags))
3795		return;
3796
3797	if (fs_info->nodesize < PAGE_SIZE) {
3798		folio = eb->folios[0];
3799		ASSERT(i == 0);
3800		if (WARN_ON(!btrfs_subpage_test_uptodate(fs_info, folio,
3801							 eb->start, eb->len)))
3802			btrfs_subpage_dump_bitmap(fs_info, folio, eb->start, eb->len);
3803	} else {
3804		WARN_ON(!folio_test_uptodate(folio));
3805	}
3806}
3807
3808static void __write_extent_buffer(const struct extent_buffer *eb,
3809				  const void *srcv, unsigned long start,
3810				  unsigned long len, bool use_memmove)
3811{
3812	const int unit_size = eb->folio_size;
3813	size_t cur;
3814	size_t offset;
 
3815	char *kaddr;
3816	const char *src = (const char *)srcv;
3817	unsigned long i = get_eb_folio_index(eb, start);
3818	/* For unmapped (dummy) ebs, no need to check their uptodate status. */
3819	const bool check_uptodate = !test_bit(EXTENT_BUFFER_UNMAPPED, &eb->bflags);
3820
3821	if (check_eb_range(eb, start, len))
3822		return;
3823
3824	if (eb->addr) {
3825		if (use_memmove)
3826			memmove(eb->addr + start, srcv, len);
3827		else
3828			memcpy(eb->addr + start, srcv, len);
3829		return;
3830	}
3831
3832	offset = get_eb_offset_in_folio(eb, start);
3833
3834	while (len > 0) {
3835		if (check_uptodate)
3836			assert_eb_folio_uptodate(eb, i);
3837
3838		cur = min(len, unit_size - offset);
3839		kaddr = folio_address(eb->folios[i]);
3840		if (use_memmove)
3841			memmove(kaddr + offset, src, cur);
3842		else
3843			memcpy(kaddr + offset, src, cur);
3844
3845		src += cur;
3846		len -= cur;
3847		offset = 0;
3848		i++;
3849	}
3850}
3851
3852void write_extent_buffer(const struct extent_buffer *eb, const void *srcv,
3853			 unsigned long start, unsigned long len)
3854{
3855	return __write_extent_buffer(eb, srcv, start, len, false);
3856}
3857
3858static void memset_extent_buffer(const struct extent_buffer *eb, int c,
3859				 unsigned long start, unsigned long len)
3860{
3861	const int unit_size = eb->folio_size;
3862	unsigned long cur = start;
3863
3864	if (eb->addr) {
3865		memset(eb->addr + start, c, len);
3866		return;
3867	}
3868
3869	while (cur < start + len) {
3870		unsigned long index = get_eb_folio_index(eb, cur);
3871		unsigned int offset = get_eb_offset_in_folio(eb, cur);
3872		unsigned int cur_len = min(start + len - cur, unit_size - offset);
3873
3874		assert_eb_folio_uptodate(eb, index);
3875		memset(folio_address(eb->folios[index]) + offset, c, cur_len);
3876
3877		cur += cur_len;
3878	}
3879}
3880
3881void memzero_extent_buffer(const struct extent_buffer *eb, unsigned long start,
3882			   unsigned long len)
3883{
3884	if (check_eb_range(eb, start, len))
3885		return;
3886	return memset_extent_buffer(eb, 0, start, len);
3887}
 
 
3888
3889void copy_extent_buffer_full(const struct extent_buffer *dst,
3890			     const struct extent_buffer *src)
3891{
3892	const int unit_size = src->folio_size;
3893	unsigned long cur = 0;
3894
3895	ASSERT(dst->len == src->len);
3896
3897	while (cur < src->len) {
3898		unsigned long index = get_eb_folio_index(src, cur);
3899		unsigned long offset = get_eb_offset_in_folio(src, cur);
3900		unsigned long cur_len = min(src->len, unit_size - offset);
3901		void *addr = folio_address(src->folios[index]) + offset;
3902
3903		write_extent_buffer(dst, addr, cur, cur_len);
 
 
3904
3905		cur += cur_len;
 
 
3906	}
3907}
3908
3909void copy_extent_buffer(const struct extent_buffer *dst,
3910			const struct extent_buffer *src,
3911			unsigned long dst_offset, unsigned long src_offset,
3912			unsigned long len)
3913{
3914	const int unit_size = dst->folio_size;
3915	u64 dst_len = dst->len;
3916	size_t cur;
3917	size_t offset;
 
3918	char *kaddr;
3919	unsigned long i = get_eb_folio_index(dst, dst_offset);
3920
3921	if (check_eb_range(dst, dst_offset, len) ||
3922	    check_eb_range(src, src_offset, len))
3923		return;
3924
3925	WARN_ON(src->len != dst_len);
3926
3927	offset = get_eb_offset_in_folio(dst, dst_offset);
 
3928
3929	while (len > 0) {
3930		assert_eb_folio_uptodate(dst, i);
 
3931
3932		cur = min(len, (unsigned long)(unit_size - offset));
3933
3934		kaddr = folio_address(dst->folios[i]);
3935		read_extent_buffer(src, kaddr + offset, src_offset, cur);
3936
3937		src_offset += cur;
3938		len -= cur;
3939		offset = 0;
3940		i++;
3941	}
3942}
3943
3944/*
3945 * Calculate the folio and offset of the byte containing the given bit number.
3946 *
3947 * @eb:           the extent buffer
3948 * @start:        offset of the bitmap item in the extent buffer
3949 * @nr:           bit number
3950 * @folio_index:  return index of the folio in the extent buffer that contains
3951 *                the given bit number
3952 * @folio_offset: return offset into the folio given by folio_index
 
 
 
 
 
 
 
 
 
 
 
 
3953 *
3954 * This helper hides the ugliness of finding the byte in an extent buffer which
3955 * contains a given bit.
3956 */
3957static inline void eb_bitmap_offset(const struct extent_buffer *eb,
3958				    unsigned long start, unsigned long nr,
3959				    unsigned long *folio_index,
3960				    size_t *folio_offset)
3961{
 
3962	size_t byte_offset = BIT_BYTE(nr);
3963	size_t offset;
3964
3965	/*
3966	 * The byte we want is the offset of the extent buffer + the offset of
3967	 * the bitmap item in the extent buffer + the offset of the byte in the
3968	 * bitmap item.
3969	 */
3970	offset = start + offset_in_eb_folio(eb, eb->start) + byte_offset;
3971
3972	*folio_index = offset >> eb->folio_shift;
3973	*folio_offset = offset_in_eb_folio(eb, offset);
3974}
3975
3976/*
3977 * Determine whether a bit in a bitmap item is set.
3978 *
3979 * @eb:     the extent buffer
3980 * @start:  offset of the bitmap item in the extent buffer
3981 * @nr:     bit number to test
3982 */
3983int extent_buffer_test_bit(const struct extent_buffer *eb, unsigned long start,
3984			   unsigned long nr)
3985{
 
 
3986	unsigned long i;
3987	size_t offset;
3988	u8 *kaddr;
3989
3990	eb_bitmap_offset(eb, start, nr, &i, &offset);
3991	assert_eb_folio_uptodate(eb, i);
3992	kaddr = folio_address(eb->folios[i]);
 
3993	return 1U & (kaddr[offset] >> (nr & (BITS_PER_BYTE - 1)));
3994}
3995
3996static u8 *extent_buffer_get_byte(const struct extent_buffer *eb, unsigned long bytenr)
3997{
3998	unsigned long index = get_eb_folio_index(eb, bytenr);
3999
4000	if (check_eb_range(eb, bytenr, 1))
4001		return NULL;
4002	return folio_address(eb->folios[index]) + get_eb_offset_in_folio(eb, bytenr);
4003}
4004
4005/*
4006 * Set an area of a bitmap to 1.
4007 *
4008 * @eb:     the extent buffer
4009 * @start:  offset of the bitmap item in the extent buffer
4010 * @pos:    bit number of the first bit
4011 * @len:    number of bits to set
4012 */
4013void extent_buffer_bitmap_set(const struct extent_buffer *eb, unsigned long start,
4014			      unsigned long pos, unsigned long len)
4015{
4016	unsigned int first_byte = start + BIT_BYTE(pos);
4017	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4018	const bool same_byte = (first_byte == last_byte);
4019	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4020	u8 *kaddr;
4021
4022	if (same_byte)
4023		mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4024
4025	/* Handle the first byte. */
4026	kaddr = extent_buffer_get_byte(eb, first_byte);
4027	*kaddr |= mask;
4028	if (same_byte)
4029		return;
4030
4031	/* Handle the byte aligned part. */
4032	ASSERT(first_byte + 1 <= last_byte);
4033	memset_extent_buffer(eb, 0xff, first_byte + 1, last_byte - first_byte - 1);
4034
4035	/* Handle the last byte. */
4036	kaddr = extent_buffer_get_byte(eb, last_byte);
4037	*kaddr |= BITMAP_LAST_BYTE_MASK(pos + len);
 
 
 
 
 
 
 
4038}
4039
4040
4041/*
4042 * Clear an area of a bitmap.
4043 *
4044 * @eb:     the extent buffer
4045 * @start:  offset of the bitmap item in the extent buffer
4046 * @pos:    bit number of the first bit
4047 * @len:    number of bits to clear
4048 */
4049void extent_buffer_bitmap_clear(const struct extent_buffer *eb,
4050				unsigned long start, unsigned long pos,
4051				unsigned long len)
4052{
4053	unsigned int first_byte = start + BIT_BYTE(pos);
4054	unsigned int last_byte = start + BIT_BYTE(pos + len - 1);
4055	const bool same_byte = (first_byte == last_byte);
4056	u8 mask = BITMAP_FIRST_BYTE_MASK(pos);
4057	u8 *kaddr;
4058
4059	if (same_byte)
4060		mask &= BITMAP_LAST_BYTE_MASK(pos + len);
4061
4062	/* Handle the first byte. */
4063	kaddr = extent_buffer_get_byte(eb, first_byte);
4064	*kaddr &= ~mask;
4065	if (same_byte)
4066		return;
4067
4068	/* Handle the byte aligned part. */
4069	ASSERT(first_byte + 1 <= last_byte);
4070	memset_extent_buffer(eb, 0, first_byte + 1, last_byte - first_byte - 1);
4071
4072	/* Handle the last byte. */
4073	kaddr = extent_buffer_get_byte(eb, last_byte);
4074	*kaddr &= ~BITMAP_LAST_BYTE_MASK(pos + len);
 
 
 
 
 
4075}
4076
4077static inline bool areas_overlap(unsigned long src, unsigned long dst, unsigned long len)
4078{
4079	unsigned long distance = (src > dst) ? src - dst : dst - src;
4080	return distance < len;
4081}
4082
4083void memcpy_extent_buffer(const struct extent_buffer *dst,
4084			  unsigned long dst_offset, unsigned long src_offset,
4085			  unsigned long len)
4086{
4087	const int unit_size = dst->folio_size;
4088	unsigned long cur_off = 0;
 
4089
4090	if (check_eb_range(dst, dst_offset, len) ||
4091	    check_eb_range(dst, src_offset, len))
4092		return;
 
 
 
 
4093
4094	if (dst->addr) {
4095		const bool use_memmove = areas_overlap(src_offset, dst_offset, len);
 
 
 
4096
4097		if (use_memmove)
4098			memmove(dst->addr + dst_offset, dst->addr + src_offset, len);
4099		else
4100			memcpy(dst->addr + dst_offset, dst->addr + src_offset, len);
4101		return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
4102	}
4103
4104	while (cur_off < len) {
4105		unsigned long cur_src = cur_off + src_offset;
4106		unsigned long folio_index = get_eb_folio_index(dst, cur_src);
4107		unsigned long folio_off = get_eb_offset_in_folio(dst, cur_src);
4108		unsigned long cur_len = min(src_offset + len - cur_src,
4109					    unit_size - folio_off);
4110		void *src_addr = folio_address(dst->folios[folio_index]) + folio_off;
4111		const bool use_memmove = areas_overlap(src_offset + cur_off,
4112						       dst_offset + cur_off, cur_len);
4113
4114		__write_extent_buffer(dst, src_addr, dst_offset + cur_off, cur_len,
4115				      use_memmove);
4116		cur_off += cur_len;
 
 
 
 
 
 
 
4117	}
4118}
4119
4120void memmove_extent_buffer(const struct extent_buffer *dst,
4121			   unsigned long dst_offset, unsigned long src_offset,
4122			   unsigned long len)
4123{
 
 
 
4124	unsigned long dst_end = dst_offset + len - 1;
4125	unsigned long src_end = src_offset + len - 1;
4126
4127	if (check_eb_range(dst, dst_offset, len) ||
4128	    check_eb_range(dst, src_offset, len))
4129		return;
4130
 
 
 
 
 
 
 
 
 
4131	if (dst_offset < src_offset) {
4132		memcpy_extent_buffer(dst, dst_offset, src_offset, len);
4133		return;
4134	}
4135
4136	if (dst->addr) {
4137		memmove(dst->addr + dst_offset, dst->addr + src_offset, len);
4138		return;
4139	}
4140
4141	while (len > 0) {
4142		unsigned long src_i;
4143		size_t cur;
4144		size_t dst_off_in_folio;
4145		size_t src_off_in_folio;
4146		void *src_addr;
4147		bool use_memmove;
4148
4149		src_i = get_eb_folio_index(dst, src_end);
4150
4151		dst_off_in_folio = get_eb_offset_in_folio(dst, dst_end);
4152		src_off_in_folio = get_eb_offset_in_folio(dst, src_end);
4153
4154		cur = min_t(unsigned long, len, src_off_in_folio + 1);
4155		cur = min(cur, dst_off_in_folio + 1);
4156
4157		src_addr = folio_address(dst->folios[src_i]) + src_off_in_folio -
4158					 cur + 1;
4159		use_memmove = areas_overlap(src_end - cur + 1, dst_end - cur + 1,
4160					    cur);
4161
4162		__write_extent_buffer(dst, src_addr, dst_end - cur + 1, cur,
4163				      use_memmove);
 
 
 
 
 
 
 
 
4164
4165		dst_end -= cur;
4166		src_end -= cur;
4167		len -= cur;
4168	}
4169}
4170
4171#define GANG_LOOKUP_SIZE	16
4172static struct extent_buffer *get_next_extent_buffer(
4173		const struct btrfs_fs_info *fs_info, struct folio *folio, u64 bytenr)
4174{
4175	struct extent_buffer *gang[GANG_LOOKUP_SIZE];
4176	struct extent_buffer *found = NULL;
4177	u64 folio_start = folio_pos(folio);
4178	u64 cur = folio_start;
4179
4180	ASSERT(in_range(bytenr, folio_start, PAGE_SIZE));
4181	lockdep_assert_held(&fs_info->buffer_lock);
4182
4183	while (cur < folio_start + PAGE_SIZE) {
4184		int ret;
4185		int i;
4186
4187		ret = radix_tree_gang_lookup(&fs_info->buffer_radix,
4188				(void **)gang, cur >> fs_info->sectorsize_bits,
4189				min_t(unsigned int, GANG_LOOKUP_SIZE,
4190				      PAGE_SIZE / fs_info->nodesize));
4191		if (ret == 0)
4192			goto out;
4193		for (i = 0; i < ret; i++) {
4194			/* Already beyond page end */
4195			if (gang[i]->start >= folio_start + PAGE_SIZE)
4196				goto out;
4197			/* Found one */
4198			if (gang[i]->start >= bytenr) {
4199				found = gang[i];
4200				goto out;
4201			}
4202		}
4203		cur = gang[ret - 1]->start + gang[ret - 1]->len;
4204	}
4205out:
4206	return found;
4207}
4208
4209static int try_release_subpage_extent_buffer(struct folio *folio)
4210{
4211	struct btrfs_fs_info *fs_info = folio_to_fs_info(folio);
4212	u64 cur = folio_pos(folio);
4213	const u64 end = cur + PAGE_SIZE;
4214	int ret;
4215
4216	while (cur < end) {
4217		struct extent_buffer *eb = NULL;
4218
4219		/*
4220		 * Unlike try_release_extent_buffer() which uses folio private
4221		 * to grab buffer, for subpage case we rely on radix tree, thus
4222		 * we need to ensure radix tree consistency.
4223		 *
4224		 * We also want an atomic snapshot of the radix tree, thus go
4225		 * with spinlock rather than RCU.
4226		 */
4227		spin_lock(&fs_info->buffer_lock);
4228		eb = get_next_extent_buffer(fs_info, folio, cur);
4229		if (!eb) {
4230			/* No more eb in the page range after or at cur */
4231			spin_unlock(&fs_info->buffer_lock);
4232			break;
4233		}
4234		cur = eb->start + eb->len;
4235
4236		/*
4237		 * The same as try_release_extent_buffer(), to ensure the eb
4238		 * won't disappear out from under us.
4239		 */
4240		spin_lock(&eb->refs_lock);
4241		if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4242			spin_unlock(&eb->refs_lock);
4243			spin_unlock(&fs_info->buffer_lock);
4244			break;
4245		}
4246		spin_unlock(&fs_info->buffer_lock);
4247
4248		/*
4249		 * If tree ref isn't set then we know the ref on this eb is a
4250		 * real ref, so just return, this eb will likely be freed soon
4251		 * anyway.
4252		 */
4253		if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4254			spin_unlock(&eb->refs_lock);
4255			break;
4256		}
4257
4258		/*
4259		 * Here we don't care about the return value, we will always
4260		 * check the folio private at the end.  And
4261		 * release_extent_buffer() will release the refs_lock.
4262		 */
4263		release_extent_buffer(eb);
4264	}
4265	/*
4266	 * Finally to check if we have cleared folio private, as if we have
4267	 * released all ebs in the page, the folio private should be cleared now.
4268	 */
4269	spin_lock(&folio->mapping->i_private_lock);
4270	if (!folio_test_private(folio))
4271		ret = 1;
4272	else
4273		ret = 0;
4274	spin_unlock(&folio->mapping->i_private_lock);
4275	return ret;
4276
4277}
4278
4279int try_release_extent_buffer(struct folio *folio)
4280{
4281	struct extent_buffer *eb;
4282
4283	if (folio_to_fs_info(folio)->nodesize < PAGE_SIZE)
4284		return try_release_subpage_extent_buffer(folio);
4285
4286	/*
4287	 * We need to make sure nobody is changing folio private, as we rely on
4288	 * folio private as the pointer to extent buffer.
4289	 */
4290	spin_lock(&folio->mapping->i_private_lock);
4291	if (!folio_test_private(folio)) {
4292		spin_unlock(&folio->mapping->i_private_lock);
4293		return 1;
4294	}
4295
4296	eb = folio_get_private(folio);
4297	BUG_ON(!eb);
4298
4299	/*
4300	 * This is a little awful but should be ok, we need to make sure that
4301	 * the eb doesn't disappear out from under us while we're looking at
4302	 * this page.
4303	 */
4304	spin_lock(&eb->refs_lock);
4305	if (atomic_read(&eb->refs) != 1 || extent_buffer_under_io(eb)) {
4306		spin_unlock(&eb->refs_lock);
4307		spin_unlock(&folio->mapping->i_private_lock);
4308		return 0;
4309	}
4310	spin_unlock(&folio->mapping->i_private_lock);
4311
4312	/*
4313	 * If tree ref isn't set then we know the ref on this eb is a real ref,
4314	 * so just return, this page will likely be freed soon anyway.
4315	 */
4316	if (!test_and_clear_bit(EXTENT_BUFFER_TREE_REF, &eb->bflags)) {
4317		spin_unlock(&eb->refs_lock);
4318		return 0;
4319	}
4320
4321	return release_extent_buffer(eb);
4322}
4323
4324/*
4325 * Attempt to readahead a child block.
4326 *
4327 * @fs_info:	the fs_info
4328 * @bytenr:	bytenr to read
4329 * @owner_root: objectid of the root that owns this eb
4330 * @gen:	generation for the uptodate check, can be 0
4331 * @level:	level for the eb
4332 *
4333 * Attempt to readahead a tree block at @bytenr.  If @gen is 0 then we do a
4334 * normal uptodate check of the eb, without checking the generation.  If we have
4335 * to read the block we will not block on anything.
4336 */
4337void btrfs_readahead_tree_block(struct btrfs_fs_info *fs_info,
4338				u64 bytenr, u64 owner_root, u64 gen, int level)
4339{
4340	struct btrfs_tree_parent_check check = {
4341		.level = level,
4342		.transid = gen
4343	};
4344	struct extent_buffer *eb;
4345	int ret;
4346
4347	eb = btrfs_find_create_tree_block(fs_info, bytenr, owner_root, level);
4348	if (IS_ERR(eb))
4349		return;
4350
4351	if (btrfs_buffer_uptodate(eb, gen, 1)) {
4352		free_extent_buffer(eb);
4353		return;
4354	}
4355
4356	ret = read_extent_buffer_pages(eb, WAIT_NONE, 0, &check);
4357	if (ret < 0)
4358		free_extent_buffer_stale(eb);
4359	else
4360		free_extent_buffer(eb);
4361}
4362
4363/*
4364 * Readahead a node's child block.
4365 *
4366 * @node:	parent node we're reading from
4367 * @slot:	slot in the parent node for the child we want to read
4368 *
4369 * A helper for btrfs_readahead_tree_block, we simply read the bytenr pointed at
4370 * the slot in the node provided.
4371 */
4372void btrfs_readahead_node_child(struct extent_buffer *node, int slot)
4373{
4374	btrfs_readahead_tree_block(node->fs_info,
4375				   btrfs_node_blockptr(node, slot),
4376				   btrfs_header_owner(node),
4377				   btrfs_node_ptr_generation(node, slot),
4378				   btrfs_header_level(node) - 1);
4379}