Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0
2
3#include <linux/slab.h>
4#include "ctree.h"
5#include "subpage.h"
6#include "btrfs_inode.h"
7
8/*
9 * Subpage (sectorsize < PAGE_SIZE) support overview:
10 *
11 * Limitations:
12 *
13 * - Only support 64K page size for now
14 * This is to make metadata handling easier, as 64K page would ensure
15 * all nodesize would fit inside one page, thus we don't need to handle
16 * cases where a tree block crosses several pages.
17 *
18 * - Only metadata read-write for now
19 * The data read-write part is in development.
20 *
21 * - Metadata can't cross 64K page boundary
22 * btrfs-progs and kernel have done that for a while, thus only ancient
23 * filesystems could have such problem. For such case, do a graceful
24 * rejection.
25 *
26 * Special behavior:
27 *
28 * - Metadata
29 * Metadata read is fully supported.
30 * Meaning when reading one tree block will only trigger the read for the
31 * needed range, other unrelated range in the same page will not be touched.
32 *
33 * Metadata write support is partial.
34 * The writeback is still for the full page, but we will only submit
35 * the dirty extent buffers in the page.
36 *
37 * This means, if we have a metadata page like this:
38 *
39 * Page offset
40 * 0 16K 32K 48K 64K
41 * |/////////| |///////////|
42 * \- Tree block A \- Tree block B
43 *
44 * Even if we just want to writeback tree block A, we will also writeback
45 * tree block B if it's also dirty.
46 *
47 * This may cause extra metadata writeback which results more COW.
48 *
49 * Implementation:
50 *
51 * - Common
52 * Both metadata and data will use a new structure, btrfs_subpage, to
53 * record the status of each sector inside a page. This provides the extra
54 * granularity needed.
55 *
56 * - Metadata
57 * Since we have multiple tree blocks inside one page, we can't rely on page
58 * locking anymore, or we will have greatly reduced concurrency or even
59 * deadlocks (hold one tree lock while trying to lock another tree lock in
60 * the same page).
61 *
62 * Thus for metadata locking, subpage support relies on io_tree locking only.
63 * This means a slightly higher tree locking latency.
64 */
65
66int btrfs_attach_subpage(const struct btrfs_fs_info *fs_info,
67 struct page *page, enum btrfs_subpage_type type)
68{
69 struct btrfs_subpage *subpage = NULL;
70 int ret;
71
72 /*
73 * We have cases like a dummy extent buffer page, which is not mappped
74 * and doesn't need to be locked.
75 */
76 if (page->mapping)
77 ASSERT(PageLocked(page));
78 /* Either not subpage, or the page already has private attached */
79 if (fs_info->sectorsize == PAGE_SIZE || PagePrivate(page))
80 return 0;
81
82 ret = btrfs_alloc_subpage(fs_info, &subpage, type);
83 if (ret < 0)
84 return ret;
85 attach_page_private(page, subpage);
86 return 0;
87}
88
89void btrfs_detach_subpage(const struct btrfs_fs_info *fs_info,
90 struct page *page)
91{
92 struct btrfs_subpage *subpage;
93
94 /* Either not subpage, or already detached */
95 if (fs_info->sectorsize == PAGE_SIZE || !PagePrivate(page))
96 return;
97
98 subpage = (struct btrfs_subpage *)detach_page_private(page);
99 ASSERT(subpage);
100 btrfs_free_subpage(subpage);
101}
102
103int btrfs_alloc_subpage(const struct btrfs_fs_info *fs_info,
104 struct btrfs_subpage **ret,
105 enum btrfs_subpage_type type)
106{
107 if (fs_info->sectorsize == PAGE_SIZE)
108 return 0;
109
110 *ret = kzalloc(sizeof(struct btrfs_subpage), GFP_NOFS);
111 if (!*ret)
112 return -ENOMEM;
113 spin_lock_init(&(*ret)->lock);
114 if (type == BTRFS_SUBPAGE_METADATA) {
115 atomic_set(&(*ret)->eb_refs, 0);
116 } else {
117 atomic_set(&(*ret)->readers, 0);
118 atomic_set(&(*ret)->writers, 0);
119 }
120 return 0;
121}
122
123void btrfs_free_subpage(struct btrfs_subpage *subpage)
124{
125 kfree(subpage);
126}
127
128/*
129 * Increase the eb_refs of current subpage.
130 *
131 * This is important for eb allocation, to prevent race with last eb freeing
132 * of the same page.
133 * With the eb_refs increased before the eb inserted into radix tree,
134 * detach_extent_buffer_page() won't detach the page private while we're still
135 * allocating the extent buffer.
136 */
137void btrfs_page_inc_eb_refs(const struct btrfs_fs_info *fs_info,
138 struct page *page)
139{
140 struct btrfs_subpage *subpage;
141
142 if (fs_info->sectorsize == PAGE_SIZE)
143 return;
144
145 ASSERT(PagePrivate(page) && page->mapping);
146 lockdep_assert_held(&page->mapping->private_lock);
147
148 subpage = (struct btrfs_subpage *)page->private;
149 atomic_inc(&subpage->eb_refs);
150}
151
152void btrfs_page_dec_eb_refs(const struct btrfs_fs_info *fs_info,
153 struct page *page)
154{
155 struct btrfs_subpage *subpage;
156
157 if (fs_info->sectorsize == PAGE_SIZE)
158 return;
159
160 ASSERT(PagePrivate(page) && page->mapping);
161 lockdep_assert_held(&page->mapping->private_lock);
162
163 subpage = (struct btrfs_subpage *)page->private;
164 ASSERT(atomic_read(&subpage->eb_refs));
165 atomic_dec(&subpage->eb_refs);
166}
167
168static void btrfs_subpage_assert(const struct btrfs_fs_info *fs_info,
169 struct page *page, u64 start, u32 len)
170{
171 /* Basic checks */
172 ASSERT(PagePrivate(page) && page->private);
173 ASSERT(IS_ALIGNED(start, fs_info->sectorsize) &&
174 IS_ALIGNED(len, fs_info->sectorsize));
175 /*
176 * The range check only works for mapped page, we can still have
177 * unmapped page like dummy extent buffer pages.
178 */
179 if (page->mapping)
180 ASSERT(page_offset(page) <= start &&
181 start + len <= page_offset(page) + PAGE_SIZE);
182}
183
184void btrfs_subpage_start_reader(const struct btrfs_fs_info *fs_info,
185 struct page *page, u64 start, u32 len)
186{
187 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
188 const int nbits = len >> fs_info->sectorsize_bits;
189
190 btrfs_subpage_assert(fs_info, page, start, len);
191
192 atomic_add(nbits, &subpage->readers);
193}
194
195void btrfs_subpage_end_reader(const struct btrfs_fs_info *fs_info,
196 struct page *page, u64 start, u32 len)
197{
198 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
199 const int nbits = len >> fs_info->sectorsize_bits;
200 bool is_data;
201 bool last;
202
203 btrfs_subpage_assert(fs_info, page, start, len);
204 is_data = is_data_inode(page->mapping->host);
205 ASSERT(atomic_read(&subpage->readers) >= nbits);
206 last = atomic_sub_and_test(nbits, &subpage->readers);
207
208 /*
209 * For data we need to unlock the page if the last read has finished.
210 *
211 * And please don't replace @last with atomic_sub_and_test() call
212 * inside if () condition.
213 * As we want the atomic_sub_and_test() to be always executed.
214 */
215 if (is_data && last)
216 unlock_page(page);
217}
218
219static void btrfs_subpage_clamp_range(struct page *page, u64 *start, u32 *len)
220{
221 u64 orig_start = *start;
222 u32 orig_len = *len;
223
224 *start = max_t(u64, page_offset(page), orig_start);
225 *len = min_t(u64, page_offset(page) + PAGE_SIZE,
226 orig_start + orig_len) - *start;
227}
228
229void btrfs_subpage_start_writer(const struct btrfs_fs_info *fs_info,
230 struct page *page, u64 start, u32 len)
231{
232 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
233 const int nbits = (len >> fs_info->sectorsize_bits);
234 int ret;
235
236 btrfs_subpage_assert(fs_info, page, start, len);
237
238 ASSERT(atomic_read(&subpage->readers) == 0);
239 ret = atomic_add_return(nbits, &subpage->writers);
240 ASSERT(ret == nbits);
241}
242
243bool btrfs_subpage_end_and_test_writer(const struct btrfs_fs_info *fs_info,
244 struct page *page, u64 start, u32 len)
245{
246 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
247 const int nbits = (len >> fs_info->sectorsize_bits);
248
249 btrfs_subpage_assert(fs_info, page, start, len);
250
251 ASSERT(atomic_read(&subpage->writers) >= nbits);
252 return atomic_sub_and_test(nbits, &subpage->writers);
253}
254
255/*
256 * Lock a page for delalloc page writeback.
257 *
258 * Return -EAGAIN if the page is not properly initialized.
259 * Return 0 with the page locked, and writer counter updated.
260 *
261 * Even with 0 returned, the page still need extra check to make sure
262 * it's really the correct page, as the caller is using
263 * find_get_pages_contig(), which can race with page invalidating.
264 */
265int btrfs_page_start_writer_lock(const struct btrfs_fs_info *fs_info,
266 struct page *page, u64 start, u32 len)
267{
268 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) {
269 lock_page(page);
270 return 0;
271 }
272 lock_page(page);
273 if (!PagePrivate(page) || !page->private) {
274 unlock_page(page);
275 return -EAGAIN;
276 }
277 btrfs_subpage_clamp_range(page, &start, &len);
278 btrfs_subpage_start_writer(fs_info, page, start, len);
279 return 0;
280}
281
282void btrfs_page_end_writer_lock(const struct btrfs_fs_info *fs_info,
283 struct page *page, u64 start, u32 len)
284{
285 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE)
286 return unlock_page(page);
287 btrfs_subpage_clamp_range(page, &start, &len);
288 if (btrfs_subpage_end_and_test_writer(fs_info, page, start, len))
289 unlock_page(page);
290}
291
292/*
293 * Convert the [start, start + len) range into a u16 bitmap
294 *
295 * For example: if start == page_offset() + 16K, len = 16K, we get 0x00f0.
296 */
297static u16 btrfs_subpage_calc_bitmap(const struct btrfs_fs_info *fs_info,
298 struct page *page, u64 start, u32 len)
299{
300 const int bit_start = offset_in_page(start) >> fs_info->sectorsize_bits;
301 const int nbits = len >> fs_info->sectorsize_bits;
302
303 btrfs_subpage_assert(fs_info, page, start, len);
304
305 /*
306 * Here nbits can be 16, thus can go beyond u16 range. We make the
307 * first left shift to be calculate in unsigned long (at least u32),
308 * then truncate the result to u16.
309 */
310 return (u16)(((1UL << nbits) - 1) << bit_start);
311}
312
313void btrfs_subpage_set_uptodate(const struct btrfs_fs_info *fs_info,
314 struct page *page, u64 start, u32 len)
315{
316 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
317 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
318 unsigned long flags;
319
320 spin_lock_irqsave(&subpage->lock, flags);
321 subpage->uptodate_bitmap |= tmp;
322 if (subpage->uptodate_bitmap == U16_MAX)
323 SetPageUptodate(page);
324 spin_unlock_irqrestore(&subpage->lock, flags);
325}
326
327void btrfs_subpage_clear_uptodate(const struct btrfs_fs_info *fs_info,
328 struct page *page, u64 start, u32 len)
329{
330 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
331 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
332 unsigned long flags;
333
334 spin_lock_irqsave(&subpage->lock, flags);
335 subpage->uptodate_bitmap &= ~tmp;
336 ClearPageUptodate(page);
337 spin_unlock_irqrestore(&subpage->lock, flags);
338}
339
340void btrfs_subpage_set_error(const struct btrfs_fs_info *fs_info,
341 struct page *page, u64 start, u32 len)
342{
343 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
344 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
345 unsigned long flags;
346
347 spin_lock_irqsave(&subpage->lock, flags);
348 subpage->error_bitmap |= tmp;
349 SetPageError(page);
350 spin_unlock_irqrestore(&subpage->lock, flags);
351}
352
353void btrfs_subpage_clear_error(const struct btrfs_fs_info *fs_info,
354 struct page *page, u64 start, u32 len)
355{
356 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
357 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
358 unsigned long flags;
359
360 spin_lock_irqsave(&subpage->lock, flags);
361 subpage->error_bitmap &= ~tmp;
362 if (subpage->error_bitmap == 0)
363 ClearPageError(page);
364 spin_unlock_irqrestore(&subpage->lock, flags);
365}
366
367void btrfs_subpage_set_dirty(const struct btrfs_fs_info *fs_info,
368 struct page *page, u64 start, u32 len)
369{
370 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
371 u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
372 unsigned long flags;
373
374 spin_lock_irqsave(&subpage->lock, flags);
375 subpage->dirty_bitmap |= tmp;
376 spin_unlock_irqrestore(&subpage->lock, flags);
377 set_page_dirty(page);
378}
379
380/*
381 * Extra clear_and_test function for subpage dirty bitmap.
382 *
383 * Return true if we're the last bits in the dirty_bitmap and clear the
384 * dirty_bitmap.
385 * Return false otherwise.
386 *
387 * NOTE: Callers should manually clear page dirty for true case, as we have
388 * extra handling for tree blocks.
389 */
390bool btrfs_subpage_clear_and_test_dirty(const struct btrfs_fs_info *fs_info,
391 struct page *page, u64 start, u32 len)
392{
393 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
394 u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
395 unsigned long flags;
396 bool last = false;
397
398 spin_lock_irqsave(&subpage->lock, flags);
399 subpage->dirty_bitmap &= ~tmp;
400 if (subpage->dirty_bitmap == 0)
401 last = true;
402 spin_unlock_irqrestore(&subpage->lock, flags);
403 return last;
404}
405
406void btrfs_subpage_clear_dirty(const struct btrfs_fs_info *fs_info,
407 struct page *page, u64 start, u32 len)
408{
409 bool last;
410
411 last = btrfs_subpage_clear_and_test_dirty(fs_info, page, start, len);
412 if (last)
413 clear_page_dirty_for_io(page);
414}
415
416void btrfs_subpage_set_writeback(const struct btrfs_fs_info *fs_info,
417 struct page *page, u64 start, u32 len)
418{
419 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
420 u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
421 unsigned long flags;
422
423 spin_lock_irqsave(&subpage->lock, flags);
424 subpage->writeback_bitmap |= tmp;
425 set_page_writeback(page);
426 spin_unlock_irqrestore(&subpage->lock, flags);
427}
428
429void btrfs_subpage_clear_writeback(const struct btrfs_fs_info *fs_info,
430 struct page *page, u64 start, u32 len)
431{
432 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
433 u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
434 unsigned long flags;
435
436 spin_lock_irqsave(&subpage->lock, flags);
437 subpage->writeback_bitmap &= ~tmp;
438 if (subpage->writeback_bitmap == 0)
439 end_page_writeback(page);
440 spin_unlock_irqrestore(&subpage->lock, flags);
441}
442
443void btrfs_subpage_set_ordered(const struct btrfs_fs_info *fs_info,
444 struct page *page, u64 start, u32 len)
445{
446 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
447 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
448 unsigned long flags;
449
450 spin_lock_irqsave(&subpage->lock, flags);
451 subpage->ordered_bitmap |= tmp;
452 SetPageOrdered(page);
453 spin_unlock_irqrestore(&subpage->lock, flags);
454}
455
456void btrfs_subpage_clear_ordered(const struct btrfs_fs_info *fs_info,
457 struct page *page, u64 start, u32 len)
458{
459 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private;
460 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len);
461 unsigned long flags;
462
463 spin_lock_irqsave(&subpage->lock, flags);
464 subpage->ordered_bitmap &= ~tmp;
465 if (subpage->ordered_bitmap == 0)
466 ClearPageOrdered(page);
467 spin_unlock_irqrestore(&subpage->lock, flags);
468}
469/*
470 * Unlike set/clear which is dependent on each page status, for test all bits
471 * are tested in the same way.
472 */
473#define IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(name) \
474bool btrfs_subpage_test_##name(const struct btrfs_fs_info *fs_info, \
475 struct page *page, u64 start, u32 len) \
476{ \
477 struct btrfs_subpage *subpage = (struct btrfs_subpage *)page->private; \
478 const u16 tmp = btrfs_subpage_calc_bitmap(fs_info, page, start, len); \
479 unsigned long flags; \
480 bool ret; \
481 \
482 spin_lock_irqsave(&subpage->lock, flags); \
483 ret = ((subpage->name##_bitmap & tmp) == tmp); \
484 spin_unlock_irqrestore(&subpage->lock, flags); \
485 return ret; \
486}
487IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(uptodate);
488IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(error);
489IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(dirty);
490IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(writeback);
491IMPLEMENT_BTRFS_SUBPAGE_TEST_OP(ordered);
492
493/*
494 * Note that, in selftests (extent-io-tests), we can have empty fs_info passed
495 * in. We only test sectorsize == PAGE_SIZE cases so far, thus we can fall
496 * back to regular sectorsize branch.
497 */
498#define IMPLEMENT_BTRFS_PAGE_OPS(name, set_page_func, clear_page_func, \
499 test_page_func) \
500void btrfs_page_set_##name(const struct btrfs_fs_info *fs_info, \
501 struct page *page, u64 start, u32 len) \
502{ \
503 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \
504 set_page_func(page); \
505 return; \
506 } \
507 btrfs_subpage_set_##name(fs_info, page, start, len); \
508} \
509void btrfs_page_clear_##name(const struct btrfs_fs_info *fs_info, \
510 struct page *page, u64 start, u32 len) \
511{ \
512 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \
513 clear_page_func(page); \
514 return; \
515 } \
516 btrfs_subpage_clear_##name(fs_info, page, start, len); \
517} \
518bool btrfs_page_test_##name(const struct btrfs_fs_info *fs_info, \
519 struct page *page, u64 start, u32 len) \
520{ \
521 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \
522 return test_page_func(page); \
523 return btrfs_subpage_test_##name(fs_info, page, start, len); \
524} \
525void btrfs_page_clamp_set_##name(const struct btrfs_fs_info *fs_info, \
526 struct page *page, u64 start, u32 len) \
527{ \
528 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \
529 set_page_func(page); \
530 return; \
531 } \
532 btrfs_subpage_clamp_range(page, &start, &len); \
533 btrfs_subpage_set_##name(fs_info, page, start, len); \
534} \
535void btrfs_page_clamp_clear_##name(const struct btrfs_fs_info *fs_info, \
536 struct page *page, u64 start, u32 len) \
537{ \
538 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) { \
539 clear_page_func(page); \
540 return; \
541 } \
542 btrfs_subpage_clamp_range(page, &start, &len); \
543 btrfs_subpage_clear_##name(fs_info, page, start, len); \
544} \
545bool btrfs_page_clamp_test_##name(const struct btrfs_fs_info *fs_info, \
546 struct page *page, u64 start, u32 len) \
547{ \
548 if (unlikely(!fs_info) || fs_info->sectorsize == PAGE_SIZE) \
549 return test_page_func(page); \
550 btrfs_subpage_clamp_range(page, &start, &len); \
551 return btrfs_subpage_test_##name(fs_info, page, start, len); \
552}
553IMPLEMENT_BTRFS_PAGE_OPS(uptodate, SetPageUptodate, ClearPageUptodate,
554 PageUptodate);
555IMPLEMENT_BTRFS_PAGE_OPS(error, SetPageError, ClearPageError, PageError);
556IMPLEMENT_BTRFS_PAGE_OPS(dirty, set_page_dirty, clear_page_dirty_for_io,
557 PageDirty);
558IMPLEMENT_BTRFS_PAGE_OPS(writeback, set_page_writeback, end_page_writeback,
559 PageWriteback);
560IMPLEMENT_BTRFS_PAGE_OPS(ordered, SetPageOrdered, ClearPageOrdered,
561 PageOrdered);