Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Copyright (C) 2016 - 2020 Christoph Hellwig
6 */
7
8#include <linux/init.h>
9#include <linux/mm.h>
10#include <linux/slab.h>
11#include <linux/kmod.h>
12#include <linux/major.h>
13#include <linux/device_cgroup.h>
14#include <linux/blkdev.h>
15#include <linux/blk-integrity.h>
16#include <linux/backing-dev.h>
17#include <linux/module.h>
18#include <linux/blkpg.h>
19#include <linux/magic.h>
20#include <linux/buffer_head.h>
21#include <linux/swap.h>
22#include <linux/writeback.h>
23#include <linux/mount.h>
24#include <linux/pseudo_fs.h>
25#include <linux/uio.h>
26#include <linux/namei.h>
27#include <linux/part_stat.h>
28#include <linux/uaccess.h>
29#include <linux/stat.h>
30#include "../fs/internal.h"
31#include "blk.h"
32
33struct bdev_inode {
34 struct block_device bdev;
35 struct inode vfs_inode;
36};
37
38static inline struct bdev_inode *BDEV_I(struct inode *inode)
39{
40 return container_of(inode, struct bdev_inode, vfs_inode);
41}
42
43struct block_device *I_BDEV(struct inode *inode)
44{
45 return &BDEV_I(inode)->bdev;
46}
47EXPORT_SYMBOL(I_BDEV);
48
49static void bdev_write_inode(struct block_device *bdev)
50{
51 struct inode *inode = bdev->bd_inode;
52 int ret;
53
54 spin_lock(&inode->i_lock);
55 while (inode->i_state & I_DIRTY) {
56 spin_unlock(&inode->i_lock);
57 ret = write_inode_now(inode, true);
58 if (ret)
59 pr_warn_ratelimited(
60 "VFS: Dirty inode writeback failed for block device %pg (err=%d).\n",
61 bdev, ret);
62 spin_lock(&inode->i_lock);
63 }
64 spin_unlock(&inode->i_lock);
65}
66
67/* Kill _all_ buffers and pagecache , dirty or not.. */
68static void kill_bdev(struct block_device *bdev)
69{
70 struct address_space *mapping = bdev->bd_inode->i_mapping;
71
72 if (mapping_empty(mapping))
73 return;
74
75 invalidate_bh_lrus();
76 truncate_inode_pages(mapping, 0);
77}
78
79/* Invalidate clean unused buffers and pagecache. */
80void invalidate_bdev(struct block_device *bdev)
81{
82 struct address_space *mapping = bdev->bd_inode->i_mapping;
83
84 if (mapping->nrpages) {
85 invalidate_bh_lrus();
86 lru_add_drain_all(); /* make sure all lru add caches are flushed */
87 invalidate_mapping_pages(mapping, 0, -1);
88 }
89}
90EXPORT_SYMBOL(invalidate_bdev);
91
92/*
93 * Drop all buffers & page cache for given bdev range. This function bails
94 * with error if bdev has other exclusive owner (such as filesystem).
95 */
96int truncate_bdev_range(struct block_device *bdev, fmode_t mode,
97 loff_t lstart, loff_t lend)
98{
99 /*
100 * If we don't hold exclusive handle for the device, upgrade to it
101 * while we discard the buffer cache to avoid discarding buffers
102 * under live filesystem.
103 */
104 if (!(mode & FMODE_EXCL)) {
105 int err = bd_prepare_to_claim(bdev, truncate_bdev_range);
106 if (err)
107 goto invalidate;
108 }
109
110 truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend);
111 if (!(mode & FMODE_EXCL))
112 bd_abort_claiming(bdev, truncate_bdev_range);
113 return 0;
114
115invalidate:
116 /*
117 * Someone else has handle exclusively open. Try invalidating instead.
118 * The 'end' argument is inclusive so the rounding is safe.
119 */
120 return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping,
121 lstart >> PAGE_SHIFT,
122 lend >> PAGE_SHIFT);
123}
124
125static void set_init_blocksize(struct block_device *bdev)
126{
127 unsigned int bsize = bdev_logical_block_size(bdev);
128 loff_t size = i_size_read(bdev->bd_inode);
129
130 while (bsize < PAGE_SIZE) {
131 if (size & bsize)
132 break;
133 bsize <<= 1;
134 }
135 bdev->bd_inode->i_blkbits = blksize_bits(bsize);
136}
137
138int set_blocksize(struct block_device *bdev, int size)
139{
140 /* Size must be a power of two, and between 512 and PAGE_SIZE */
141 if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size))
142 return -EINVAL;
143
144 /* Size cannot be smaller than the size supported by the device */
145 if (size < bdev_logical_block_size(bdev))
146 return -EINVAL;
147
148 /* Don't change the size if it is same as current */
149 if (bdev->bd_inode->i_blkbits != blksize_bits(size)) {
150 sync_blockdev(bdev);
151 bdev->bd_inode->i_blkbits = blksize_bits(size);
152 kill_bdev(bdev);
153 }
154 return 0;
155}
156
157EXPORT_SYMBOL(set_blocksize);
158
159int sb_set_blocksize(struct super_block *sb, int size)
160{
161 if (set_blocksize(sb->s_bdev, size))
162 return 0;
163 /* If we get here, we know size is power of two
164 * and it's value is between 512 and PAGE_SIZE */
165 sb->s_blocksize = size;
166 sb->s_blocksize_bits = blksize_bits(size);
167 return sb->s_blocksize;
168}
169
170EXPORT_SYMBOL(sb_set_blocksize);
171
172int sb_min_blocksize(struct super_block *sb, int size)
173{
174 int minsize = bdev_logical_block_size(sb->s_bdev);
175 if (size < minsize)
176 size = minsize;
177 return sb_set_blocksize(sb, size);
178}
179
180EXPORT_SYMBOL(sb_min_blocksize);
181
182int sync_blockdev_nowait(struct block_device *bdev)
183{
184 if (!bdev)
185 return 0;
186 return filemap_flush(bdev->bd_inode->i_mapping);
187}
188EXPORT_SYMBOL_GPL(sync_blockdev_nowait);
189
190/*
191 * Write out and wait upon all the dirty data associated with a block
192 * device via its mapping. Does not take the superblock lock.
193 */
194int sync_blockdev(struct block_device *bdev)
195{
196 if (!bdev)
197 return 0;
198 return filemap_write_and_wait(bdev->bd_inode->i_mapping);
199}
200EXPORT_SYMBOL(sync_blockdev);
201
202int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend)
203{
204 return filemap_write_and_wait_range(bdev->bd_inode->i_mapping,
205 lstart, lend);
206}
207EXPORT_SYMBOL(sync_blockdev_range);
208
209/*
210 * Write out and wait upon all dirty data associated with this
211 * device. Filesystem data as well as the underlying block
212 * device. Takes the superblock lock.
213 */
214int fsync_bdev(struct block_device *bdev)
215{
216 struct super_block *sb = get_super(bdev);
217 if (sb) {
218 int res = sync_filesystem(sb);
219 drop_super(sb);
220 return res;
221 }
222 return sync_blockdev(bdev);
223}
224EXPORT_SYMBOL(fsync_bdev);
225
226/**
227 * freeze_bdev - lock a filesystem and force it into a consistent state
228 * @bdev: blockdevice to lock
229 *
230 * If a superblock is found on this device, we take the s_umount semaphore
231 * on it to make sure nobody unmounts until the snapshot creation is done.
232 * The reference counter (bd_fsfreeze_count) guarantees that only the last
233 * unfreeze process can unfreeze the frozen filesystem actually when multiple
234 * freeze requests arrive simultaneously. It counts up in freeze_bdev() and
235 * count down in thaw_bdev(). When it becomes 0, thaw_bdev() will unfreeze
236 * actually.
237 */
238int freeze_bdev(struct block_device *bdev)
239{
240 struct super_block *sb;
241 int error = 0;
242
243 mutex_lock(&bdev->bd_fsfreeze_mutex);
244 if (++bdev->bd_fsfreeze_count > 1)
245 goto done;
246
247 sb = get_active_super(bdev);
248 if (!sb)
249 goto sync;
250 if (sb->s_op->freeze_super)
251 error = sb->s_op->freeze_super(sb);
252 else
253 error = freeze_super(sb);
254 deactivate_super(sb);
255
256 if (error) {
257 bdev->bd_fsfreeze_count--;
258 goto done;
259 }
260 bdev->bd_fsfreeze_sb = sb;
261
262sync:
263 sync_blockdev(bdev);
264done:
265 mutex_unlock(&bdev->bd_fsfreeze_mutex);
266 return error;
267}
268EXPORT_SYMBOL(freeze_bdev);
269
270/**
271 * thaw_bdev - unlock filesystem
272 * @bdev: blockdevice to unlock
273 *
274 * Unlocks the filesystem and marks it writeable again after freeze_bdev().
275 */
276int thaw_bdev(struct block_device *bdev)
277{
278 struct super_block *sb;
279 int error = -EINVAL;
280
281 mutex_lock(&bdev->bd_fsfreeze_mutex);
282 if (!bdev->bd_fsfreeze_count)
283 goto out;
284
285 error = 0;
286 if (--bdev->bd_fsfreeze_count > 0)
287 goto out;
288
289 sb = bdev->bd_fsfreeze_sb;
290 if (!sb)
291 goto out;
292
293 if (sb->s_op->thaw_super)
294 error = sb->s_op->thaw_super(sb);
295 else
296 error = thaw_super(sb);
297 if (error)
298 bdev->bd_fsfreeze_count++;
299 else
300 bdev->bd_fsfreeze_sb = NULL;
301out:
302 mutex_unlock(&bdev->bd_fsfreeze_mutex);
303 return error;
304}
305EXPORT_SYMBOL(thaw_bdev);
306
307/**
308 * bdev_read_page() - Start reading a page from a block device
309 * @bdev: The device to read the page from
310 * @sector: The offset on the device to read the page to (need not be aligned)
311 * @page: The page to read
312 *
313 * On entry, the page should be locked. It will be unlocked when the page
314 * has been read. If the block driver implements rw_page synchronously,
315 * that will be true on exit from this function, but it need not be.
316 *
317 * Errors returned by this function are usually "soft", eg out of memory, or
318 * queue full; callers should try a different route to read this page rather
319 * than propagate an error back up the stack.
320 *
321 * Return: negative errno if an error occurs, 0 if submission was successful.
322 */
323int bdev_read_page(struct block_device *bdev, sector_t sector,
324 struct page *page)
325{
326 const struct block_device_operations *ops = bdev->bd_disk->fops;
327 int result = -EOPNOTSUPP;
328
329 if (!ops->rw_page || bdev_get_integrity(bdev))
330 return result;
331
332 result = blk_queue_enter(bdev_get_queue(bdev), 0);
333 if (result)
334 return result;
335 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,
336 REQ_OP_READ);
337 blk_queue_exit(bdev_get_queue(bdev));
338 return result;
339}
340
341/**
342 * bdev_write_page() - Start writing a page to a block device
343 * @bdev: The device to write the page to
344 * @sector: The offset on the device to write the page to (need not be aligned)
345 * @page: The page to write
346 * @wbc: The writeback_control for the write
347 *
348 * On entry, the page should be locked and not currently under writeback.
349 * On exit, if the write started successfully, the page will be unlocked and
350 * under writeback. If the write failed already (eg the driver failed to
351 * queue the page to the device), the page will still be locked. If the
352 * caller is a ->writepage implementation, it will need to unlock the page.
353 *
354 * Errors returned by this function are usually "soft", eg out of memory, or
355 * queue full; callers should try a different route to write this page rather
356 * than propagate an error back up the stack.
357 *
358 * Return: negative errno if an error occurs, 0 if submission was successful.
359 */
360int bdev_write_page(struct block_device *bdev, sector_t sector,
361 struct page *page, struct writeback_control *wbc)
362{
363 int result;
364 const struct block_device_operations *ops = bdev->bd_disk->fops;
365
366 if (!ops->rw_page || bdev_get_integrity(bdev))
367 return -EOPNOTSUPP;
368 result = blk_queue_enter(bdev_get_queue(bdev), 0);
369 if (result)
370 return result;
371
372 set_page_writeback(page);
373 result = ops->rw_page(bdev, sector + get_start_sect(bdev), page,
374 REQ_OP_WRITE);
375 if (result) {
376 end_page_writeback(page);
377 } else {
378 clean_page_buffers(page);
379 unlock_page(page);
380 }
381 blk_queue_exit(bdev_get_queue(bdev));
382 return result;
383}
384
385/*
386 * pseudo-fs
387 */
388
389static __cacheline_aligned_in_smp DEFINE_SPINLOCK(bdev_lock);
390static struct kmem_cache * bdev_cachep __read_mostly;
391
392static struct inode *bdev_alloc_inode(struct super_block *sb)
393{
394 struct bdev_inode *ei = alloc_inode_sb(sb, bdev_cachep, GFP_KERNEL);
395
396 if (!ei)
397 return NULL;
398 memset(&ei->bdev, 0, sizeof(ei->bdev));
399 return &ei->vfs_inode;
400}
401
402static void bdev_free_inode(struct inode *inode)
403{
404 struct block_device *bdev = I_BDEV(inode);
405
406 free_percpu(bdev->bd_stats);
407 kfree(bdev->bd_meta_info);
408
409 if (!bdev_is_partition(bdev)) {
410 if (bdev->bd_disk && bdev->bd_disk->bdi)
411 bdi_put(bdev->bd_disk->bdi);
412 kfree(bdev->bd_disk);
413 }
414
415 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
416 blk_free_ext_minor(MINOR(bdev->bd_dev));
417
418 kmem_cache_free(bdev_cachep, BDEV_I(inode));
419}
420
421static void init_once(void *data)
422{
423 struct bdev_inode *ei = data;
424
425 inode_init_once(&ei->vfs_inode);
426}
427
428static void bdev_evict_inode(struct inode *inode)
429{
430 truncate_inode_pages_final(&inode->i_data);
431 invalidate_inode_buffers(inode); /* is it needed here? */
432 clear_inode(inode);
433}
434
435static const struct super_operations bdev_sops = {
436 .statfs = simple_statfs,
437 .alloc_inode = bdev_alloc_inode,
438 .free_inode = bdev_free_inode,
439 .drop_inode = generic_delete_inode,
440 .evict_inode = bdev_evict_inode,
441};
442
443static int bd_init_fs_context(struct fs_context *fc)
444{
445 struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC);
446 if (!ctx)
447 return -ENOMEM;
448 fc->s_iflags |= SB_I_CGROUPWB;
449 ctx->ops = &bdev_sops;
450 return 0;
451}
452
453static struct file_system_type bd_type = {
454 .name = "bdev",
455 .init_fs_context = bd_init_fs_context,
456 .kill_sb = kill_anon_super,
457};
458
459struct super_block *blockdev_superblock __read_mostly;
460EXPORT_SYMBOL_GPL(blockdev_superblock);
461
462void __init bdev_cache_init(void)
463{
464 int err;
465 static struct vfsmount *bd_mnt;
466
467 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
468 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
469 SLAB_MEM_SPREAD|SLAB_ACCOUNT|SLAB_PANIC),
470 init_once);
471 err = register_filesystem(&bd_type);
472 if (err)
473 panic("Cannot register bdev pseudo-fs");
474 bd_mnt = kern_mount(&bd_type);
475 if (IS_ERR(bd_mnt))
476 panic("Cannot create bdev pseudo-fs");
477 blockdev_superblock = bd_mnt->mnt_sb; /* For writeback */
478}
479
480struct block_device *bdev_alloc(struct gendisk *disk, u8 partno)
481{
482 struct block_device *bdev;
483 struct inode *inode;
484
485 inode = new_inode(blockdev_superblock);
486 if (!inode)
487 return NULL;
488 inode->i_mode = S_IFBLK;
489 inode->i_rdev = 0;
490 inode->i_data.a_ops = &def_blk_aops;
491 mapping_set_gfp_mask(&inode->i_data, GFP_USER);
492
493 bdev = I_BDEV(inode);
494 mutex_init(&bdev->bd_fsfreeze_mutex);
495 spin_lock_init(&bdev->bd_size_lock);
496 bdev->bd_partno = partno;
497 bdev->bd_inode = inode;
498 bdev->bd_queue = disk->queue;
499 bdev->bd_stats = alloc_percpu(struct disk_stats);
500 if (!bdev->bd_stats) {
501 iput(inode);
502 return NULL;
503 }
504 bdev->bd_disk = disk;
505 return bdev;
506}
507
508void bdev_add(struct block_device *bdev, dev_t dev)
509{
510 bdev->bd_dev = dev;
511 bdev->bd_inode->i_rdev = dev;
512 bdev->bd_inode->i_ino = dev;
513 insert_inode_hash(bdev->bd_inode);
514}
515
516long nr_blockdev_pages(void)
517{
518 struct inode *inode;
519 long ret = 0;
520
521 spin_lock(&blockdev_superblock->s_inode_list_lock);
522 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list)
523 ret += inode->i_mapping->nrpages;
524 spin_unlock(&blockdev_superblock->s_inode_list_lock);
525
526 return ret;
527}
528
529/**
530 * bd_may_claim - test whether a block device can be claimed
531 * @bdev: block device of interest
532 * @whole: whole block device containing @bdev, may equal @bdev
533 * @holder: holder trying to claim @bdev
534 *
535 * Test whether @bdev can be claimed by @holder.
536 *
537 * CONTEXT:
538 * spin_lock(&bdev_lock).
539 *
540 * RETURNS:
541 * %true if @bdev can be claimed, %false otherwise.
542 */
543static bool bd_may_claim(struct block_device *bdev, struct block_device *whole,
544 void *holder)
545{
546 if (bdev->bd_holder == holder)
547 return true; /* already a holder */
548 else if (bdev->bd_holder != NULL)
549 return false; /* held by someone else */
550 else if (whole == bdev)
551 return true; /* is a whole device which isn't held */
552
553 else if (whole->bd_holder == bd_may_claim)
554 return true; /* is a partition of a device that is being partitioned */
555 else if (whole->bd_holder != NULL)
556 return false; /* is a partition of a held device */
557 else
558 return true; /* is a partition of an un-held device */
559}
560
561/**
562 * bd_prepare_to_claim - claim a block device
563 * @bdev: block device of interest
564 * @holder: holder trying to claim @bdev
565 *
566 * Claim @bdev. This function fails if @bdev is already claimed by another
567 * holder and waits if another claiming is in progress. return, the caller
568 * has ownership of bd_claiming and bd_holder[s].
569 *
570 * RETURNS:
571 * 0 if @bdev can be claimed, -EBUSY otherwise.
572 */
573int bd_prepare_to_claim(struct block_device *bdev, void *holder)
574{
575 struct block_device *whole = bdev_whole(bdev);
576
577 if (WARN_ON_ONCE(!holder))
578 return -EINVAL;
579retry:
580 spin_lock(&bdev_lock);
581 /* if someone else claimed, fail */
582 if (!bd_may_claim(bdev, whole, holder)) {
583 spin_unlock(&bdev_lock);
584 return -EBUSY;
585 }
586
587 /* if claiming is already in progress, wait for it to finish */
588 if (whole->bd_claiming) {
589 wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0);
590 DEFINE_WAIT(wait);
591
592 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
593 spin_unlock(&bdev_lock);
594 schedule();
595 finish_wait(wq, &wait);
596 goto retry;
597 }
598
599 /* yay, all mine */
600 whole->bd_claiming = holder;
601 spin_unlock(&bdev_lock);
602 return 0;
603}
604EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */
605
606static void bd_clear_claiming(struct block_device *whole, void *holder)
607{
608 lockdep_assert_held(&bdev_lock);
609 /* tell others that we're done */
610 BUG_ON(whole->bd_claiming != holder);
611 whole->bd_claiming = NULL;
612 wake_up_bit(&whole->bd_claiming, 0);
613}
614
615/**
616 * bd_finish_claiming - finish claiming of a block device
617 * @bdev: block device of interest
618 * @holder: holder that has claimed @bdev
619 *
620 * Finish exclusive open of a block device. Mark the device as exlusively
621 * open by the holder and wake up all waiters for exclusive open to finish.
622 */
623static void bd_finish_claiming(struct block_device *bdev, void *holder)
624{
625 struct block_device *whole = bdev_whole(bdev);
626
627 spin_lock(&bdev_lock);
628 BUG_ON(!bd_may_claim(bdev, whole, holder));
629 /*
630 * Note that for a whole device bd_holders will be incremented twice,
631 * and bd_holder will be set to bd_may_claim before being set to holder
632 */
633 whole->bd_holders++;
634 whole->bd_holder = bd_may_claim;
635 bdev->bd_holders++;
636 bdev->bd_holder = holder;
637 bd_clear_claiming(whole, holder);
638 spin_unlock(&bdev_lock);
639}
640
641/**
642 * bd_abort_claiming - abort claiming of a block device
643 * @bdev: block device of interest
644 * @holder: holder that has claimed @bdev
645 *
646 * Abort claiming of a block device when the exclusive open failed. This can be
647 * also used when exclusive open is not actually desired and we just needed
648 * to block other exclusive openers for a while.
649 */
650void bd_abort_claiming(struct block_device *bdev, void *holder)
651{
652 spin_lock(&bdev_lock);
653 bd_clear_claiming(bdev_whole(bdev), holder);
654 spin_unlock(&bdev_lock);
655}
656EXPORT_SYMBOL(bd_abort_claiming);
657
658static void blkdev_flush_mapping(struct block_device *bdev)
659{
660 WARN_ON_ONCE(bdev->bd_holders);
661 sync_blockdev(bdev);
662 kill_bdev(bdev);
663 bdev_write_inode(bdev);
664}
665
666static int blkdev_get_whole(struct block_device *bdev, fmode_t mode)
667{
668 struct gendisk *disk = bdev->bd_disk;
669 int ret;
670
671 if (disk->fops->open) {
672 ret = disk->fops->open(bdev, mode);
673 if (ret) {
674 /* avoid ghost partitions on a removed medium */
675 if (ret == -ENOMEDIUM &&
676 test_bit(GD_NEED_PART_SCAN, &disk->state))
677 bdev_disk_changed(disk, true);
678 return ret;
679 }
680 }
681
682 if (!atomic_read(&bdev->bd_openers))
683 set_init_blocksize(bdev);
684 if (test_bit(GD_NEED_PART_SCAN, &disk->state))
685 bdev_disk_changed(disk, false);
686 atomic_inc(&bdev->bd_openers);
687 return 0;
688}
689
690static void blkdev_put_whole(struct block_device *bdev, fmode_t mode)
691{
692 if (atomic_dec_and_test(&bdev->bd_openers))
693 blkdev_flush_mapping(bdev);
694 if (bdev->bd_disk->fops->release)
695 bdev->bd_disk->fops->release(bdev->bd_disk, mode);
696}
697
698static int blkdev_get_part(struct block_device *part, fmode_t mode)
699{
700 struct gendisk *disk = part->bd_disk;
701 int ret;
702
703 if (atomic_read(&part->bd_openers))
704 goto done;
705
706 ret = blkdev_get_whole(bdev_whole(part), mode);
707 if (ret)
708 return ret;
709
710 ret = -ENXIO;
711 if (!bdev_nr_sectors(part))
712 goto out_blkdev_put;
713
714 disk->open_partitions++;
715 set_init_blocksize(part);
716done:
717 atomic_inc(&part->bd_openers);
718 return 0;
719
720out_blkdev_put:
721 blkdev_put_whole(bdev_whole(part), mode);
722 return ret;
723}
724
725static void blkdev_put_part(struct block_device *part, fmode_t mode)
726{
727 struct block_device *whole = bdev_whole(part);
728
729 if (!atomic_dec_and_test(&part->bd_openers))
730 return;
731 blkdev_flush_mapping(part);
732 whole->bd_disk->open_partitions--;
733 blkdev_put_whole(whole, mode);
734}
735
736struct block_device *blkdev_get_no_open(dev_t dev)
737{
738 struct block_device *bdev;
739 struct inode *inode;
740
741 inode = ilookup(blockdev_superblock, dev);
742 if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) {
743 blk_request_module(dev);
744 inode = ilookup(blockdev_superblock, dev);
745 if (inode)
746 pr_warn_ratelimited(
747"block device autoloading is deprecated and will be removed.\n");
748 }
749 if (!inode)
750 return NULL;
751
752 /* switch from the inode reference to a device mode one: */
753 bdev = &BDEV_I(inode)->bdev;
754 if (!kobject_get_unless_zero(&bdev->bd_device.kobj))
755 bdev = NULL;
756 iput(inode);
757 return bdev;
758}
759
760void blkdev_put_no_open(struct block_device *bdev)
761{
762 put_device(&bdev->bd_device);
763}
764
765/**
766 * blkdev_get_by_dev - open a block device by device number
767 * @dev: device number of block device to open
768 * @mode: FMODE_* mask
769 * @holder: exclusive holder identifier
770 *
771 * Open the block device described by device number @dev. If @mode includes
772 * %FMODE_EXCL, the block device is opened with exclusive access. Specifying
773 * %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may nest for
774 * the same @holder.
775 *
776 * Use this interface ONLY if you really do not have anything better - i.e. when
777 * you are behind a truly sucky interface and all you are given is a device
778 * number. Everything else should use blkdev_get_by_path().
779 *
780 * CONTEXT:
781 * Might sleep.
782 *
783 * RETURNS:
784 * Reference to the block_device on success, ERR_PTR(-errno) on failure.
785 */
786struct block_device *blkdev_get_by_dev(dev_t dev, fmode_t mode, void *holder)
787{
788 bool unblock_events = true;
789 struct block_device *bdev;
790 struct gendisk *disk;
791 int ret;
792
793 ret = devcgroup_check_permission(DEVCG_DEV_BLOCK,
794 MAJOR(dev), MINOR(dev),
795 ((mode & FMODE_READ) ? DEVCG_ACC_READ : 0) |
796 ((mode & FMODE_WRITE) ? DEVCG_ACC_WRITE : 0));
797 if (ret)
798 return ERR_PTR(ret);
799
800 bdev = blkdev_get_no_open(dev);
801 if (!bdev)
802 return ERR_PTR(-ENXIO);
803 disk = bdev->bd_disk;
804
805 if (mode & FMODE_EXCL) {
806 ret = bd_prepare_to_claim(bdev, holder);
807 if (ret)
808 goto put_blkdev;
809 }
810
811 disk_block_events(disk);
812
813 mutex_lock(&disk->open_mutex);
814 ret = -ENXIO;
815 if (!disk_live(disk))
816 goto abort_claiming;
817 if (!try_module_get(disk->fops->owner))
818 goto abort_claiming;
819 if (bdev_is_partition(bdev))
820 ret = blkdev_get_part(bdev, mode);
821 else
822 ret = blkdev_get_whole(bdev, mode);
823 if (ret)
824 goto put_module;
825 if (mode & FMODE_EXCL) {
826 bd_finish_claiming(bdev, holder);
827
828 /*
829 * Block event polling for write claims if requested. Any write
830 * holder makes the write_holder state stick until all are
831 * released. This is good enough and tracking individual
832 * writeable reference is too fragile given the way @mode is
833 * used in blkdev_get/put().
834 */
835 if ((mode & FMODE_WRITE) && !bdev->bd_write_holder &&
836 (disk->event_flags & DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE)) {
837 bdev->bd_write_holder = true;
838 unblock_events = false;
839 }
840 }
841 mutex_unlock(&disk->open_mutex);
842
843 if (unblock_events)
844 disk_unblock_events(disk);
845 return bdev;
846put_module:
847 module_put(disk->fops->owner);
848abort_claiming:
849 if (mode & FMODE_EXCL)
850 bd_abort_claiming(bdev, holder);
851 mutex_unlock(&disk->open_mutex);
852 disk_unblock_events(disk);
853put_blkdev:
854 blkdev_put_no_open(bdev);
855 return ERR_PTR(ret);
856}
857EXPORT_SYMBOL(blkdev_get_by_dev);
858
859/**
860 * blkdev_get_by_path - open a block device by name
861 * @path: path to the block device to open
862 * @mode: FMODE_* mask
863 * @holder: exclusive holder identifier
864 *
865 * Open the block device described by the device file at @path. If @mode
866 * includes %FMODE_EXCL, the block device is opened with exclusive access.
867 * Specifying %FMODE_EXCL with a %NULL @holder is invalid. Exclusive opens may
868 * nest for the same @holder.
869 *
870 * CONTEXT:
871 * Might sleep.
872 *
873 * RETURNS:
874 * Reference to the block_device on success, ERR_PTR(-errno) on failure.
875 */
876struct block_device *blkdev_get_by_path(const char *path, fmode_t mode,
877 void *holder)
878{
879 struct block_device *bdev;
880 dev_t dev;
881 int error;
882
883 error = lookup_bdev(path, &dev);
884 if (error)
885 return ERR_PTR(error);
886
887 bdev = blkdev_get_by_dev(dev, mode, holder);
888 if (!IS_ERR(bdev) && (mode & FMODE_WRITE) && bdev_read_only(bdev)) {
889 blkdev_put(bdev, mode);
890 return ERR_PTR(-EACCES);
891 }
892
893 return bdev;
894}
895EXPORT_SYMBOL(blkdev_get_by_path);
896
897void blkdev_put(struct block_device *bdev, fmode_t mode)
898{
899 struct gendisk *disk = bdev->bd_disk;
900
901 /*
902 * Sync early if it looks like we're the last one. If someone else
903 * opens the block device between now and the decrement of bd_openers
904 * then we did a sync that we didn't need to, but that's not the end
905 * of the world and we want to avoid long (could be several minute)
906 * syncs while holding the mutex.
907 */
908 if (atomic_read(&bdev->bd_openers) == 1)
909 sync_blockdev(bdev);
910
911 mutex_lock(&disk->open_mutex);
912 if (mode & FMODE_EXCL) {
913 struct block_device *whole = bdev_whole(bdev);
914 bool bdev_free;
915
916 /*
917 * Release a claim on the device. The holder fields
918 * are protected with bdev_lock. open_mutex is to
919 * synchronize disk_holder unlinking.
920 */
921 spin_lock(&bdev_lock);
922
923 WARN_ON_ONCE(--bdev->bd_holders < 0);
924 WARN_ON_ONCE(--whole->bd_holders < 0);
925
926 if ((bdev_free = !bdev->bd_holders))
927 bdev->bd_holder = NULL;
928 if (!whole->bd_holders)
929 whole->bd_holder = NULL;
930
931 spin_unlock(&bdev_lock);
932
933 /*
934 * If this was the last claim, remove holder link and
935 * unblock evpoll if it was a write holder.
936 */
937 if (bdev_free && bdev->bd_write_holder) {
938 disk_unblock_events(disk);
939 bdev->bd_write_holder = false;
940 }
941 }
942
943 /*
944 * Trigger event checking and tell drivers to flush MEDIA_CHANGE
945 * event. This is to ensure detection of media removal commanded
946 * from userland - e.g. eject(1).
947 */
948 disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE);
949
950 if (bdev_is_partition(bdev))
951 blkdev_put_part(bdev, mode);
952 else
953 blkdev_put_whole(bdev, mode);
954 mutex_unlock(&disk->open_mutex);
955
956 module_put(disk->fops->owner);
957 blkdev_put_no_open(bdev);
958}
959EXPORT_SYMBOL(blkdev_put);
960
961/**
962 * lookup_bdev() - Look up a struct block_device by name.
963 * @pathname: Name of the block device in the filesystem.
964 * @dev: Pointer to the block device's dev_t, if found.
965 *
966 * Lookup the block device's dev_t at @pathname in the current
967 * namespace if possible and return it in @dev.
968 *
969 * Context: May sleep.
970 * Return: 0 if succeeded, negative errno otherwise.
971 */
972int lookup_bdev(const char *pathname, dev_t *dev)
973{
974 struct inode *inode;
975 struct path path;
976 int error;
977
978 if (!pathname || !*pathname)
979 return -EINVAL;
980
981 error = kern_path(pathname, LOOKUP_FOLLOW, &path);
982 if (error)
983 return error;
984
985 inode = d_backing_inode(path.dentry);
986 error = -ENOTBLK;
987 if (!S_ISBLK(inode->i_mode))
988 goto out_path_put;
989 error = -EACCES;
990 if (!may_open_dev(&path))
991 goto out_path_put;
992
993 *dev = inode->i_rdev;
994 error = 0;
995out_path_put:
996 path_put(&path);
997 return error;
998}
999EXPORT_SYMBOL(lookup_bdev);
1000
1001int __invalidate_device(struct block_device *bdev, bool kill_dirty)
1002{
1003 struct super_block *sb = get_super(bdev);
1004 int res = 0;
1005
1006 if (sb) {
1007 /*
1008 * no need to lock the super, get_super holds the
1009 * read mutex so the filesystem cannot go away
1010 * under us (->put_super runs with the write lock
1011 * hold).
1012 */
1013 shrink_dcache_sb(sb);
1014 res = invalidate_inodes(sb, kill_dirty);
1015 drop_super(sb);
1016 }
1017 invalidate_bdev(bdev);
1018 return res;
1019}
1020EXPORT_SYMBOL(__invalidate_device);
1021
1022void sync_bdevs(bool wait)
1023{
1024 struct inode *inode, *old_inode = NULL;
1025
1026 spin_lock(&blockdev_superblock->s_inode_list_lock);
1027 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
1028 struct address_space *mapping = inode->i_mapping;
1029 struct block_device *bdev;
1030
1031 spin_lock(&inode->i_lock);
1032 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) ||
1033 mapping->nrpages == 0) {
1034 spin_unlock(&inode->i_lock);
1035 continue;
1036 }
1037 __iget(inode);
1038 spin_unlock(&inode->i_lock);
1039 spin_unlock(&blockdev_superblock->s_inode_list_lock);
1040 /*
1041 * We hold a reference to 'inode' so it couldn't have been
1042 * removed from s_inodes list while we dropped the
1043 * s_inode_list_lock We cannot iput the inode now as we can
1044 * be holding the last reference and we cannot iput it under
1045 * s_inode_list_lock. So we keep the reference and iput it
1046 * later.
1047 */
1048 iput(old_inode);
1049 old_inode = inode;
1050 bdev = I_BDEV(inode);
1051
1052 mutex_lock(&bdev->bd_disk->open_mutex);
1053 if (!atomic_read(&bdev->bd_openers)) {
1054 ; /* skip */
1055 } else if (wait) {
1056 /*
1057 * We keep the error status of individual mapping so
1058 * that applications can catch the writeback error using
1059 * fsync(2). See filemap_fdatawait_keep_errors() for
1060 * details.
1061 */
1062 filemap_fdatawait_keep_errors(inode->i_mapping);
1063 } else {
1064 filemap_fdatawrite(inode->i_mapping);
1065 }
1066 mutex_unlock(&bdev->bd_disk->open_mutex);
1067
1068 spin_lock(&blockdev_superblock->s_inode_list_lock);
1069 }
1070 spin_unlock(&blockdev_superblock->s_inode_list_lock);
1071 iput(old_inode);
1072}
1073
1074/*
1075 * Handle STATX_DIOALIGN for block devices.
1076 *
1077 * Note that the inode passed to this is the inode of a block device node file,
1078 * not the block device's internal inode. Therefore it is *not* valid to use
1079 * I_BDEV() here; the block device has to be looked up by i_rdev instead.
1080 */
1081void bdev_statx_dioalign(struct inode *inode, struct kstat *stat)
1082{
1083 struct block_device *bdev;
1084
1085 bdev = blkdev_get_no_open(inode->i_rdev);
1086 if (!bdev)
1087 return;
1088
1089 stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;
1090 stat->dio_offset_align = bdev_logical_block_size(bdev);
1091 stat->result_mask |= STATX_DIOALIGN;
1092
1093 blkdev_put_no_open(bdev);
1094}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 1991, 1992 Linus Torvalds
4 * Copyright (C) 2001 Andrea Arcangeli <andrea@suse.de> SuSE
5 * Copyright (C) 2016 - 2020 Christoph Hellwig
6 */
7
8#include <linux/init.h>
9#include <linux/mm.h>
10#include <linux/slab.h>
11#include <linux/kmod.h>
12#include <linux/major.h>
13#include <linux/device_cgroup.h>
14#include <linux/blkdev.h>
15#include <linux/blk-integrity.h>
16#include <linux/backing-dev.h>
17#include <linux/module.h>
18#include <linux/blkpg.h>
19#include <linux/magic.h>
20#include <linux/buffer_head.h>
21#include <linux/swap.h>
22#include <linux/writeback.h>
23#include <linux/mount.h>
24#include <linux/pseudo_fs.h>
25#include <linux/uio.h>
26#include <linux/namei.h>
27#include <linux/part_stat.h>
28#include <linux/uaccess.h>
29#include <linux/stat.h>
30#include "../fs/internal.h"
31#include "blk.h"
32
33/* Should we allow writing to mounted block devices? */
34static bool bdev_allow_write_mounted = IS_ENABLED(CONFIG_BLK_DEV_WRITE_MOUNTED);
35
36struct bdev_inode {
37 struct block_device bdev;
38 struct inode vfs_inode;
39};
40
41static inline struct bdev_inode *BDEV_I(struct inode *inode)
42{
43 return container_of(inode, struct bdev_inode, vfs_inode);
44}
45
46struct block_device *I_BDEV(struct inode *inode)
47{
48 return &BDEV_I(inode)->bdev;
49}
50EXPORT_SYMBOL(I_BDEV);
51
52struct block_device *file_bdev(struct file *bdev_file)
53{
54 return I_BDEV(bdev_file->f_mapping->host);
55}
56EXPORT_SYMBOL(file_bdev);
57
58static void bdev_write_inode(struct block_device *bdev)
59{
60 struct inode *inode = bdev->bd_inode;
61 int ret;
62
63 spin_lock(&inode->i_lock);
64 while (inode->i_state & I_DIRTY) {
65 spin_unlock(&inode->i_lock);
66 ret = write_inode_now(inode, true);
67 if (ret)
68 pr_warn_ratelimited(
69 "VFS: Dirty inode writeback failed for block device %pg (err=%d).\n",
70 bdev, ret);
71 spin_lock(&inode->i_lock);
72 }
73 spin_unlock(&inode->i_lock);
74}
75
76/* Kill _all_ buffers and pagecache , dirty or not.. */
77static void kill_bdev(struct block_device *bdev)
78{
79 struct address_space *mapping = bdev->bd_inode->i_mapping;
80
81 if (mapping_empty(mapping))
82 return;
83
84 invalidate_bh_lrus();
85 truncate_inode_pages(mapping, 0);
86}
87
88/* Invalidate clean unused buffers and pagecache. */
89void invalidate_bdev(struct block_device *bdev)
90{
91 struct address_space *mapping = bdev->bd_inode->i_mapping;
92
93 if (mapping->nrpages) {
94 invalidate_bh_lrus();
95 lru_add_drain_all(); /* make sure all lru add caches are flushed */
96 invalidate_mapping_pages(mapping, 0, -1);
97 }
98}
99EXPORT_SYMBOL(invalidate_bdev);
100
101/*
102 * Drop all buffers & page cache for given bdev range. This function bails
103 * with error if bdev has other exclusive owner (such as filesystem).
104 */
105int truncate_bdev_range(struct block_device *bdev, blk_mode_t mode,
106 loff_t lstart, loff_t lend)
107{
108 /*
109 * If we don't hold exclusive handle for the device, upgrade to it
110 * while we discard the buffer cache to avoid discarding buffers
111 * under live filesystem.
112 */
113 if (!(mode & BLK_OPEN_EXCL)) {
114 int err = bd_prepare_to_claim(bdev, truncate_bdev_range, NULL);
115 if (err)
116 goto invalidate;
117 }
118
119 truncate_inode_pages_range(bdev->bd_inode->i_mapping, lstart, lend);
120 if (!(mode & BLK_OPEN_EXCL))
121 bd_abort_claiming(bdev, truncate_bdev_range);
122 return 0;
123
124invalidate:
125 /*
126 * Someone else has handle exclusively open. Try invalidating instead.
127 * The 'end' argument is inclusive so the rounding is safe.
128 */
129 return invalidate_inode_pages2_range(bdev->bd_inode->i_mapping,
130 lstart >> PAGE_SHIFT,
131 lend >> PAGE_SHIFT);
132}
133
134static void set_init_blocksize(struct block_device *bdev)
135{
136 unsigned int bsize = bdev_logical_block_size(bdev);
137 loff_t size = i_size_read(bdev->bd_inode);
138
139 while (bsize < PAGE_SIZE) {
140 if (size & bsize)
141 break;
142 bsize <<= 1;
143 }
144 bdev->bd_inode->i_blkbits = blksize_bits(bsize);
145}
146
147int set_blocksize(struct block_device *bdev, int size)
148{
149 /* Size must be a power of two, and between 512 and PAGE_SIZE */
150 if (size > PAGE_SIZE || size < 512 || !is_power_of_2(size))
151 return -EINVAL;
152
153 /* Size cannot be smaller than the size supported by the device */
154 if (size < bdev_logical_block_size(bdev))
155 return -EINVAL;
156
157 /* Don't change the size if it is same as current */
158 if (bdev->bd_inode->i_blkbits != blksize_bits(size)) {
159 sync_blockdev(bdev);
160 bdev->bd_inode->i_blkbits = blksize_bits(size);
161 kill_bdev(bdev);
162 }
163 return 0;
164}
165
166EXPORT_SYMBOL(set_blocksize);
167
168int sb_set_blocksize(struct super_block *sb, int size)
169{
170 if (set_blocksize(sb->s_bdev, size))
171 return 0;
172 /* If we get here, we know size is power of two
173 * and it's value is between 512 and PAGE_SIZE */
174 sb->s_blocksize = size;
175 sb->s_blocksize_bits = blksize_bits(size);
176 return sb->s_blocksize;
177}
178
179EXPORT_SYMBOL(sb_set_blocksize);
180
181int sb_min_blocksize(struct super_block *sb, int size)
182{
183 int minsize = bdev_logical_block_size(sb->s_bdev);
184 if (size < minsize)
185 size = minsize;
186 return sb_set_blocksize(sb, size);
187}
188
189EXPORT_SYMBOL(sb_min_blocksize);
190
191int sync_blockdev_nowait(struct block_device *bdev)
192{
193 if (!bdev)
194 return 0;
195 return filemap_flush(bdev->bd_inode->i_mapping);
196}
197EXPORT_SYMBOL_GPL(sync_blockdev_nowait);
198
199/*
200 * Write out and wait upon all the dirty data associated with a block
201 * device via its mapping. Does not take the superblock lock.
202 */
203int sync_blockdev(struct block_device *bdev)
204{
205 if (!bdev)
206 return 0;
207 return filemap_write_and_wait(bdev->bd_inode->i_mapping);
208}
209EXPORT_SYMBOL(sync_blockdev);
210
211int sync_blockdev_range(struct block_device *bdev, loff_t lstart, loff_t lend)
212{
213 return filemap_write_and_wait_range(bdev->bd_inode->i_mapping,
214 lstart, lend);
215}
216EXPORT_SYMBOL(sync_blockdev_range);
217
218/**
219 * bdev_freeze - lock a filesystem and force it into a consistent state
220 * @bdev: blockdevice to lock
221 *
222 * If a superblock is found on this device, we take the s_umount semaphore
223 * on it to make sure nobody unmounts until the snapshot creation is done.
224 * The reference counter (bd_fsfreeze_count) guarantees that only the last
225 * unfreeze process can unfreeze the frozen filesystem actually when multiple
226 * freeze requests arrive simultaneously. It counts up in bdev_freeze() and
227 * count down in bdev_thaw(). When it becomes 0, thaw_bdev() will unfreeze
228 * actually.
229 *
230 * Return: On success zero is returned, negative error code on failure.
231 */
232int bdev_freeze(struct block_device *bdev)
233{
234 int error = 0;
235
236 mutex_lock(&bdev->bd_fsfreeze_mutex);
237
238 if (atomic_inc_return(&bdev->bd_fsfreeze_count) > 1) {
239 mutex_unlock(&bdev->bd_fsfreeze_mutex);
240 return 0;
241 }
242
243 mutex_lock(&bdev->bd_holder_lock);
244 if (bdev->bd_holder_ops && bdev->bd_holder_ops->freeze) {
245 error = bdev->bd_holder_ops->freeze(bdev);
246 lockdep_assert_not_held(&bdev->bd_holder_lock);
247 } else {
248 mutex_unlock(&bdev->bd_holder_lock);
249 error = sync_blockdev(bdev);
250 }
251
252 if (error)
253 atomic_dec(&bdev->bd_fsfreeze_count);
254
255 mutex_unlock(&bdev->bd_fsfreeze_mutex);
256 return error;
257}
258EXPORT_SYMBOL(bdev_freeze);
259
260/**
261 * bdev_thaw - unlock filesystem
262 * @bdev: blockdevice to unlock
263 *
264 * Unlocks the filesystem and marks it writeable again after bdev_freeze().
265 *
266 * Return: On success zero is returned, negative error code on failure.
267 */
268int bdev_thaw(struct block_device *bdev)
269{
270 int error = -EINVAL, nr_freeze;
271
272 mutex_lock(&bdev->bd_fsfreeze_mutex);
273
274 /*
275 * If this returns < 0 it means that @bd_fsfreeze_count was
276 * already 0 and no decrement was performed.
277 */
278 nr_freeze = atomic_dec_if_positive(&bdev->bd_fsfreeze_count);
279 if (nr_freeze < 0)
280 goto out;
281
282 error = 0;
283 if (nr_freeze > 0)
284 goto out;
285
286 mutex_lock(&bdev->bd_holder_lock);
287 if (bdev->bd_holder_ops && bdev->bd_holder_ops->thaw) {
288 error = bdev->bd_holder_ops->thaw(bdev);
289 lockdep_assert_not_held(&bdev->bd_holder_lock);
290 } else {
291 mutex_unlock(&bdev->bd_holder_lock);
292 }
293
294 if (error)
295 atomic_inc(&bdev->bd_fsfreeze_count);
296out:
297 mutex_unlock(&bdev->bd_fsfreeze_mutex);
298 return error;
299}
300EXPORT_SYMBOL(bdev_thaw);
301
302/*
303 * pseudo-fs
304 */
305
306static __cacheline_aligned_in_smp DEFINE_MUTEX(bdev_lock);
307static struct kmem_cache *bdev_cachep __ro_after_init;
308
309static struct inode *bdev_alloc_inode(struct super_block *sb)
310{
311 struct bdev_inode *ei = alloc_inode_sb(sb, bdev_cachep, GFP_KERNEL);
312
313 if (!ei)
314 return NULL;
315 memset(&ei->bdev, 0, sizeof(ei->bdev));
316 return &ei->vfs_inode;
317}
318
319static void bdev_free_inode(struct inode *inode)
320{
321 struct block_device *bdev = I_BDEV(inode);
322
323 free_percpu(bdev->bd_stats);
324 kfree(bdev->bd_meta_info);
325
326 if (!bdev_is_partition(bdev)) {
327 if (bdev->bd_disk && bdev->bd_disk->bdi)
328 bdi_put(bdev->bd_disk->bdi);
329 kfree(bdev->bd_disk);
330 }
331
332 if (MAJOR(bdev->bd_dev) == BLOCK_EXT_MAJOR)
333 blk_free_ext_minor(MINOR(bdev->bd_dev));
334
335 kmem_cache_free(bdev_cachep, BDEV_I(inode));
336}
337
338static void init_once(void *data)
339{
340 struct bdev_inode *ei = data;
341
342 inode_init_once(&ei->vfs_inode);
343}
344
345static void bdev_evict_inode(struct inode *inode)
346{
347 truncate_inode_pages_final(&inode->i_data);
348 invalidate_inode_buffers(inode); /* is it needed here? */
349 clear_inode(inode);
350}
351
352static const struct super_operations bdev_sops = {
353 .statfs = simple_statfs,
354 .alloc_inode = bdev_alloc_inode,
355 .free_inode = bdev_free_inode,
356 .drop_inode = generic_delete_inode,
357 .evict_inode = bdev_evict_inode,
358};
359
360static int bd_init_fs_context(struct fs_context *fc)
361{
362 struct pseudo_fs_context *ctx = init_pseudo(fc, BDEVFS_MAGIC);
363 if (!ctx)
364 return -ENOMEM;
365 fc->s_iflags |= SB_I_CGROUPWB;
366 ctx->ops = &bdev_sops;
367 return 0;
368}
369
370static struct file_system_type bd_type = {
371 .name = "bdev",
372 .init_fs_context = bd_init_fs_context,
373 .kill_sb = kill_anon_super,
374};
375
376struct super_block *blockdev_superblock __ro_after_init;
377struct vfsmount *blockdev_mnt __ro_after_init;
378EXPORT_SYMBOL_GPL(blockdev_superblock);
379
380void __init bdev_cache_init(void)
381{
382 int err;
383
384 bdev_cachep = kmem_cache_create("bdev_cache", sizeof(struct bdev_inode),
385 0, (SLAB_HWCACHE_ALIGN|SLAB_RECLAIM_ACCOUNT|
386 SLAB_ACCOUNT|SLAB_PANIC),
387 init_once);
388 err = register_filesystem(&bd_type);
389 if (err)
390 panic("Cannot register bdev pseudo-fs");
391 blockdev_mnt = kern_mount(&bd_type);
392 if (IS_ERR(blockdev_mnt))
393 panic("Cannot create bdev pseudo-fs");
394 blockdev_superblock = blockdev_mnt->mnt_sb; /* For writeback */
395}
396
397struct block_device *bdev_alloc(struct gendisk *disk, u8 partno)
398{
399 struct block_device *bdev;
400 struct inode *inode;
401
402 inode = new_inode(blockdev_superblock);
403 if (!inode)
404 return NULL;
405 inode->i_mode = S_IFBLK;
406 inode->i_rdev = 0;
407 inode->i_data.a_ops = &def_blk_aops;
408 mapping_set_gfp_mask(&inode->i_data, GFP_USER);
409
410 bdev = I_BDEV(inode);
411 mutex_init(&bdev->bd_fsfreeze_mutex);
412 spin_lock_init(&bdev->bd_size_lock);
413 mutex_init(&bdev->bd_holder_lock);
414 bdev->bd_partno = partno;
415 bdev->bd_inode = inode;
416 bdev->bd_queue = disk->queue;
417 if (partno)
418 bdev->bd_has_submit_bio = disk->part0->bd_has_submit_bio;
419 else
420 bdev->bd_has_submit_bio = false;
421 bdev->bd_stats = alloc_percpu(struct disk_stats);
422 if (!bdev->bd_stats) {
423 iput(inode);
424 return NULL;
425 }
426 bdev->bd_disk = disk;
427 return bdev;
428}
429
430void bdev_set_nr_sectors(struct block_device *bdev, sector_t sectors)
431{
432 spin_lock(&bdev->bd_size_lock);
433 i_size_write(bdev->bd_inode, (loff_t)sectors << SECTOR_SHIFT);
434 bdev->bd_nr_sectors = sectors;
435 spin_unlock(&bdev->bd_size_lock);
436}
437
438void bdev_add(struct block_device *bdev, dev_t dev)
439{
440 if (bdev_stable_writes(bdev))
441 mapping_set_stable_writes(bdev->bd_inode->i_mapping);
442 bdev->bd_dev = dev;
443 bdev->bd_inode->i_rdev = dev;
444 bdev->bd_inode->i_ino = dev;
445 insert_inode_hash(bdev->bd_inode);
446}
447
448long nr_blockdev_pages(void)
449{
450 struct inode *inode;
451 long ret = 0;
452
453 spin_lock(&blockdev_superblock->s_inode_list_lock);
454 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list)
455 ret += inode->i_mapping->nrpages;
456 spin_unlock(&blockdev_superblock->s_inode_list_lock);
457
458 return ret;
459}
460
461/**
462 * bd_may_claim - test whether a block device can be claimed
463 * @bdev: block device of interest
464 * @holder: holder trying to claim @bdev
465 * @hops: holder ops
466 *
467 * Test whether @bdev can be claimed by @holder.
468 *
469 * RETURNS:
470 * %true if @bdev can be claimed, %false otherwise.
471 */
472static bool bd_may_claim(struct block_device *bdev, void *holder,
473 const struct blk_holder_ops *hops)
474{
475 struct block_device *whole = bdev_whole(bdev);
476
477 lockdep_assert_held(&bdev_lock);
478
479 if (bdev->bd_holder) {
480 /*
481 * The same holder can always re-claim.
482 */
483 if (bdev->bd_holder == holder) {
484 if (WARN_ON_ONCE(bdev->bd_holder_ops != hops))
485 return false;
486 return true;
487 }
488 return false;
489 }
490
491 /*
492 * If the whole devices holder is set to bd_may_claim, a partition on
493 * the device is claimed, but not the whole device.
494 */
495 if (whole != bdev &&
496 whole->bd_holder && whole->bd_holder != bd_may_claim)
497 return false;
498 return true;
499}
500
501/**
502 * bd_prepare_to_claim - claim a block device
503 * @bdev: block device of interest
504 * @holder: holder trying to claim @bdev
505 * @hops: holder ops.
506 *
507 * Claim @bdev. This function fails if @bdev is already claimed by another
508 * holder and waits if another claiming is in progress. return, the caller
509 * has ownership of bd_claiming and bd_holder[s].
510 *
511 * RETURNS:
512 * 0 if @bdev can be claimed, -EBUSY otherwise.
513 */
514int bd_prepare_to_claim(struct block_device *bdev, void *holder,
515 const struct blk_holder_ops *hops)
516{
517 struct block_device *whole = bdev_whole(bdev);
518
519 if (WARN_ON_ONCE(!holder))
520 return -EINVAL;
521retry:
522 mutex_lock(&bdev_lock);
523 /* if someone else claimed, fail */
524 if (!bd_may_claim(bdev, holder, hops)) {
525 mutex_unlock(&bdev_lock);
526 return -EBUSY;
527 }
528
529 /* if claiming is already in progress, wait for it to finish */
530 if (whole->bd_claiming) {
531 wait_queue_head_t *wq = bit_waitqueue(&whole->bd_claiming, 0);
532 DEFINE_WAIT(wait);
533
534 prepare_to_wait(wq, &wait, TASK_UNINTERRUPTIBLE);
535 mutex_unlock(&bdev_lock);
536 schedule();
537 finish_wait(wq, &wait);
538 goto retry;
539 }
540
541 /* yay, all mine */
542 whole->bd_claiming = holder;
543 mutex_unlock(&bdev_lock);
544 return 0;
545}
546EXPORT_SYMBOL_GPL(bd_prepare_to_claim); /* only for the loop driver */
547
548static void bd_clear_claiming(struct block_device *whole, void *holder)
549{
550 lockdep_assert_held(&bdev_lock);
551 /* tell others that we're done */
552 BUG_ON(whole->bd_claiming != holder);
553 whole->bd_claiming = NULL;
554 wake_up_bit(&whole->bd_claiming, 0);
555}
556
557/**
558 * bd_finish_claiming - finish claiming of a block device
559 * @bdev: block device of interest
560 * @holder: holder that has claimed @bdev
561 * @hops: block device holder operations
562 *
563 * Finish exclusive open of a block device. Mark the device as exlusively
564 * open by the holder and wake up all waiters for exclusive open to finish.
565 */
566static void bd_finish_claiming(struct block_device *bdev, void *holder,
567 const struct blk_holder_ops *hops)
568{
569 struct block_device *whole = bdev_whole(bdev);
570
571 mutex_lock(&bdev_lock);
572 BUG_ON(!bd_may_claim(bdev, holder, hops));
573 /*
574 * Note that for a whole device bd_holders will be incremented twice,
575 * and bd_holder will be set to bd_may_claim before being set to holder
576 */
577 whole->bd_holders++;
578 whole->bd_holder = bd_may_claim;
579 bdev->bd_holders++;
580 mutex_lock(&bdev->bd_holder_lock);
581 bdev->bd_holder = holder;
582 bdev->bd_holder_ops = hops;
583 mutex_unlock(&bdev->bd_holder_lock);
584 bd_clear_claiming(whole, holder);
585 mutex_unlock(&bdev_lock);
586}
587
588/**
589 * bd_abort_claiming - abort claiming of a block device
590 * @bdev: block device of interest
591 * @holder: holder that has claimed @bdev
592 *
593 * Abort claiming of a block device when the exclusive open failed. This can be
594 * also used when exclusive open is not actually desired and we just needed
595 * to block other exclusive openers for a while.
596 */
597void bd_abort_claiming(struct block_device *bdev, void *holder)
598{
599 mutex_lock(&bdev_lock);
600 bd_clear_claiming(bdev_whole(bdev), holder);
601 mutex_unlock(&bdev_lock);
602}
603EXPORT_SYMBOL(bd_abort_claiming);
604
605static void bd_end_claim(struct block_device *bdev, void *holder)
606{
607 struct block_device *whole = bdev_whole(bdev);
608 bool unblock = false;
609
610 /*
611 * Release a claim on the device. The holder fields are protected with
612 * bdev_lock. open_mutex is used to synchronize disk_holder unlinking.
613 */
614 mutex_lock(&bdev_lock);
615 WARN_ON_ONCE(bdev->bd_holder != holder);
616 WARN_ON_ONCE(--bdev->bd_holders < 0);
617 WARN_ON_ONCE(--whole->bd_holders < 0);
618 if (!bdev->bd_holders) {
619 mutex_lock(&bdev->bd_holder_lock);
620 bdev->bd_holder = NULL;
621 bdev->bd_holder_ops = NULL;
622 mutex_unlock(&bdev->bd_holder_lock);
623 if (bdev->bd_write_holder)
624 unblock = true;
625 }
626 if (!whole->bd_holders)
627 whole->bd_holder = NULL;
628 mutex_unlock(&bdev_lock);
629
630 /*
631 * If this was the last claim, remove holder link and unblock evpoll if
632 * it was a write holder.
633 */
634 if (unblock) {
635 disk_unblock_events(bdev->bd_disk);
636 bdev->bd_write_holder = false;
637 }
638}
639
640static void blkdev_flush_mapping(struct block_device *bdev)
641{
642 WARN_ON_ONCE(bdev->bd_holders);
643 sync_blockdev(bdev);
644 kill_bdev(bdev);
645 bdev_write_inode(bdev);
646}
647
648static void blkdev_put_whole(struct block_device *bdev)
649{
650 if (atomic_dec_and_test(&bdev->bd_openers))
651 blkdev_flush_mapping(bdev);
652 if (bdev->bd_disk->fops->release)
653 bdev->bd_disk->fops->release(bdev->bd_disk);
654}
655
656static int blkdev_get_whole(struct block_device *bdev, blk_mode_t mode)
657{
658 struct gendisk *disk = bdev->bd_disk;
659 int ret;
660
661 if (disk->fops->open) {
662 ret = disk->fops->open(disk, mode);
663 if (ret) {
664 /* avoid ghost partitions on a removed medium */
665 if (ret == -ENOMEDIUM &&
666 test_bit(GD_NEED_PART_SCAN, &disk->state))
667 bdev_disk_changed(disk, true);
668 return ret;
669 }
670 }
671
672 if (!atomic_read(&bdev->bd_openers))
673 set_init_blocksize(bdev);
674 atomic_inc(&bdev->bd_openers);
675 if (test_bit(GD_NEED_PART_SCAN, &disk->state)) {
676 /*
677 * Only return scanning errors if we are called from contexts
678 * that explicitly want them, e.g. the BLKRRPART ioctl.
679 */
680 ret = bdev_disk_changed(disk, false);
681 if (ret && (mode & BLK_OPEN_STRICT_SCAN)) {
682 blkdev_put_whole(bdev);
683 return ret;
684 }
685 }
686 return 0;
687}
688
689static int blkdev_get_part(struct block_device *part, blk_mode_t mode)
690{
691 struct gendisk *disk = part->bd_disk;
692 int ret;
693
694 ret = blkdev_get_whole(bdev_whole(part), mode);
695 if (ret)
696 return ret;
697
698 ret = -ENXIO;
699 if (!bdev_nr_sectors(part))
700 goto out_blkdev_put;
701
702 if (!atomic_read(&part->bd_openers)) {
703 disk->open_partitions++;
704 set_init_blocksize(part);
705 }
706 atomic_inc(&part->bd_openers);
707 return 0;
708
709out_blkdev_put:
710 blkdev_put_whole(bdev_whole(part));
711 return ret;
712}
713
714int bdev_permission(dev_t dev, blk_mode_t mode, void *holder)
715{
716 int ret;
717
718 ret = devcgroup_check_permission(DEVCG_DEV_BLOCK,
719 MAJOR(dev), MINOR(dev),
720 ((mode & BLK_OPEN_READ) ? DEVCG_ACC_READ : 0) |
721 ((mode & BLK_OPEN_WRITE) ? DEVCG_ACC_WRITE : 0));
722 if (ret)
723 return ret;
724
725 /* Blocking writes requires exclusive opener */
726 if (mode & BLK_OPEN_RESTRICT_WRITES && !holder)
727 return -EINVAL;
728
729 /*
730 * We're using error pointers to indicate to ->release() when we
731 * failed to open that block device. Also this doesn't make sense.
732 */
733 if (WARN_ON_ONCE(IS_ERR(holder)))
734 return -EINVAL;
735
736 return 0;
737}
738
739static void blkdev_put_part(struct block_device *part)
740{
741 struct block_device *whole = bdev_whole(part);
742
743 if (atomic_dec_and_test(&part->bd_openers)) {
744 blkdev_flush_mapping(part);
745 whole->bd_disk->open_partitions--;
746 }
747 blkdev_put_whole(whole);
748}
749
750struct block_device *blkdev_get_no_open(dev_t dev)
751{
752 struct block_device *bdev;
753 struct inode *inode;
754
755 inode = ilookup(blockdev_superblock, dev);
756 if (!inode && IS_ENABLED(CONFIG_BLOCK_LEGACY_AUTOLOAD)) {
757 blk_request_module(dev);
758 inode = ilookup(blockdev_superblock, dev);
759 if (inode)
760 pr_warn_ratelimited(
761"block device autoloading is deprecated and will be removed.\n");
762 }
763 if (!inode)
764 return NULL;
765
766 /* switch from the inode reference to a device mode one: */
767 bdev = &BDEV_I(inode)->bdev;
768 if (!kobject_get_unless_zero(&bdev->bd_device.kobj))
769 bdev = NULL;
770 iput(inode);
771 return bdev;
772}
773
774void blkdev_put_no_open(struct block_device *bdev)
775{
776 put_device(&bdev->bd_device);
777}
778
779static bool bdev_writes_blocked(struct block_device *bdev)
780{
781 return bdev->bd_writers < 0;
782}
783
784static void bdev_block_writes(struct block_device *bdev)
785{
786 bdev->bd_writers--;
787}
788
789static void bdev_unblock_writes(struct block_device *bdev)
790{
791 bdev->bd_writers++;
792}
793
794static bool bdev_may_open(struct block_device *bdev, blk_mode_t mode)
795{
796 if (bdev_allow_write_mounted)
797 return true;
798 /* Writes blocked? */
799 if (mode & BLK_OPEN_WRITE && bdev_writes_blocked(bdev))
800 return false;
801 if (mode & BLK_OPEN_RESTRICT_WRITES && bdev->bd_writers > 0)
802 return false;
803 return true;
804}
805
806static void bdev_claim_write_access(struct block_device *bdev, blk_mode_t mode)
807{
808 if (bdev_allow_write_mounted)
809 return;
810
811 /* Claim exclusive or shared write access. */
812 if (mode & BLK_OPEN_RESTRICT_WRITES)
813 bdev_block_writes(bdev);
814 else if (mode & BLK_OPEN_WRITE)
815 bdev->bd_writers++;
816}
817
818static inline bool bdev_unclaimed(const struct file *bdev_file)
819{
820 return bdev_file->private_data == BDEV_I(bdev_file->f_mapping->host);
821}
822
823static void bdev_yield_write_access(struct file *bdev_file)
824{
825 struct block_device *bdev;
826
827 if (bdev_allow_write_mounted)
828 return;
829
830 if (bdev_unclaimed(bdev_file))
831 return;
832
833 bdev = file_bdev(bdev_file);
834
835 if (bdev_file->f_mode & FMODE_WRITE_RESTRICTED)
836 bdev_unblock_writes(bdev);
837 else if (bdev_file->f_mode & FMODE_WRITE)
838 bdev->bd_writers--;
839}
840
841/**
842 * bdev_open - open a block device
843 * @bdev: block device to open
844 * @mode: open mode (BLK_OPEN_*)
845 * @holder: exclusive holder identifier
846 * @hops: holder operations
847 * @bdev_file: file for the block device
848 *
849 * Open the block device. If @holder is not %NULL, the block device is opened
850 * with exclusive access. Exclusive opens may nest for the same @holder.
851 *
852 * CONTEXT:
853 * Might sleep.
854 *
855 * RETURNS:
856 * zero on success, -errno on failure.
857 */
858int bdev_open(struct block_device *bdev, blk_mode_t mode, void *holder,
859 const struct blk_holder_ops *hops, struct file *bdev_file)
860{
861 bool unblock_events = true;
862 struct gendisk *disk = bdev->bd_disk;
863 int ret;
864
865 if (holder) {
866 mode |= BLK_OPEN_EXCL;
867 ret = bd_prepare_to_claim(bdev, holder, hops);
868 if (ret)
869 return ret;
870 } else {
871 if (WARN_ON_ONCE(mode & BLK_OPEN_EXCL))
872 return -EIO;
873 }
874
875 disk_block_events(disk);
876
877 mutex_lock(&disk->open_mutex);
878 ret = -ENXIO;
879 if (!disk_live(disk))
880 goto abort_claiming;
881 if (!try_module_get(disk->fops->owner))
882 goto abort_claiming;
883 ret = -EBUSY;
884 if (!bdev_may_open(bdev, mode))
885 goto put_module;
886 if (bdev_is_partition(bdev))
887 ret = blkdev_get_part(bdev, mode);
888 else
889 ret = blkdev_get_whole(bdev, mode);
890 if (ret)
891 goto put_module;
892 bdev_claim_write_access(bdev, mode);
893 if (holder) {
894 bd_finish_claiming(bdev, holder, hops);
895
896 /*
897 * Block event polling for write claims if requested. Any write
898 * holder makes the write_holder state stick until all are
899 * released. This is good enough and tracking individual
900 * writeable reference is too fragile given the way @mode is
901 * used in blkdev_get/put().
902 */
903 if ((mode & BLK_OPEN_WRITE) && !bdev->bd_write_holder &&
904 (disk->event_flags & DISK_EVENT_FLAG_BLOCK_ON_EXCL_WRITE)) {
905 bdev->bd_write_holder = true;
906 unblock_events = false;
907 }
908 }
909 mutex_unlock(&disk->open_mutex);
910
911 if (unblock_events)
912 disk_unblock_events(disk);
913
914 bdev_file->f_flags |= O_LARGEFILE;
915 bdev_file->f_mode |= FMODE_BUF_RASYNC | FMODE_CAN_ODIRECT;
916 if (bdev_nowait(bdev))
917 bdev_file->f_mode |= FMODE_NOWAIT;
918 if (mode & BLK_OPEN_RESTRICT_WRITES)
919 bdev_file->f_mode |= FMODE_WRITE_RESTRICTED;
920 bdev_file->f_mapping = bdev->bd_inode->i_mapping;
921 bdev_file->f_wb_err = filemap_sample_wb_err(bdev_file->f_mapping);
922 bdev_file->private_data = holder;
923
924 return 0;
925put_module:
926 module_put(disk->fops->owner);
927abort_claiming:
928 if (holder)
929 bd_abort_claiming(bdev, holder);
930 mutex_unlock(&disk->open_mutex);
931 disk_unblock_events(disk);
932 return ret;
933}
934
935/*
936 * If BLK_OPEN_WRITE_IOCTL is set then this is a historical quirk
937 * associated with the floppy driver where it has allowed ioctls if the
938 * file was opened for writing, but does not allow reads or writes.
939 * Make sure that this quirk is reflected in @f_flags.
940 *
941 * It can also happen if a block device is opened as O_RDWR | O_WRONLY.
942 */
943static unsigned blk_to_file_flags(blk_mode_t mode)
944{
945 unsigned int flags = 0;
946
947 if ((mode & (BLK_OPEN_READ | BLK_OPEN_WRITE)) ==
948 (BLK_OPEN_READ | BLK_OPEN_WRITE))
949 flags |= O_RDWR;
950 else if (mode & BLK_OPEN_WRITE_IOCTL)
951 flags |= O_RDWR | O_WRONLY;
952 else if (mode & BLK_OPEN_WRITE)
953 flags |= O_WRONLY;
954 else if (mode & BLK_OPEN_READ)
955 flags |= O_RDONLY; /* homeopathic, because O_RDONLY is 0 */
956 else
957 WARN_ON_ONCE(true);
958
959 if (mode & BLK_OPEN_NDELAY)
960 flags |= O_NDELAY;
961
962 return flags;
963}
964
965struct file *bdev_file_open_by_dev(dev_t dev, blk_mode_t mode, void *holder,
966 const struct blk_holder_ops *hops)
967{
968 struct file *bdev_file;
969 struct block_device *bdev;
970 unsigned int flags;
971 int ret;
972
973 ret = bdev_permission(dev, mode, holder);
974 if (ret)
975 return ERR_PTR(ret);
976
977 bdev = blkdev_get_no_open(dev);
978 if (!bdev)
979 return ERR_PTR(-ENXIO);
980
981 flags = blk_to_file_flags(mode);
982 bdev_file = alloc_file_pseudo_noaccount(bdev->bd_inode,
983 blockdev_mnt, "", flags | O_LARGEFILE, &def_blk_fops);
984 if (IS_ERR(bdev_file)) {
985 blkdev_put_no_open(bdev);
986 return bdev_file;
987 }
988 ihold(bdev->bd_inode);
989
990 ret = bdev_open(bdev, mode, holder, hops, bdev_file);
991 if (ret) {
992 /* We failed to open the block device. Let ->release() know. */
993 bdev_file->private_data = ERR_PTR(ret);
994 fput(bdev_file);
995 return ERR_PTR(ret);
996 }
997 return bdev_file;
998}
999EXPORT_SYMBOL(bdev_file_open_by_dev);
1000
1001struct file *bdev_file_open_by_path(const char *path, blk_mode_t mode,
1002 void *holder,
1003 const struct blk_holder_ops *hops)
1004{
1005 struct file *file;
1006 dev_t dev;
1007 int error;
1008
1009 error = lookup_bdev(path, &dev);
1010 if (error)
1011 return ERR_PTR(error);
1012
1013 file = bdev_file_open_by_dev(dev, mode, holder, hops);
1014 if (!IS_ERR(file) && (mode & BLK_OPEN_WRITE)) {
1015 if (bdev_read_only(file_bdev(file))) {
1016 fput(file);
1017 file = ERR_PTR(-EACCES);
1018 }
1019 }
1020
1021 return file;
1022}
1023EXPORT_SYMBOL(bdev_file_open_by_path);
1024
1025static inline void bd_yield_claim(struct file *bdev_file)
1026{
1027 struct block_device *bdev = file_bdev(bdev_file);
1028 void *holder = bdev_file->private_data;
1029
1030 lockdep_assert_held(&bdev->bd_disk->open_mutex);
1031
1032 if (WARN_ON_ONCE(IS_ERR_OR_NULL(holder)))
1033 return;
1034
1035 if (!bdev_unclaimed(bdev_file))
1036 bd_end_claim(bdev, holder);
1037}
1038
1039void bdev_release(struct file *bdev_file)
1040{
1041 struct block_device *bdev = file_bdev(bdev_file);
1042 void *holder = bdev_file->private_data;
1043 struct gendisk *disk = bdev->bd_disk;
1044
1045 /* We failed to open that block device. */
1046 if (IS_ERR(holder))
1047 goto put_no_open;
1048
1049 /*
1050 * Sync early if it looks like we're the last one. If someone else
1051 * opens the block device between now and the decrement of bd_openers
1052 * then we did a sync that we didn't need to, but that's not the end
1053 * of the world and we want to avoid long (could be several minute)
1054 * syncs while holding the mutex.
1055 */
1056 if (atomic_read(&bdev->bd_openers) == 1)
1057 sync_blockdev(bdev);
1058
1059 mutex_lock(&disk->open_mutex);
1060 bdev_yield_write_access(bdev_file);
1061
1062 if (holder)
1063 bd_yield_claim(bdev_file);
1064
1065 /*
1066 * Trigger event checking and tell drivers to flush MEDIA_CHANGE
1067 * event. This is to ensure detection of media removal commanded
1068 * from userland - e.g. eject(1).
1069 */
1070 disk_flush_events(disk, DISK_EVENT_MEDIA_CHANGE);
1071
1072 if (bdev_is_partition(bdev))
1073 blkdev_put_part(bdev);
1074 else
1075 blkdev_put_whole(bdev);
1076 mutex_unlock(&disk->open_mutex);
1077
1078 module_put(disk->fops->owner);
1079put_no_open:
1080 blkdev_put_no_open(bdev);
1081}
1082
1083/**
1084 * bdev_fput - yield claim to the block device and put the file
1085 * @bdev_file: open block device
1086 *
1087 * Yield claim on the block device and put the file. Ensure that the
1088 * block device can be reclaimed before the file is closed which is a
1089 * deferred operation.
1090 */
1091void bdev_fput(struct file *bdev_file)
1092{
1093 if (WARN_ON_ONCE(bdev_file->f_op != &def_blk_fops))
1094 return;
1095
1096 if (bdev_file->private_data) {
1097 struct block_device *bdev = file_bdev(bdev_file);
1098 struct gendisk *disk = bdev->bd_disk;
1099
1100 mutex_lock(&disk->open_mutex);
1101 bdev_yield_write_access(bdev_file);
1102 bd_yield_claim(bdev_file);
1103 /*
1104 * Tell release we already gave up our hold on the
1105 * device and if write restrictions are available that
1106 * we already gave up write access to the device.
1107 */
1108 bdev_file->private_data = BDEV_I(bdev_file->f_mapping->host);
1109 mutex_unlock(&disk->open_mutex);
1110 }
1111
1112 fput(bdev_file);
1113}
1114EXPORT_SYMBOL(bdev_fput);
1115
1116/**
1117 * lookup_bdev() - Look up a struct block_device by name.
1118 * @pathname: Name of the block device in the filesystem.
1119 * @dev: Pointer to the block device's dev_t, if found.
1120 *
1121 * Lookup the block device's dev_t at @pathname in the current
1122 * namespace if possible and return it in @dev.
1123 *
1124 * Context: May sleep.
1125 * Return: 0 if succeeded, negative errno otherwise.
1126 */
1127int lookup_bdev(const char *pathname, dev_t *dev)
1128{
1129 struct inode *inode;
1130 struct path path;
1131 int error;
1132
1133 if (!pathname || !*pathname)
1134 return -EINVAL;
1135
1136 error = kern_path(pathname, LOOKUP_FOLLOW, &path);
1137 if (error)
1138 return error;
1139
1140 inode = d_backing_inode(path.dentry);
1141 error = -ENOTBLK;
1142 if (!S_ISBLK(inode->i_mode))
1143 goto out_path_put;
1144 error = -EACCES;
1145 if (!may_open_dev(&path))
1146 goto out_path_put;
1147
1148 *dev = inode->i_rdev;
1149 error = 0;
1150out_path_put:
1151 path_put(&path);
1152 return error;
1153}
1154EXPORT_SYMBOL(lookup_bdev);
1155
1156/**
1157 * bdev_mark_dead - mark a block device as dead
1158 * @bdev: block device to operate on
1159 * @surprise: indicate a surprise removal
1160 *
1161 * Tell the file system that this devices or media is dead. If @surprise is set
1162 * to %true the device or media is already gone, if not we are preparing for an
1163 * orderly removal.
1164 *
1165 * This calls into the file system, which then typicall syncs out all dirty data
1166 * and writes back inodes and then invalidates any cached data in the inodes on
1167 * the file system. In addition we also invalidate the block device mapping.
1168 */
1169void bdev_mark_dead(struct block_device *bdev, bool surprise)
1170{
1171 mutex_lock(&bdev->bd_holder_lock);
1172 if (bdev->bd_holder_ops && bdev->bd_holder_ops->mark_dead)
1173 bdev->bd_holder_ops->mark_dead(bdev, surprise);
1174 else {
1175 mutex_unlock(&bdev->bd_holder_lock);
1176 sync_blockdev(bdev);
1177 }
1178
1179 invalidate_bdev(bdev);
1180}
1181/*
1182 * New drivers should not use this directly. There are some drivers however
1183 * that needs this for historical reasons. For example, the DASD driver has
1184 * historically had a shutdown to offline mode that doesn't actually remove the
1185 * gendisk that otherwise looks a lot like a safe device removal.
1186 */
1187EXPORT_SYMBOL_GPL(bdev_mark_dead);
1188
1189void sync_bdevs(bool wait)
1190{
1191 struct inode *inode, *old_inode = NULL;
1192
1193 spin_lock(&blockdev_superblock->s_inode_list_lock);
1194 list_for_each_entry(inode, &blockdev_superblock->s_inodes, i_sb_list) {
1195 struct address_space *mapping = inode->i_mapping;
1196 struct block_device *bdev;
1197
1198 spin_lock(&inode->i_lock);
1199 if (inode->i_state & (I_FREEING|I_WILL_FREE|I_NEW) ||
1200 mapping->nrpages == 0) {
1201 spin_unlock(&inode->i_lock);
1202 continue;
1203 }
1204 __iget(inode);
1205 spin_unlock(&inode->i_lock);
1206 spin_unlock(&blockdev_superblock->s_inode_list_lock);
1207 /*
1208 * We hold a reference to 'inode' so it couldn't have been
1209 * removed from s_inodes list while we dropped the
1210 * s_inode_list_lock We cannot iput the inode now as we can
1211 * be holding the last reference and we cannot iput it under
1212 * s_inode_list_lock. So we keep the reference and iput it
1213 * later.
1214 */
1215 iput(old_inode);
1216 old_inode = inode;
1217 bdev = I_BDEV(inode);
1218
1219 mutex_lock(&bdev->bd_disk->open_mutex);
1220 if (!atomic_read(&bdev->bd_openers)) {
1221 ; /* skip */
1222 } else if (wait) {
1223 /*
1224 * We keep the error status of individual mapping so
1225 * that applications can catch the writeback error using
1226 * fsync(2). See filemap_fdatawait_keep_errors() for
1227 * details.
1228 */
1229 filemap_fdatawait_keep_errors(inode->i_mapping);
1230 } else {
1231 filemap_fdatawrite(inode->i_mapping);
1232 }
1233 mutex_unlock(&bdev->bd_disk->open_mutex);
1234
1235 spin_lock(&blockdev_superblock->s_inode_list_lock);
1236 }
1237 spin_unlock(&blockdev_superblock->s_inode_list_lock);
1238 iput(old_inode);
1239}
1240
1241/*
1242 * Handle STATX_DIOALIGN for block devices.
1243 *
1244 * Note that the inode passed to this is the inode of a block device node file,
1245 * not the block device's internal inode. Therefore it is *not* valid to use
1246 * I_BDEV() here; the block device has to be looked up by i_rdev instead.
1247 */
1248void bdev_statx_dioalign(struct inode *inode, struct kstat *stat)
1249{
1250 struct block_device *bdev;
1251
1252 bdev = blkdev_get_no_open(inode->i_rdev);
1253 if (!bdev)
1254 return;
1255
1256 stat->dio_mem_align = bdev_dma_alignment(bdev) + 1;
1257 stat->dio_offset_align = bdev_logical_block_size(bdev);
1258 stat->result_mask |= STATX_DIOALIGN;
1259
1260 blkdev_put_no_open(bdev);
1261}
1262
1263static int __init setup_bdev_allow_write_mounted(char *str)
1264{
1265 if (kstrtobool(str, &bdev_allow_write_mounted))
1266 pr_warn("Invalid option string for bdev_allow_write_mounted:"
1267 " '%s'\n", str);
1268 return 1;
1269}
1270__setup("bdev_allow_write_mounted=", setup_bdev_allow_write_mounted);