Loading...
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"
9#include "dm-rq.h"
10#include "dm-uevent.h"
11
12#include <linux/init.h>
13#include <linux/module.h>
14#include <linux/mutex.h>
15#include <linux/blkpg.h>
16#include <linux/bio.h>
17#include <linux/mempool.h>
18#include <linux/slab.h>
19#include <linux/idr.h>
20#include <linux/hdreg.h>
21#include <linux/delay.h>
22#include <linux/wait.h>
23#include <linux/pr.h>
24
25#define DM_MSG_PREFIX "core"
26
27#ifdef CONFIG_PRINTK
28/*
29 * ratelimit state to be used in DMXXX_LIMIT().
30 */
31DEFINE_RATELIMIT_STATE(dm_ratelimit_state,
32 DEFAULT_RATELIMIT_INTERVAL,
33 DEFAULT_RATELIMIT_BURST);
34EXPORT_SYMBOL(dm_ratelimit_state);
35#endif
36
37/*
38 * Cookies are numeric values sent with CHANGE and REMOVE
39 * uevents while resuming, removing or renaming the device.
40 */
41#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
42#define DM_COOKIE_LENGTH 24
43
44static const char *_name = DM_NAME;
45
46static unsigned int major = 0;
47static unsigned int _major = 0;
48
49static DEFINE_IDR(_minor_idr);
50
51static DEFINE_SPINLOCK(_minor_lock);
52
53static void do_deferred_remove(struct work_struct *w);
54
55static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
56
57static struct workqueue_struct *deferred_remove_workqueue;
58
59/*
60 * One of these is allocated per bio.
61 */
62struct dm_io {
63 struct mapped_device *md;
64 int error;
65 atomic_t io_count;
66 struct bio *bio;
67 unsigned long start_time;
68 spinlock_t endio_lock;
69 struct dm_stats_aux stats_aux;
70};
71
72#define MINOR_ALLOCED ((void *)-1)
73
74/*
75 * Bits for the md->flags field.
76 */
77#define DMF_BLOCK_IO_FOR_SUSPEND 0
78#define DMF_SUSPENDED 1
79#define DMF_FROZEN 2
80#define DMF_FREEING 3
81#define DMF_DELETING 4
82#define DMF_NOFLUSH_SUSPENDING 5
83#define DMF_DEFERRED_REMOVE 6
84#define DMF_SUSPENDED_INTERNALLY 7
85
86#define DM_NUMA_NODE NUMA_NO_NODE
87static int dm_numa_node = DM_NUMA_NODE;
88
89/*
90 * For mempools pre-allocation at the table loading time.
91 */
92struct dm_md_mempools {
93 mempool_t *io_pool;
94 mempool_t *rq_pool;
95 struct bio_set *bs;
96};
97
98struct table_device {
99 struct list_head list;
100 atomic_t count;
101 struct dm_dev dm_dev;
102};
103
104static struct kmem_cache *_io_cache;
105static struct kmem_cache *_rq_tio_cache;
106static struct kmem_cache *_rq_cache;
107
108/*
109 * Bio-based DM's mempools' reserved IOs set by the user.
110 */
111#define RESERVED_BIO_BASED_IOS 16
112static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
113
114static int __dm_get_module_param_int(int *module_param, int min, int max)
115{
116 int param = ACCESS_ONCE(*module_param);
117 int modified_param = 0;
118 bool modified = true;
119
120 if (param < min)
121 modified_param = min;
122 else if (param > max)
123 modified_param = max;
124 else
125 modified = false;
126
127 if (modified) {
128 (void)cmpxchg(module_param, param, modified_param);
129 param = modified_param;
130 }
131
132 return param;
133}
134
135unsigned __dm_get_module_param(unsigned *module_param,
136 unsigned def, unsigned max)
137{
138 unsigned param = ACCESS_ONCE(*module_param);
139 unsigned modified_param = 0;
140
141 if (!param)
142 modified_param = def;
143 else if (param > max)
144 modified_param = max;
145
146 if (modified_param) {
147 (void)cmpxchg(module_param, param, modified_param);
148 param = modified_param;
149 }
150
151 return param;
152}
153
154unsigned dm_get_reserved_bio_based_ios(void)
155{
156 return __dm_get_module_param(&reserved_bio_based_ios,
157 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
158}
159EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
160
161static unsigned dm_get_numa_node(void)
162{
163 return __dm_get_module_param_int(&dm_numa_node,
164 DM_NUMA_NODE, num_online_nodes() - 1);
165}
166
167static int __init local_init(void)
168{
169 int r = -ENOMEM;
170
171 /* allocate a slab for the dm_ios */
172 _io_cache = KMEM_CACHE(dm_io, 0);
173 if (!_io_cache)
174 return r;
175
176 _rq_tio_cache = KMEM_CACHE(dm_rq_target_io, 0);
177 if (!_rq_tio_cache)
178 goto out_free_io_cache;
179
180 _rq_cache = kmem_cache_create("dm_old_clone_request", sizeof(struct request),
181 __alignof__(struct request), 0, NULL);
182 if (!_rq_cache)
183 goto out_free_rq_tio_cache;
184
185 r = dm_uevent_init();
186 if (r)
187 goto out_free_rq_cache;
188
189 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
190 if (!deferred_remove_workqueue) {
191 r = -ENOMEM;
192 goto out_uevent_exit;
193 }
194
195 _major = major;
196 r = register_blkdev(_major, _name);
197 if (r < 0)
198 goto out_free_workqueue;
199
200 if (!_major)
201 _major = r;
202
203 return 0;
204
205out_free_workqueue:
206 destroy_workqueue(deferred_remove_workqueue);
207out_uevent_exit:
208 dm_uevent_exit();
209out_free_rq_cache:
210 kmem_cache_destroy(_rq_cache);
211out_free_rq_tio_cache:
212 kmem_cache_destroy(_rq_tio_cache);
213out_free_io_cache:
214 kmem_cache_destroy(_io_cache);
215
216 return r;
217}
218
219static void local_exit(void)
220{
221 flush_scheduled_work();
222 destroy_workqueue(deferred_remove_workqueue);
223
224 kmem_cache_destroy(_rq_cache);
225 kmem_cache_destroy(_rq_tio_cache);
226 kmem_cache_destroy(_io_cache);
227 unregister_blkdev(_major, _name);
228 dm_uevent_exit();
229
230 _major = 0;
231
232 DMINFO("cleaned up");
233}
234
235static int (*_inits[])(void) __initdata = {
236 local_init,
237 dm_target_init,
238 dm_linear_init,
239 dm_stripe_init,
240 dm_io_init,
241 dm_kcopyd_init,
242 dm_interface_init,
243 dm_statistics_init,
244};
245
246static void (*_exits[])(void) = {
247 local_exit,
248 dm_target_exit,
249 dm_linear_exit,
250 dm_stripe_exit,
251 dm_io_exit,
252 dm_kcopyd_exit,
253 dm_interface_exit,
254 dm_statistics_exit,
255};
256
257static int __init dm_init(void)
258{
259 const int count = ARRAY_SIZE(_inits);
260
261 int r, i;
262
263 for (i = 0; i < count; i++) {
264 r = _inits[i]();
265 if (r)
266 goto bad;
267 }
268
269 return 0;
270
271 bad:
272 while (i--)
273 _exits[i]();
274
275 return r;
276}
277
278static void __exit dm_exit(void)
279{
280 int i = ARRAY_SIZE(_exits);
281
282 while (i--)
283 _exits[i]();
284
285 /*
286 * Should be empty by this point.
287 */
288 idr_destroy(&_minor_idr);
289}
290
291/*
292 * Block device functions
293 */
294int dm_deleting_md(struct mapped_device *md)
295{
296 return test_bit(DMF_DELETING, &md->flags);
297}
298
299static int dm_blk_open(struct block_device *bdev, fmode_t mode)
300{
301 struct mapped_device *md;
302
303 spin_lock(&_minor_lock);
304
305 md = bdev->bd_disk->private_data;
306 if (!md)
307 goto out;
308
309 if (test_bit(DMF_FREEING, &md->flags) ||
310 dm_deleting_md(md)) {
311 md = NULL;
312 goto out;
313 }
314
315 dm_get(md);
316 atomic_inc(&md->open_count);
317out:
318 spin_unlock(&_minor_lock);
319
320 return md ? 0 : -ENXIO;
321}
322
323static void dm_blk_close(struct gendisk *disk, fmode_t mode)
324{
325 struct mapped_device *md;
326
327 spin_lock(&_minor_lock);
328
329 md = disk->private_data;
330 if (WARN_ON(!md))
331 goto out;
332
333 if (atomic_dec_and_test(&md->open_count) &&
334 (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
335 queue_work(deferred_remove_workqueue, &deferred_remove_work);
336
337 dm_put(md);
338out:
339 spin_unlock(&_minor_lock);
340}
341
342int dm_open_count(struct mapped_device *md)
343{
344 return atomic_read(&md->open_count);
345}
346
347/*
348 * Guarantees nothing is using the device before it's deleted.
349 */
350int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
351{
352 int r = 0;
353
354 spin_lock(&_minor_lock);
355
356 if (dm_open_count(md)) {
357 r = -EBUSY;
358 if (mark_deferred)
359 set_bit(DMF_DEFERRED_REMOVE, &md->flags);
360 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
361 r = -EEXIST;
362 else
363 set_bit(DMF_DELETING, &md->flags);
364
365 spin_unlock(&_minor_lock);
366
367 return r;
368}
369
370int dm_cancel_deferred_remove(struct mapped_device *md)
371{
372 int r = 0;
373
374 spin_lock(&_minor_lock);
375
376 if (test_bit(DMF_DELETING, &md->flags))
377 r = -EBUSY;
378 else
379 clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
380
381 spin_unlock(&_minor_lock);
382
383 return r;
384}
385
386static void do_deferred_remove(struct work_struct *w)
387{
388 dm_deferred_remove();
389}
390
391sector_t dm_get_size(struct mapped_device *md)
392{
393 return get_capacity(md->disk);
394}
395
396struct request_queue *dm_get_md_queue(struct mapped_device *md)
397{
398 return md->queue;
399}
400
401struct dm_stats *dm_get_stats(struct mapped_device *md)
402{
403 return &md->stats;
404}
405
406static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
407{
408 struct mapped_device *md = bdev->bd_disk->private_data;
409
410 return dm_get_geometry(md, geo);
411}
412
413static int dm_grab_bdev_for_ioctl(struct mapped_device *md,
414 struct block_device **bdev,
415 fmode_t *mode)
416{
417 struct dm_target *tgt;
418 struct dm_table *map;
419 int srcu_idx, r;
420
421retry:
422 r = -ENOTTY;
423 map = dm_get_live_table(md, &srcu_idx);
424 if (!map || !dm_table_get_size(map))
425 goto out;
426
427 /* We only support devices that have a single target */
428 if (dm_table_get_num_targets(map) != 1)
429 goto out;
430
431 tgt = dm_table_get_target(map, 0);
432 if (!tgt->type->prepare_ioctl)
433 goto out;
434
435 if (dm_suspended_md(md)) {
436 r = -EAGAIN;
437 goto out;
438 }
439
440 r = tgt->type->prepare_ioctl(tgt, bdev, mode);
441 if (r < 0)
442 goto out;
443
444 bdgrab(*bdev);
445 dm_put_live_table(md, srcu_idx);
446 return r;
447
448out:
449 dm_put_live_table(md, srcu_idx);
450 if (r == -ENOTCONN && !fatal_signal_pending(current)) {
451 msleep(10);
452 goto retry;
453 }
454 return r;
455}
456
457static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
458 unsigned int cmd, unsigned long arg)
459{
460 struct mapped_device *md = bdev->bd_disk->private_data;
461 int r;
462
463 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
464 if (r < 0)
465 return r;
466
467 if (r > 0) {
468 /*
469 * Target determined this ioctl is being issued against
470 * a logical partition of the parent bdev; so extra
471 * validation is needed.
472 */
473 r = scsi_verify_blk_ioctl(NULL, cmd);
474 if (r)
475 goto out;
476 }
477
478 r = __blkdev_driver_ioctl(bdev, mode, cmd, arg);
479out:
480 bdput(bdev);
481 return r;
482}
483
484static struct dm_io *alloc_io(struct mapped_device *md)
485{
486 return mempool_alloc(md->io_pool, GFP_NOIO);
487}
488
489static void free_io(struct mapped_device *md, struct dm_io *io)
490{
491 mempool_free(io, md->io_pool);
492}
493
494static void free_tio(struct dm_target_io *tio)
495{
496 bio_put(&tio->clone);
497}
498
499int md_in_flight(struct mapped_device *md)
500{
501 return atomic_read(&md->pending[READ]) +
502 atomic_read(&md->pending[WRITE]);
503}
504
505static void start_io_acct(struct dm_io *io)
506{
507 struct mapped_device *md = io->md;
508 struct bio *bio = io->bio;
509 int cpu;
510 int rw = bio_data_dir(bio);
511
512 io->start_time = jiffies;
513
514 cpu = part_stat_lock();
515 part_round_stats(cpu, &dm_disk(md)->part0);
516 part_stat_unlock();
517 atomic_set(&dm_disk(md)->part0.in_flight[rw],
518 atomic_inc_return(&md->pending[rw]));
519
520 if (unlikely(dm_stats_used(&md->stats)))
521 dm_stats_account_io(&md->stats, bio_data_dir(bio),
522 bio->bi_iter.bi_sector, bio_sectors(bio),
523 false, 0, &io->stats_aux);
524}
525
526static void end_io_acct(struct dm_io *io)
527{
528 struct mapped_device *md = io->md;
529 struct bio *bio = io->bio;
530 unsigned long duration = jiffies - io->start_time;
531 int pending;
532 int rw = bio_data_dir(bio);
533
534 generic_end_io_acct(rw, &dm_disk(md)->part0, io->start_time);
535
536 if (unlikely(dm_stats_used(&md->stats)))
537 dm_stats_account_io(&md->stats, bio_data_dir(bio),
538 bio->bi_iter.bi_sector, bio_sectors(bio),
539 true, duration, &io->stats_aux);
540
541 /*
542 * After this is decremented the bio must not be touched if it is
543 * a flush.
544 */
545 pending = atomic_dec_return(&md->pending[rw]);
546 atomic_set(&dm_disk(md)->part0.in_flight[rw], pending);
547 pending += atomic_read(&md->pending[rw^0x1]);
548
549 /* nudge anyone waiting on suspend queue */
550 if (!pending)
551 wake_up(&md->wait);
552}
553
554/*
555 * Add the bio to the list of deferred io.
556 */
557static void queue_io(struct mapped_device *md, struct bio *bio)
558{
559 unsigned long flags;
560
561 spin_lock_irqsave(&md->deferred_lock, flags);
562 bio_list_add(&md->deferred, bio);
563 spin_unlock_irqrestore(&md->deferred_lock, flags);
564 queue_work(md->wq, &md->work);
565}
566
567/*
568 * Everyone (including functions in this file), should use this
569 * function to access the md->map field, and make sure they call
570 * dm_put_live_table() when finished.
571 */
572struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
573{
574 *srcu_idx = srcu_read_lock(&md->io_barrier);
575
576 return srcu_dereference(md->map, &md->io_barrier);
577}
578
579void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
580{
581 srcu_read_unlock(&md->io_barrier, srcu_idx);
582}
583
584void dm_sync_table(struct mapped_device *md)
585{
586 synchronize_srcu(&md->io_barrier);
587 synchronize_rcu_expedited();
588}
589
590/*
591 * A fast alternative to dm_get_live_table/dm_put_live_table.
592 * The caller must not block between these two functions.
593 */
594static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
595{
596 rcu_read_lock();
597 return rcu_dereference(md->map);
598}
599
600static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
601{
602 rcu_read_unlock();
603}
604
605/*
606 * Open a table device so we can use it as a map destination.
607 */
608static int open_table_device(struct table_device *td, dev_t dev,
609 struct mapped_device *md)
610{
611 static char *_claim_ptr = "I belong to device-mapper";
612 struct block_device *bdev;
613
614 int r;
615
616 BUG_ON(td->dm_dev.bdev);
617
618 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _claim_ptr);
619 if (IS_ERR(bdev))
620 return PTR_ERR(bdev);
621
622 r = bd_link_disk_holder(bdev, dm_disk(md));
623 if (r) {
624 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
625 return r;
626 }
627
628 td->dm_dev.bdev = bdev;
629 return 0;
630}
631
632/*
633 * Close a table device that we've been using.
634 */
635static void close_table_device(struct table_device *td, struct mapped_device *md)
636{
637 if (!td->dm_dev.bdev)
638 return;
639
640 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
641 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
642 td->dm_dev.bdev = NULL;
643}
644
645static struct table_device *find_table_device(struct list_head *l, dev_t dev,
646 fmode_t mode) {
647 struct table_device *td;
648
649 list_for_each_entry(td, l, list)
650 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
651 return td;
652
653 return NULL;
654}
655
656int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
657 struct dm_dev **result) {
658 int r;
659 struct table_device *td;
660
661 mutex_lock(&md->table_devices_lock);
662 td = find_table_device(&md->table_devices, dev, mode);
663 if (!td) {
664 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
665 if (!td) {
666 mutex_unlock(&md->table_devices_lock);
667 return -ENOMEM;
668 }
669
670 td->dm_dev.mode = mode;
671 td->dm_dev.bdev = NULL;
672
673 if ((r = open_table_device(td, dev, md))) {
674 mutex_unlock(&md->table_devices_lock);
675 kfree(td);
676 return r;
677 }
678
679 format_dev_t(td->dm_dev.name, dev);
680
681 atomic_set(&td->count, 0);
682 list_add(&td->list, &md->table_devices);
683 }
684 atomic_inc(&td->count);
685 mutex_unlock(&md->table_devices_lock);
686
687 *result = &td->dm_dev;
688 return 0;
689}
690EXPORT_SYMBOL_GPL(dm_get_table_device);
691
692void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
693{
694 struct table_device *td = container_of(d, struct table_device, dm_dev);
695
696 mutex_lock(&md->table_devices_lock);
697 if (atomic_dec_and_test(&td->count)) {
698 close_table_device(td, md);
699 list_del(&td->list);
700 kfree(td);
701 }
702 mutex_unlock(&md->table_devices_lock);
703}
704EXPORT_SYMBOL(dm_put_table_device);
705
706static void free_table_devices(struct list_head *devices)
707{
708 struct list_head *tmp, *next;
709
710 list_for_each_safe(tmp, next, devices) {
711 struct table_device *td = list_entry(tmp, struct table_device, list);
712
713 DMWARN("dm_destroy: %s still exists with %d references",
714 td->dm_dev.name, atomic_read(&td->count));
715 kfree(td);
716 }
717}
718
719/*
720 * Get the geometry associated with a dm device
721 */
722int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
723{
724 *geo = md->geometry;
725
726 return 0;
727}
728
729/*
730 * Set the geometry of a device.
731 */
732int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
733{
734 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
735
736 if (geo->start > sz) {
737 DMWARN("Start sector is beyond the geometry limits.");
738 return -EINVAL;
739 }
740
741 md->geometry = *geo;
742
743 return 0;
744}
745
746/*-----------------------------------------------------------------
747 * CRUD START:
748 * A more elegant soln is in the works that uses the queue
749 * merge fn, unfortunately there are a couple of changes to
750 * the block layer that I want to make for this. So in the
751 * interests of getting something for people to use I give
752 * you this clearly demarcated crap.
753 *---------------------------------------------------------------*/
754
755static int __noflush_suspending(struct mapped_device *md)
756{
757 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
758}
759
760/*
761 * Decrements the number of outstanding ios that a bio has been
762 * cloned into, completing the original io if necc.
763 */
764static void dec_pending(struct dm_io *io, int error)
765{
766 unsigned long flags;
767 int io_error;
768 struct bio *bio;
769 struct mapped_device *md = io->md;
770
771 /* Push-back supersedes any I/O errors */
772 if (unlikely(error)) {
773 spin_lock_irqsave(&io->endio_lock, flags);
774 if (!(io->error > 0 && __noflush_suspending(md)))
775 io->error = error;
776 spin_unlock_irqrestore(&io->endio_lock, flags);
777 }
778
779 if (atomic_dec_and_test(&io->io_count)) {
780 if (io->error == DM_ENDIO_REQUEUE) {
781 /*
782 * Target requested pushing back the I/O.
783 */
784 spin_lock_irqsave(&md->deferred_lock, flags);
785 if (__noflush_suspending(md))
786 bio_list_add_head(&md->deferred, io->bio);
787 else
788 /* noflush suspend was interrupted. */
789 io->error = -EIO;
790 spin_unlock_irqrestore(&md->deferred_lock, flags);
791 }
792
793 io_error = io->error;
794 bio = io->bio;
795 end_io_acct(io);
796 free_io(md, io);
797
798 if (io_error == DM_ENDIO_REQUEUE)
799 return;
800
801 if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
802 /*
803 * Preflush done for flush with data, reissue
804 * without REQ_PREFLUSH.
805 */
806 bio->bi_opf &= ~REQ_PREFLUSH;
807 queue_io(md, bio);
808 } else {
809 /* done with normal IO or empty flush */
810 trace_block_bio_complete(md->queue, bio, io_error);
811 bio->bi_error = io_error;
812 bio_endio(bio);
813 }
814 }
815}
816
817void disable_write_same(struct mapped_device *md)
818{
819 struct queue_limits *limits = dm_get_queue_limits(md);
820
821 /* device doesn't really support WRITE SAME, disable it */
822 limits->max_write_same_sectors = 0;
823}
824
825static void clone_endio(struct bio *bio)
826{
827 int error = bio->bi_error;
828 int r = error;
829 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
830 struct dm_io *io = tio->io;
831 struct mapped_device *md = tio->io->md;
832 dm_endio_fn endio = tio->ti->type->end_io;
833
834 if (endio) {
835 r = endio(tio->ti, bio, error);
836 if (r < 0 || r == DM_ENDIO_REQUEUE)
837 /*
838 * error and requeue request are handled
839 * in dec_pending().
840 */
841 error = r;
842 else if (r == DM_ENDIO_INCOMPLETE)
843 /* The target will handle the io */
844 return;
845 else if (r) {
846 DMWARN("unimplemented target endio return value: %d", r);
847 BUG();
848 }
849 }
850
851 if (unlikely(r == -EREMOTEIO && (bio_op(bio) == REQ_OP_WRITE_SAME) &&
852 !bdev_get_queue(bio->bi_bdev)->limits.max_write_same_sectors))
853 disable_write_same(md);
854
855 free_tio(tio);
856 dec_pending(io, error);
857}
858
859/*
860 * Return maximum size of I/O possible at the supplied sector up to the current
861 * target boundary.
862 */
863static sector_t max_io_len_target_boundary(sector_t sector, struct dm_target *ti)
864{
865 sector_t target_offset = dm_target_offset(ti, sector);
866
867 return ti->len - target_offset;
868}
869
870static sector_t max_io_len(sector_t sector, struct dm_target *ti)
871{
872 sector_t len = max_io_len_target_boundary(sector, ti);
873 sector_t offset, max_len;
874
875 /*
876 * Does the target need to split even further?
877 */
878 if (ti->max_io_len) {
879 offset = dm_target_offset(ti, sector);
880 if (unlikely(ti->max_io_len & (ti->max_io_len - 1)))
881 max_len = sector_div(offset, ti->max_io_len);
882 else
883 max_len = offset & (ti->max_io_len - 1);
884 max_len = ti->max_io_len - max_len;
885
886 if (len > max_len)
887 len = max_len;
888 }
889
890 return len;
891}
892
893int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
894{
895 if (len > UINT_MAX) {
896 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
897 (unsigned long long)len, UINT_MAX);
898 ti->error = "Maximum size of target IO is too large";
899 return -EINVAL;
900 }
901
902 ti->max_io_len = (uint32_t) len;
903
904 return 0;
905}
906EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
907
908static long dm_blk_direct_access(struct block_device *bdev, sector_t sector,
909 void **kaddr, pfn_t *pfn, long size)
910{
911 struct mapped_device *md = bdev->bd_disk->private_data;
912 struct dm_table *map;
913 struct dm_target *ti;
914 int srcu_idx;
915 long len, ret = -EIO;
916
917 map = dm_get_live_table(md, &srcu_idx);
918 if (!map)
919 goto out;
920
921 ti = dm_table_find_target(map, sector);
922 if (!dm_target_is_valid(ti))
923 goto out;
924
925 len = max_io_len(sector, ti) << SECTOR_SHIFT;
926 size = min(len, size);
927
928 if (ti->type->direct_access)
929 ret = ti->type->direct_access(ti, sector, kaddr, pfn, size);
930out:
931 dm_put_live_table(md, srcu_idx);
932 return min(ret, size);
933}
934
935/*
936 * A target may call dm_accept_partial_bio only from the map routine. It is
937 * allowed for all bio types except REQ_PREFLUSH.
938 *
939 * dm_accept_partial_bio informs the dm that the target only wants to process
940 * additional n_sectors sectors of the bio and the rest of the data should be
941 * sent in a next bio.
942 *
943 * A diagram that explains the arithmetics:
944 * +--------------------+---------------+-------+
945 * | 1 | 2 | 3 |
946 * +--------------------+---------------+-------+
947 *
948 * <-------------- *tio->len_ptr --------------->
949 * <------- bi_size ------->
950 * <-- n_sectors -->
951 *
952 * Region 1 was already iterated over with bio_advance or similar function.
953 * (it may be empty if the target doesn't use bio_advance)
954 * Region 2 is the remaining bio size that the target wants to process.
955 * (it may be empty if region 1 is non-empty, although there is no reason
956 * to make it empty)
957 * The target requires that region 3 is to be sent in the next bio.
958 *
959 * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
960 * the partially processed part (the sum of regions 1+2) must be the same for all
961 * copies of the bio.
962 */
963void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
964{
965 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
966 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
967 BUG_ON(bio->bi_opf & REQ_PREFLUSH);
968 BUG_ON(bi_size > *tio->len_ptr);
969 BUG_ON(n_sectors > bi_size);
970 *tio->len_ptr -= bi_size - n_sectors;
971 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
972}
973EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
974
975/*
976 * Flush current->bio_list when the target map method blocks.
977 * This fixes deadlocks in snapshot and possibly in other targets.
978 */
979struct dm_offload {
980 struct blk_plug plug;
981 struct blk_plug_cb cb;
982};
983
984static void flush_current_bio_list(struct blk_plug_cb *cb, bool from_schedule)
985{
986 struct dm_offload *o = container_of(cb, struct dm_offload, cb);
987 struct bio_list list;
988 struct bio *bio;
989 int i;
990
991 INIT_LIST_HEAD(&o->cb.list);
992
993 if (unlikely(!current->bio_list))
994 return;
995
996 for (i = 0; i < 2; i++) {
997 list = current->bio_list[i];
998 bio_list_init(¤t->bio_list[i]);
999
1000 while ((bio = bio_list_pop(&list))) {
1001 struct bio_set *bs = bio->bi_pool;
1002 if (unlikely(!bs) || bs == fs_bio_set) {
1003 bio_list_add(¤t->bio_list[i], bio);
1004 continue;
1005 }
1006
1007 spin_lock(&bs->rescue_lock);
1008 bio_list_add(&bs->rescue_list, bio);
1009 queue_work(bs->rescue_workqueue, &bs->rescue_work);
1010 spin_unlock(&bs->rescue_lock);
1011 }
1012 }
1013}
1014
1015static void dm_offload_start(struct dm_offload *o)
1016{
1017 blk_start_plug(&o->plug);
1018 o->cb.callback = flush_current_bio_list;
1019 list_add(&o->cb.list, ¤t->plug->cb_list);
1020}
1021
1022static void dm_offload_end(struct dm_offload *o)
1023{
1024 list_del(&o->cb.list);
1025 blk_finish_plug(&o->plug);
1026}
1027
1028static void __map_bio(struct dm_target_io *tio)
1029{
1030 int r;
1031 sector_t sector;
1032 struct dm_offload o;
1033 struct bio *clone = &tio->clone;
1034 struct dm_target *ti = tio->ti;
1035
1036 clone->bi_end_io = clone_endio;
1037
1038 /*
1039 * Map the clone. If r == 0 we don't need to do
1040 * anything, the target has assumed ownership of
1041 * this io.
1042 */
1043 atomic_inc(&tio->io->io_count);
1044 sector = clone->bi_iter.bi_sector;
1045
1046 dm_offload_start(&o);
1047 r = ti->type->map(ti, clone);
1048 dm_offload_end(&o);
1049
1050 if (r == DM_MAPIO_REMAPPED) {
1051 /* the bio has been remapped so dispatch it */
1052
1053 trace_block_bio_remap(bdev_get_queue(clone->bi_bdev), clone,
1054 tio->io->bio->bi_bdev->bd_dev, sector);
1055
1056 generic_make_request(clone);
1057 } else if (r < 0 || r == DM_MAPIO_REQUEUE) {
1058 /* error the io and bail out, or requeue it if needed */
1059 dec_pending(tio->io, r);
1060 free_tio(tio);
1061 } else if (r != DM_MAPIO_SUBMITTED) {
1062 DMWARN("unimplemented target map return value: %d", r);
1063 BUG();
1064 }
1065}
1066
1067struct clone_info {
1068 struct mapped_device *md;
1069 struct dm_table *map;
1070 struct bio *bio;
1071 struct dm_io *io;
1072 sector_t sector;
1073 unsigned sector_count;
1074};
1075
1076static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
1077{
1078 bio->bi_iter.bi_sector = sector;
1079 bio->bi_iter.bi_size = to_bytes(len);
1080}
1081
1082/*
1083 * Creates a bio that consists of range of complete bvecs.
1084 */
1085static int clone_bio(struct dm_target_io *tio, struct bio *bio,
1086 sector_t sector, unsigned len)
1087{
1088 struct bio *clone = &tio->clone;
1089
1090 __bio_clone_fast(clone, bio);
1091
1092 if (bio_integrity(bio)) {
1093 int r = bio_integrity_clone(clone, bio, GFP_NOIO);
1094 if (r < 0)
1095 return r;
1096 }
1097
1098 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1099 clone->bi_iter.bi_size = to_bytes(len);
1100
1101 if (bio_integrity(bio))
1102 bio_integrity_trim(clone, 0, len);
1103
1104 return 0;
1105}
1106
1107static struct dm_target_io *alloc_tio(struct clone_info *ci,
1108 struct dm_target *ti,
1109 unsigned target_bio_nr)
1110{
1111 struct dm_target_io *tio;
1112 struct bio *clone;
1113
1114 clone = bio_alloc_bioset(GFP_NOIO, 0, ci->md->bs);
1115 tio = container_of(clone, struct dm_target_io, clone);
1116
1117 tio->io = ci->io;
1118 tio->ti = ti;
1119 tio->target_bio_nr = target_bio_nr;
1120
1121 return tio;
1122}
1123
1124static void __clone_and_map_simple_bio(struct clone_info *ci,
1125 struct dm_target *ti,
1126 unsigned target_bio_nr, unsigned *len)
1127{
1128 struct dm_target_io *tio = alloc_tio(ci, ti, target_bio_nr);
1129 struct bio *clone = &tio->clone;
1130
1131 tio->len_ptr = len;
1132
1133 __bio_clone_fast(clone, ci->bio);
1134 if (len)
1135 bio_setup_sector(clone, ci->sector, *len);
1136
1137 __map_bio(tio);
1138}
1139
1140static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1141 unsigned num_bios, unsigned *len)
1142{
1143 unsigned target_bio_nr;
1144
1145 for (target_bio_nr = 0; target_bio_nr < num_bios; target_bio_nr++)
1146 __clone_and_map_simple_bio(ci, ti, target_bio_nr, len);
1147}
1148
1149static int __send_empty_flush(struct clone_info *ci)
1150{
1151 unsigned target_nr = 0;
1152 struct dm_target *ti;
1153
1154 BUG_ON(bio_has_data(ci->bio));
1155 while ((ti = dm_table_get_target(ci->map, target_nr++)))
1156 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
1157
1158 return 0;
1159}
1160
1161static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
1162 sector_t sector, unsigned *len)
1163{
1164 struct bio *bio = ci->bio;
1165 struct dm_target_io *tio;
1166 unsigned target_bio_nr;
1167 unsigned num_target_bios = 1;
1168 int r = 0;
1169
1170 /*
1171 * Does the target want to receive duplicate copies of the bio?
1172 */
1173 if (bio_data_dir(bio) == WRITE && ti->num_write_bios)
1174 num_target_bios = ti->num_write_bios(ti, bio);
1175
1176 for (target_bio_nr = 0; target_bio_nr < num_target_bios; target_bio_nr++) {
1177 tio = alloc_tio(ci, ti, target_bio_nr);
1178 tio->len_ptr = len;
1179 r = clone_bio(tio, bio, sector, *len);
1180 if (r < 0) {
1181 free_tio(tio);
1182 break;
1183 }
1184 __map_bio(tio);
1185 }
1186
1187 return r;
1188}
1189
1190typedef unsigned (*get_num_bios_fn)(struct dm_target *ti);
1191
1192static unsigned get_num_discard_bios(struct dm_target *ti)
1193{
1194 return ti->num_discard_bios;
1195}
1196
1197static unsigned get_num_write_same_bios(struct dm_target *ti)
1198{
1199 return ti->num_write_same_bios;
1200}
1201
1202typedef bool (*is_split_required_fn)(struct dm_target *ti);
1203
1204static bool is_split_required_for_discard(struct dm_target *ti)
1205{
1206 return ti->split_discard_bios;
1207}
1208
1209static int __send_changing_extent_only(struct clone_info *ci,
1210 get_num_bios_fn get_num_bios,
1211 is_split_required_fn is_split_required)
1212{
1213 struct dm_target *ti;
1214 unsigned len;
1215 unsigned num_bios;
1216
1217 do {
1218 ti = dm_table_find_target(ci->map, ci->sector);
1219 if (!dm_target_is_valid(ti))
1220 return -EIO;
1221
1222 /*
1223 * Even though the device advertised support for this type of
1224 * request, that does not mean every target supports it, and
1225 * reconfiguration might also have changed that since the
1226 * check was performed.
1227 */
1228 num_bios = get_num_bios ? get_num_bios(ti) : 0;
1229 if (!num_bios)
1230 return -EOPNOTSUPP;
1231
1232 if (is_split_required && !is_split_required(ti))
1233 len = min((sector_t)ci->sector_count, max_io_len_target_boundary(ci->sector, ti));
1234 else
1235 len = min((sector_t)ci->sector_count, max_io_len(ci->sector, ti));
1236
1237 __send_duplicate_bios(ci, ti, num_bios, &len);
1238
1239 ci->sector += len;
1240 } while (ci->sector_count -= len);
1241
1242 return 0;
1243}
1244
1245static int __send_discard(struct clone_info *ci)
1246{
1247 return __send_changing_extent_only(ci, get_num_discard_bios,
1248 is_split_required_for_discard);
1249}
1250
1251static int __send_write_same(struct clone_info *ci)
1252{
1253 return __send_changing_extent_only(ci, get_num_write_same_bios, NULL);
1254}
1255
1256/*
1257 * Select the correct strategy for processing a non-flush bio.
1258 */
1259static int __split_and_process_non_flush(struct clone_info *ci)
1260{
1261 struct bio *bio = ci->bio;
1262 struct dm_target *ti;
1263 unsigned len;
1264 int r;
1265
1266 if (unlikely(bio_op(bio) == REQ_OP_DISCARD))
1267 return __send_discard(ci);
1268 else if (unlikely(bio_op(bio) == REQ_OP_WRITE_SAME))
1269 return __send_write_same(ci);
1270
1271 ti = dm_table_find_target(ci->map, ci->sector);
1272 if (!dm_target_is_valid(ti))
1273 return -EIO;
1274
1275 len = min_t(sector_t, max_io_len(ci->sector, ti), ci->sector_count);
1276
1277 r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1278 if (r < 0)
1279 return r;
1280
1281 ci->sector += len;
1282 ci->sector_count -= len;
1283
1284 return 0;
1285}
1286
1287/*
1288 * Entry point to split a bio into clones and submit them to the targets.
1289 */
1290static void __split_and_process_bio(struct mapped_device *md,
1291 struct dm_table *map, struct bio *bio)
1292{
1293 struct clone_info ci;
1294 int error = 0;
1295
1296 if (unlikely(!map)) {
1297 bio_io_error(bio);
1298 return;
1299 }
1300
1301 ci.map = map;
1302 ci.md = md;
1303 ci.io = alloc_io(md);
1304 ci.io->error = 0;
1305 atomic_set(&ci.io->io_count, 1);
1306 ci.io->bio = bio;
1307 ci.io->md = md;
1308 spin_lock_init(&ci.io->endio_lock);
1309 ci.sector = bio->bi_iter.bi_sector;
1310
1311 start_io_acct(ci.io);
1312
1313 if (bio->bi_opf & REQ_PREFLUSH) {
1314 ci.bio = &ci.md->flush_bio;
1315 ci.sector_count = 0;
1316 error = __send_empty_flush(&ci);
1317 /* dec_pending submits any data associated with flush */
1318 } else {
1319 ci.bio = bio;
1320 ci.sector_count = bio_sectors(bio);
1321 while (ci.sector_count && !error)
1322 error = __split_and_process_non_flush(&ci);
1323 }
1324
1325 /* drop the extra reference count */
1326 dec_pending(ci.io, error);
1327}
1328/*-----------------------------------------------------------------
1329 * CRUD END
1330 *---------------------------------------------------------------*/
1331
1332/*
1333 * The request function that just remaps the bio built up by
1334 * dm_merge_bvec.
1335 */
1336static blk_qc_t dm_make_request(struct request_queue *q, struct bio *bio)
1337{
1338 int rw = bio_data_dir(bio);
1339 struct mapped_device *md = q->queuedata;
1340 int srcu_idx;
1341 struct dm_table *map;
1342
1343 map = dm_get_live_table(md, &srcu_idx);
1344
1345 generic_start_io_acct(rw, bio_sectors(bio), &dm_disk(md)->part0);
1346
1347 /* if we're suspended, we have to queue this io for later */
1348 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
1349 dm_put_live_table(md, srcu_idx);
1350
1351 if (!(bio->bi_opf & REQ_RAHEAD))
1352 queue_io(md, bio);
1353 else
1354 bio_io_error(bio);
1355 return BLK_QC_T_NONE;
1356 }
1357
1358 __split_and_process_bio(md, map, bio);
1359 dm_put_live_table(md, srcu_idx);
1360 return BLK_QC_T_NONE;
1361}
1362
1363static int dm_any_congested(void *congested_data, int bdi_bits)
1364{
1365 int r = bdi_bits;
1366 struct mapped_device *md = congested_data;
1367 struct dm_table *map;
1368
1369 if (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
1370 if (dm_request_based(md)) {
1371 /*
1372 * With request-based DM we only need to check the
1373 * top-level queue for congestion.
1374 */
1375 r = md->queue->backing_dev_info.wb.state & bdi_bits;
1376 } else {
1377 map = dm_get_live_table_fast(md);
1378 if (map)
1379 r = dm_table_any_congested(map, bdi_bits);
1380 dm_put_live_table_fast(md);
1381 }
1382 }
1383
1384 return r;
1385}
1386
1387/*-----------------------------------------------------------------
1388 * An IDR is used to keep track of allocated minor numbers.
1389 *---------------------------------------------------------------*/
1390static void free_minor(int minor)
1391{
1392 spin_lock(&_minor_lock);
1393 idr_remove(&_minor_idr, minor);
1394 spin_unlock(&_minor_lock);
1395}
1396
1397/*
1398 * See if the device with a specific minor # is free.
1399 */
1400static int specific_minor(int minor)
1401{
1402 int r;
1403
1404 if (minor >= (1 << MINORBITS))
1405 return -EINVAL;
1406
1407 idr_preload(GFP_KERNEL);
1408 spin_lock(&_minor_lock);
1409
1410 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1411
1412 spin_unlock(&_minor_lock);
1413 idr_preload_end();
1414 if (r < 0)
1415 return r == -ENOSPC ? -EBUSY : r;
1416 return 0;
1417}
1418
1419static int next_free_minor(int *minor)
1420{
1421 int r;
1422
1423 idr_preload(GFP_KERNEL);
1424 spin_lock(&_minor_lock);
1425
1426 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1427
1428 spin_unlock(&_minor_lock);
1429 idr_preload_end();
1430 if (r < 0)
1431 return r;
1432 *minor = r;
1433 return 0;
1434}
1435
1436static const struct block_device_operations dm_blk_dops;
1437
1438static void dm_wq_work(struct work_struct *work);
1439
1440void dm_init_md_queue(struct mapped_device *md)
1441{
1442 /*
1443 * Request-based dm devices cannot be stacked on top of bio-based dm
1444 * devices. The type of this dm device may not have been decided yet.
1445 * The type is decided at the first table loading time.
1446 * To prevent problematic device stacking, clear the queue flag
1447 * for request stacking support until then.
1448 *
1449 * This queue is new, so no concurrency on the queue_flags.
1450 */
1451 queue_flag_clear_unlocked(QUEUE_FLAG_STACKABLE, md->queue);
1452
1453 /*
1454 * Initialize data that will only be used by a non-blk-mq DM queue
1455 * - must do so here (in alloc_dev callchain) before queue is used
1456 */
1457 md->queue->queuedata = md;
1458 md->queue->backing_dev_info.congested_data = md;
1459}
1460
1461void dm_init_normal_md_queue(struct mapped_device *md)
1462{
1463 md->use_blk_mq = false;
1464 dm_init_md_queue(md);
1465
1466 /*
1467 * Initialize aspects of queue that aren't relevant for blk-mq
1468 */
1469 md->queue->backing_dev_info.congested_fn = dm_any_congested;
1470 blk_queue_bounce_limit(md->queue, BLK_BOUNCE_ANY);
1471}
1472
1473static void cleanup_mapped_device(struct mapped_device *md)
1474{
1475 if (md->wq)
1476 destroy_workqueue(md->wq);
1477 if (md->kworker_task)
1478 kthread_stop(md->kworker_task);
1479 mempool_destroy(md->io_pool);
1480 mempool_destroy(md->rq_pool);
1481 if (md->bs)
1482 bioset_free(md->bs);
1483
1484 if (md->disk) {
1485 spin_lock(&_minor_lock);
1486 md->disk->private_data = NULL;
1487 spin_unlock(&_minor_lock);
1488 del_gendisk(md->disk);
1489 put_disk(md->disk);
1490 }
1491
1492 if (md->queue)
1493 blk_cleanup_queue(md->queue);
1494
1495 cleanup_srcu_struct(&md->io_barrier);
1496
1497 if (md->bdev) {
1498 bdput(md->bdev);
1499 md->bdev = NULL;
1500 }
1501
1502 dm_mq_cleanup_mapped_device(md);
1503}
1504
1505/*
1506 * Allocate and initialise a blank device with a given minor.
1507 */
1508static struct mapped_device *alloc_dev(int minor)
1509{
1510 int r, numa_node_id = dm_get_numa_node();
1511 struct mapped_device *md;
1512 void *old_md;
1513
1514 md = kzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
1515 if (!md) {
1516 DMWARN("unable to allocate device, out of memory.");
1517 return NULL;
1518 }
1519
1520 if (!try_module_get(THIS_MODULE))
1521 goto bad_module_get;
1522
1523 /* get a minor number for the dev */
1524 if (minor == DM_ANY_MINOR)
1525 r = next_free_minor(&minor);
1526 else
1527 r = specific_minor(minor);
1528 if (r < 0)
1529 goto bad_minor;
1530
1531 r = init_srcu_struct(&md->io_barrier);
1532 if (r < 0)
1533 goto bad_io_barrier;
1534
1535 md->numa_node_id = numa_node_id;
1536 md->use_blk_mq = dm_use_blk_mq_default();
1537 md->init_tio_pdu = false;
1538 md->type = DM_TYPE_NONE;
1539 mutex_init(&md->suspend_lock);
1540 mutex_init(&md->type_lock);
1541 mutex_init(&md->table_devices_lock);
1542 spin_lock_init(&md->deferred_lock);
1543 atomic_set(&md->holders, 1);
1544 atomic_set(&md->open_count, 0);
1545 atomic_set(&md->event_nr, 0);
1546 atomic_set(&md->uevent_seq, 0);
1547 INIT_LIST_HEAD(&md->uevent_list);
1548 INIT_LIST_HEAD(&md->table_devices);
1549 spin_lock_init(&md->uevent_lock);
1550
1551 md->queue = blk_alloc_queue_node(GFP_KERNEL, numa_node_id);
1552 if (!md->queue)
1553 goto bad;
1554
1555 dm_init_md_queue(md);
1556
1557 md->disk = alloc_disk_node(1, numa_node_id);
1558 if (!md->disk)
1559 goto bad;
1560
1561 atomic_set(&md->pending[0], 0);
1562 atomic_set(&md->pending[1], 0);
1563 init_waitqueue_head(&md->wait);
1564 INIT_WORK(&md->work, dm_wq_work);
1565 init_waitqueue_head(&md->eventq);
1566 init_completion(&md->kobj_holder.completion);
1567 md->kworker_task = NULL;
1568
1569 md->disk->major = _major;
1570 md->disk->first_minor = minor;
1571 md->disk->fops = &dm_blk_dops;
1572 md->disk->queue = md->queue;
1573 md->disk->private_data = md;
1574 sprintf(md->disk->disk_name, "dm-%d", minor);
1575 add_disk(md->disk);
1576 format_dev_t(md->name, MKDEV(_major, minor));
1577
1578 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
1579 if (!md->wq)
1580 goto bad;
1581
1582 md->bdev = bdget_disk(md->disk, 0);
1583 if (!md->bdev)
1584 goto bad;
1585
1586 bio_init(&md->flush_bio, NULL, 0);
1587 md->flush_bio.bi_bdev = md->bdev;
1588 md->flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH;
1589
1590 dm_stats_init(&md->stats);
1591
1592 /* Populate the mapping, nobody knows we exist yet */
1593 spin_lock(&_minor_lock);
1594 old_md = idr_replace(&_minor_idr, md, minor);
1595 spin_unlock(&_minor_lock);
1596
1597 BUG_ON(old_md != MINOR_ALLOCED);
1598
1599 return md;
1600
1601bad:
1602 cleanup_mapped_device(md);
1603bad_io_barrier:
1604 free_minor(minor);
1605bad_minor:
1606 module_put(THIS_MODULE);
1607bad_module_get:
1608 kfree(md);
1609 return NULL;
1610}
1611
1612static void unlock_fs(struct mapped_device *md);
1613
1614static void free_dev(struct mapped_device *md)
1615{
1616 int minor = MINOR(disk_devt(md->disk));
1617
1618 unlock_fs(md);
1619
1620 cleanup_mapped_device(md);
1621
1622 free_table_devices(&md->table_devices);
1623 dm_stats_cleanup(&md->stats);
1624 free_minor(minor);
1625
1626 module_put(THIS_MODULE);
1627 kfree(md);
1628}
1629
1630static void __bind_mempools(struct mapped_device *md, struct dm_table *t)
1631{
1632 struct dm_md_mempools *p = dm_table_get_md_mempools(t);
1633
1634 if (md->bs) {
1635 /* The md already has necessary mempools. */
1636 if (dm_table_bio_based(t)) {
1637 /*
1638 * Reload bioset because front_pad may have changed
1639 * because a different table was loaded.
1640 */
1641 bioset_free(md->bs);
1642 md->bs = p->bs;
1643 p->bs = NULL;
1644 }
1645 /*
1646 * There's no need to reload with request-based dm
1647 * because the size of front_pad doesn't change.
1648 * Note for future: If you are to reload bioset,
1649 * prep-ed requests in the queue may refer
1650 * to bio from the old bioset, so you must walk
1651 * through the queue to unprep.
1652 */
1653 goto out;
1654 }
1655
1656 BUG_ON(!p || md->io_pool || md->rq_pool || md->bs);
1657
1658 md->io_pool = p->io_pool;
1659 p->io_pool = NULL;
1660 md->rq_pool = p->rq_pool;
1661 p->rq_pool = NULL;
1662 md->bs = p->bs;
1663 p->bs = NULL;
1664
1665out:
1666 /* mempool bind completed, no longer need any mempools in the table */
1667 dm_table_free_md_mempools(t);
1668}
1669
1670/*
1671 * Bind a table to the device.
1672 */
1673static void event_callback(void *context)
1674{
1675 unsigned long flags;
1676 LIST_HEAD(uevents);
1677 struct mapped_device *md = (struct mapped_device *) context;
1678
1679 spin_lock_irqsave(&md->uevent_lock, flags);
1680 list_splice_init(&md->uevent_list, &uevents);
1681 spin_unlock_irqrestore(&md->uevent_lock, flags);
1682
1683 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
1684
1685 atomic_inc(&md->event_nr);
1686 wake_up(&md->eventq);
1687}
1688
1689/*
1690 * Protected by md->suspend_lock obtained by dm_swap_table().
1691 */
1692static void __set_size(struct mapped_device *md, sector_t size)
1693{
1694 set_capacity(md->disk, size);
1695
1696 i_size_write(md->bdev->bd_inode, (loff_t)size << SECTOR_SHIFT);
1697}
1698
1699/*
1700 * Returns old map, which caller must destroy.
1701 */
1702static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
1703 struct queue_limits *limits)
1704{
1705 struct dm_table *old_map;
1706 struct request_queue *q = md->queue;
1707 sector_t size;
1708
1709 lockdep_assert_held(&md->suspend_lock);
1710
1711 size = dm_table_get_size(t);
1712
1713 /*
1714 * Wipe any geometry if the size of the table changed.
1715 */
1716 if (size != dm_get_size(md))
1717 memset(&md->geometry, 0, sizeof(md->geometry));
1718
1719 __set_size(md, size);
1720
1721 dm_table_event_callback(t, event_callback, md);
1722
1723 /*
1724 * The queue hasn't been stopped yet, if the old table type wasn't
1725 * for request-based during suspension. So stop it to prevent
1726 * I/O mapping before resume.
1727 * This must be done before setting the queue restrictions,
1728 * because request-based dm may be run just after the setting.
1729 */
1730 if (dm_table_request_based(t)) {
1731 dm_stop_queue(q);
1732 /*
1733 * Leverage the fact that request-based DM targets are
1734 * immutable singletons and establish md->immutable_target
1735 * - used to optimize both dm_request_fn and dm_mq_queue_rq
1736 */
1737 md->immutable_target = dm_table_get_immutable_target(t);
1738 }
1739
1740 __bind_mempools(md, t);
1741
1742 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
1743 rcu_assign_pointer(md->map, (void *)t);
1744 md->immutable_target_type = dm_table_get_immutable_target_type(t);
1745
1746 dm_table_set_restrictions(t, q, limits);
1747 if (old_map)
1748 dm_sync_table(md);
1749
1750 return old_map;
1751}
1752
1753/*
1754 * Returns unbound table for the caller to free.
1755 */
1756static struct dm_table *__unbind(struct mapped_device *md)
1757{
1758 struct dm_table *map = rcu_dereference_protected(md->map, 1);
1759
1760 if (!map)
1761 return NULL;
1762
1763 dm_table_event_callback(map, NULL, NULL);
1764 RCU_INIT_POINTER(md->map, NULL);
1765 dm_sync_table(md);
1766
1767 return map;
1768}
1769
1770/*
1771 * Constructor for a new device.
1772 */
1773int dm_create(int minor, struct mapped_device **result)
1774{
1775 struct mapped_device *md;
1776
1777 md = alloc_dev(minor);
1778 if (!md)
1779 return -ENXIO;
1780
1781 dm_sysfs_init(md);
1782
1783 *result = md;
1784 return 0;
1785}
1786
1787/*
1788 * Functions to manage md->type.
1789 * All are required to hold md->type_lock.
1790 */
1791void dm_lock_md_type(struct mapped_device *md)
1792{
1793 mutex_lock(&md->type_lock);
1794}
1795
1796void dm_unlock_md_type(struct mapped_device *md)
1797{
1798 mutex_unlock(&md->type_lock);
1799}
1800
1801void dm_set_md_type(struct mapped_device *md, unsigned type)
1802{
1803 BUG_ON(!mutex_is_locked(&md->type_lock));
1804 md->type = type;
1805}
1806
1807unsigned dm_get_md_type(struct mapped_device *md)
1808{
1809 return md->type;
1810}
1811
1812struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
1813{
1814 return md->immutable_target_type;
1815}
1816
1817/*
1818 * The queue_limits are only valid as long as you have a reference
1819 * count on 'md'.
1820 */
1821struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
1822{
1823 BUG_ON(!atomic_read(&md->holders));
1824 return &md->queue->limits;
1825}
1826EXPORT_SYMBOL_GPL(dm_get_queue_limits);
1827
1828/*
1829 * Setup the DM device's queue based on md's type
1830 */
1831int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
1832{
1833 int r;
1834 unsigned type = dm_get_md_type(md);
1835
1836 switch (type) {
1837 case DM_TYPE_REQUEST_BASED:
1838 r = dm_old_init_request_queue(md);
1839 if (r) {
1840 DMERR("Cannot initialize queue for request-based mapped device");
1841 return r;
1842 }
1843 break;
1844 case DM_TYPE_MQ_REQUEST_BASED:
1845 r = dm_mq_init_request_queue(md, t);
1846 if (r) {
1847 DMERR("Cannot initialize queue for request-based dm-mq mapped device");
1848 return r;
1849 }
1850 break;
1851 case DM_TYPE_BIO_BASED:
1852 case DM_TYPE_DAX_BIO_BASED:
1853 dm_init_normal_md_queue(md);
1854 blk_queue_make_request(md->queue, dm_make_request);
1855 /*
1856 * DM handles splitting bios as needed. Free the bio_split bioset
1857 * since it won't be used (saves 1 process per bio-based DM device).
1858 */
1859 bioset_free(md->queue->bio_split);
1860 md->queue->bio_split = NULL;
1861
1862 if (type == DM_TYPE_DAX_BIO_BASED)
1863 queue_flag_set_unlocked(QUEUE_FLAG_DAX, md->queue);
1864 break;
1865 }
1866
1867 return 0;
1868}
1869
1870struct mapped_device *dm_get_md(dev_t dev)
1871{
1872 struct mapped_device *md;
1873 unsigned minor = MINOR(dev);
1874
1875 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
1876 return NULL;
1877
1878 spin_lock(&_minor_lock);
1879
1880 md = idr_find(&_minor_idr, minor);
1881 if (md) {
1882 if ((md == MINOR_ALLOCED ||
1883 (MINOR(disk_devt(dm_disk(md))) != minor) ||
1884 dm_deleting_md(md) ||
1885 test_bit(DMF_FREEING, &md->flags))) {
1886 md = NULL;
1887 goto out;
1888 }
1889 dm_get(md);
1890 }
1891
1892out:
1893 spin_unlock(&_minor_lock);
1894
1895 return md;
1896}
1897EXPORT_SYMBOL_GPL(dm_get_md);
1898
1899void *dm_get_mdptr(struct mapped_device *md)
1900{
1901 return md->interface_ptr;
1902}
1903
1904void dm_set_mdptr(struct mapped_device *md, void *ptr)
1905{
1906 md->interface_ptr = ptr;
1907}
1908
1909void dm_get(struct mapped_device *md)
1910{
1911 atomic_inc(&md->holders);
1912 BUG_ON(test_bit(DMF_FREEING, &md->flags));
1913}
1914
1915int dm_hold(struct mapped_device *md)
1916{
1917 spin_lock(&_minor_lock);
1918 if (test_bit(DMF_FREEING, &md->flags)) {
1919 spin_unlock(&_minor_lock);
1920 return -EBUSY;
1921 }
1922 dm_get(md);
1923 spin_unlock(&_minor_lock);
1924 return 0;
1925}
1926EXPORT_SYMBOL_GPL(dm_hold);
1927
1928const char *dm_device_name(struct mapped_device *md)
1929{
1930 return md->name;
1931}
1932EXPORT_SYMBOL_GPL(dm_device_name);
1933
1934static void __dm_destroy(struct mapped_device *md, bool wait)
1935{
1936 struct request_queue *q = dm_get_md_queue(md);
1937 struct dm_table *map;
1938 int srcu_idx;
1939
1940 might_sleep();
1941
1942 spin_lock(&_minor_lock);
1943 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
1944 set_bit(DMF_FREEING, &md->flags);
1945 spin_unlock(&_minor_lock);
1946
1947 blk_set_queue_dying(q);
1948
1949 if (dm_request_based(md) && md->kworker_task)
1950 kthread_flush_worker(&md->kworker);
1951
1952 /*
1953 * Take suspend_lock so that presuspend and postsuspend methods
1954 * do not race with internal suspend.
1955 */
1956 mutex_lock(&md->suspend_lock);
1957 map = dm_get_live_table(md, &srcu_idx);
1958 if (!dm_suspended_md(md)) {
1959 dm_table_presuspend_targets(map);
1960 dm_table_postsuspend_targets(map);
1961 }
1962 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
1963 dm_put_live_table(md, srcu_idx);
1964 mutex_unlock(&md->suspend_lock);
1965
1966 /*
1967 * Rare, but there may be I/O requests still going to complete,
1968 * for example. Wait for all references to disappear.
1969 * No one should increment the reference count of the mapped_device,
1970 * after the mapped_device state becomes DMF_FREEING.
1971 */
1972 if (wait)
1973 while (atomic_read(&md->holders))
1974 msleep(1);
1975 else if (atomic_read(&md->holders))
1976 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
1977 dm_device_name(md), atomic_read(&md->holders));
1978
1979 dm_sysfs_exit(md);
1980 dm_table_destroy(__unbind(md));
1981 free_dev(md);
1982}
1983
1984void dm_destroy(struct mapped_device *md)
1985{
1986 __dm_destroy(md, true);
1987}
1988
1989void dm_destroy_immediate(struct mapped_device *md)
1990{
1991 __dm_destroy(md, false);
1992}
1993
1994void dm_put(struct mapped_device *md)
1995{
1996 atomic_dec(&md->holders);
1997}
1998EXPORT_SYMBOL_GPL(dm_put);
1999
2000static int dm_wait_for_completion(struct mapped_device *md, long task_state)
2001{
2002 int r = 0;
2003 DEFINE_WAIT(wait);
2004
2005 while (1) {
2006 prepare_to_wait(&md->wait, &wait, task_state);
2007
2008 if (!md_in_flight(md))
2009 break;
2010
2011 if (signal_pending_state(task_state, current)) {
2012 r = -EINTR;
2013 break;
2014 }
2015
2016 io_schedule();
2017 }
2018 finish_wait(&md->wait, &wait);
2019
2020 return r;
2021}
2022
2023/*
2024 * Process the deferred bios
2025 */
2026static void dm_wq_work(struct work_struct *work)
2027{
2028 struct mapped_device *md = container_of(work, struct mapped_device,
2029 work);
2030 struct bio *c;
2031 int srcu_idx;
2032 struct dm_table *map;
2033
2034 map = dm_get_live_table(md, &srcu_idx);
2035
2036 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2037 spin_lock_irq(&md->deferred_lock);
2038 c = bio_list_pop(&md->deferred);
2039 spin_unlock_irq(&md->deferred_lock);
2040
2041 if (!c)
2042 break;
2043
2044 if (dm_request_based(md))
2045 generic_make_request(c);
2046 else
2047 __split_and_process_bio(md, map, c);
2048 }
2049
2050 dm_put_live_table(md, srcu_idx);
2051}
2052
2053static void dm_queue_flush(struct mapped_device *md)
2054{
2055 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2056 smp_mb__after_atomic();
2057 queue_work(md->wq, &md->work);
2058}
2059
2060/*
2061 * Swap in a new table, returning the old one for the caller to destroy.
2062 */
2063struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2064{
2065 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2066 struct queue_limits limits;
2067 int r;
2068
2069 mutex_lock(&md->suspend_lock);
2070
2071 /* device must be suspended */
2072 if (!dm_suspended_md(md))
2073 goto out;
2074
2075 /*
2076 * If the new table has no data devices, retain the existing limits.
2077 * This helps multipath with queue_if_no_path if all paths disappear,
2078 * then new I/O is queued based on these limits, and then some paths
2079 * reappear.
2080 */
2081 if (dm_table_has_no_data_devices(table)) {
2082 live_map = dm_get_live_table_fast(md);
2083 if (live_map)
2084 limits = md->queue->limits;
2085 dm_put_live_table_fast(md);
2086 }
2087
2088 if (!live_map) {
2089 r = dm_calculate_queue_limits(table, &limits);
2090 if (r) {
2091 map = ERR_PTR(r);
2092 goto out;
2093 }
2094 }
2095
2096 map = __bind(md, table, &limits);
2097
2098out:
2099 mutex_unlock(&md->suspend_lock);
2100 return map;
2101}
2102
2103/*
2104 * Functions to lock and unlock any filesystem running on the
2105 * device.
2106 */
2107static int lock_fs(struct mapped_device *md)
2108{
2109 int r;
2110
2111 WARN_ON(md->frozen_sb);
2112
2113 md->frozen_sb = freeze_bdev(md->bdev);
2114 if (IS_ERR(md->frozen_sb)) {
2115 r = PTR_ERR(md->frozen_sb);
2116 md->frozen_sb = NULL;
2117 return r;
2118 }
2119
2120 set_bit(DMF_FROZEN, &md->flags);
2121
2122 return 0;
2123}
2124
2125static void unlock_fs(struct mapped_device *md)
2126{
2127 if (!test_bit(DMF_FROZEN, &md->flags))
2128 return;
2129
2130 thaw_bdev(md->bdev, md->frozen_sb);
2131 md->frozen_sb = NULL;
2132 clear_bit(DMF_FROZEN, &md->flags);
2133}
2134
2135/*
2136 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2137 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2138 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2139 *
2140 * If __dm_suspend returns 0, the device is completely quiescent
2141 * now. There is no request-processing activity. All new requests
2142 * are being added to md->deferred list.
2143 *
2144 * Caller must hold md->suspend_lock
2145 */
2146static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2147 unsigned suspend_flags, long task_state,
2148 int dmf_suspended_flag)
2149{
2150 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2151 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2152 int r;
2153
2154 lockdep_assert_held(&md->suspend_lock);
2155
2156 /*
2157 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2158 * This flag is cleared before dm_suspend returns.
2159 */
2160 if (noflush)
2161 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2162
2163 /*
2164 * This gets reverted if there's an error later and the targets
2165 * provide the .presuspend_undo hook.
2166 */
2167 dm_table_presuspend_targets(map);
2168
2169 /*
2170 * Flush I/O to the device.
2171 * Any I/O submitted after lock_fs() may not be flushed.
2172 * noflush takes precedence over do_lockfs.
2173 * (lock_fs() flushes I/Os and waits for them to complete.)
2174 */
2175 if (!noflush && do_lockfs) {
2176 r = lock_fs(md);
2177 if (r) {
2178 dm_table_presuspend_undo_targets(map);
2179 return r;
2180 }
2181 }
2182
2183 /*
2184 * Here we must make sure that no processes are submitting requests
2185 * to target drivers i.e. no one may be executing
2186 * __split_and_process_bio. This is called from dm_request and
2187 * dm_wq_work.
2188 *
2189 * To get all processes out of __split_and_process_bio in dm_request,
2190 * we take the write lock. To prevent any process from reentering
2191 * __split_and_process_bio from dm_request and quiesce the thread
2192 * (dm_wq_work), we set BMF_BLOCK_IO_FOR_SUSPEND and call
2193 * flush_workqueue(md->wq).
2194 */
2195 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2196 if (map)
2197 synchronize_srcu(&md->io_barrier);
2198
2199 /*
2200 * Stop md->queue before flushing md->wq in case request-based
2201 * dm defers requests to md->wq from md->queue.
2202 */
2203 if (dm_request_based(md)) {
2204 dm_stop_queue(md->queue);
2205 if (md->kworker_task)
2206 kthread_flush_worker(&md->kworker);
2207 }
2208
2209 flush_workqueue(md->wq);
2210
2211 /*
2212 * At this point no more requests are entering target request routines.
2213 * We call dm_wait_for_completion to wait for all existing requests
2214 * to finish.
2215 */
2216 r = dm_wait_for_completion(md, task_state);
2217 if (!r)
2218 set_bit(dmf_suspended_flag, &md->flags);
2219
2220 if (noflush)
2221 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2222 if (map)
2223 synchronize_srcu(&md->io_barrier);
2224
2225 /* were we interrupted ? */
2226 if (r < 0) {
2227 dm_queue_flush(md);
2228
2229 if (dm_request_based(md))
2230 dm_start_queue(md->queue);
2231
2232 unlock_fs(md);
2233 dm_table_presuspend_undo_targets(map);
2234 /* pushback list is already flushed, so skip flush */
2235 }
2236
2237 return r;
2238}
2239
2240/*
2241 * We need to be able to change a mapping table under a mounted
2242 * filesystem. For example we might want to move some data in
2243 * the background. Before the table can be swapped with
2244 * dm_bind_table, dm_suspend must be called to flush any in
2245 * flight bios and ensure that any further io gets deferred.
2246 */
2247/*
2248 * Suspend mechanism in request-based dm.
2249 *
2250 * 1. Flush all I/Os by lock_fs() if needed.
2251 * 2. Stop dispatching any I/O by stopping the request_queue.
2252 * 3. Wait for all in-flight I/Os to be completed or requeued.
2253 *
2254 * To abort suspend, start the request_queue.
2255 */
2256int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2257{
2258 struct dm_table *map = NULL;
2259 int r = 0;
2260
2261retry:
2262 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2263
2264 if (dm_suspended_md(md)) {
2265 r = -EINVAL;
2266 goto out_unlock;
2267 }
2268
2269 if (dm_suspended_internally_md(md)) {
2270 /* already internally suspended, wait for internal resume */
2271 mutex_unlock(&md->suspend_lock);
2272 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2273 if (r)
2274 return r;
2275 goto retry;
2276 }
2277
2278 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2279
2280 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
2281 if (r)
2282 goto out_unlock;
2283
2284 dm_table_postsuspend_targets(map);
2285
2286out_unlock:
2287 mutex_unlock(&md->suspend_lock);
2288 return r;
2289}
2290
2291static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2292{
2293 if (map) {
2294 int r = dm_table_resume_targets(map);
2295 if (r)
2296 return r;
2297 }
2298
2299 dm_queue_flush(md);
2300
2301 /*
2302 * Flushing deferred I/Os must be done after targets are resumed
2303 * so that mapping of targets can work correctly.
2304 * Request-based dm is queueing the deferred I/Os in its request_queue.
2305 */
2306 if (dm_request_based(md))
2307 dm_start_queue(md->queue);
2308
2309 unlock_fs(md);
2310
2311 return 0;
2312}
2313
2314int dm_resume(struct mapped_device *md)
2315{
2316 int r;
2317 struct dm_table *map = NULL;
2318
2319retry:
2320 r = -EINVAL;
2321 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2322
2323 if (!dm_suspended_md(md))
2324 goto out;
2325
2326 if (dm_suspended_internally_md(md)) {
2327 /* already internally suspended, wait for internal resume */
2328 mutex_unlock(&md->suspend_lock);
2329 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2330 if (r)
2331 return r;
2332 goto retry;
2333 }
2334
2335 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2336 if (!map || !dm_table_get_size(map))
2337 goto out;
2338
2339 r = __dm_resume(md, map);
2340 if (r)
2341 goto out;
2342
2343 clear_bit(DMF_SUSPENDED, &md->flags);
2344out:
2345 mutex_unlock(&md->suspend_lock);
2346
2347 return r;
2348}
2349
2350/*
2351 * Internal suspend/resume works like userspace-driven suspend. It waits
2352 * until all bios finish and prevents issuing new bios to the target drivers.
2353 * It may be used only from the kernel.
2354 */
2355
2356static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2357{
2358 struct dm_table *map = NULL;
2359
2360 if (md->internal_suspend_count++)
2361 return; /* nested internal suspend */
2362
2363 if (dm_suspended_md(md)) {
2364 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2365 return; /* nest suspend */
2366 }
2367
2368 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2369
2370 /*
2371 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2372 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
2373 * would require changing .presuspend to return an error -- avoid this
2374 * until there is a need for more elaborate variants of internal suspend.
2375 */
2376 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2377 DMF_SUSPENDED_INTERNALLY);
2378
2379 dm_table_postsuspend_targets(map);
2380}
2381
2382static void __dm_internal_resume(struct mapped_device *md)
2383{
2384 BUG_ON(!md->internal_suspend_count);
2385
2386 if (--md->internal_suspend_count)
2387 return; /* resume from nested internal suspend */
2388
2389 if (dm_suspended_md(md))
2390 goto done; /* resume from nested suspend */
2391
2392 /*
2393 * NOTE: existing callers don't need to call dm_table_resume_targets
2394 * (which may fail -- so best to avoid it for now by passing NULL map)
2395 */
2396 (void) __dm_resume(md, NULL);
2397
2398done:
2399 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2400 smp_mb__after_atomic();
2401 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2402}
2403
2404void dm_internal_suspend_noflush(struct mapped_device *md)
2405{
2406 mutex_lock(&md->suspend_lock);
2407 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2408 mutex_unlock(&md->suspend_lock);
2409}
2410EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2411
2412void dm_internal_resume(struct mapped_device *md)
2413{
2414 mutex_lock(&md->suspend_lock);
2415 __dm_internal_resume(md);
2416 mutex_unlock(&md->suspend_lock);
2417}
2418EXPORT_SYMBOL_GPL(dm_internal_resume);
2419
2420/*
2421 * Fast variants of internal suspend/resume hold md->suspend_lock,
2422 * which prevents interaction with userspace-driven suspend.
2423 */
2424
2425void dm_internal_suspend_fast(struct mapped_device *md)
2426{
2427 mutex_lock(&md->suspend_lock);
2428 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2429 return;
2430
2431 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2432 synchronize_srcu(&md->io_barrier);
2433 flush_workqueue(md->wq);
2434 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2435}
2436EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
2437
2438void dm_internal_resume_fast(struct mapped_device *md)
2439{
2440 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2441 goto done;
2442
2443 dm_queue_flush(md);
2444
2445done:
2446 mutex_unlock(&md->suspend_lock);
2447}
2448EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
2449
2450/*-----------------------------------------------------------------
2451 * Event notification.
2452 *---------------------------------------------------------------*/
2453int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
2454 unsigned cookie)
2455{
2456 char udev_cookie[DM_COOKIE_LENGTH];
2457 char *envp[] = { udev_cookie, NULL };
2458
2459 if (!cookie)
2460 return kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
2461 else {
2462 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2463 DM_COOKIE_ENV_VAR_NAME, cookie);
2464 return kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2465 action, envp);
2466 }
2467}
2468
2469uint32_t dm_next_uevent_seq(struct mapped_device *md)
2470{
2471 return atomic_add_return(1, &md->uevent_seq);
2472}
2473
2474uint32_t dm_get_event_nr(struct mapped_device *md)
2475{
2476 return atomic_read(&md->event_nr);
2477}
2478
2479int dm_wait_event(struct mapped_device *md, int event_nr)
2480{
2481 return wait_event_interruptible(md->eventq,
2482 (event_nr != atomic_read(&md->event_nr)));
2483}
2484
2485void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2486{
2487 unsigned long flags;
2488
2489 spin_lock_irqsave(&md->uevent_lock, flags);
2490 list_add(elist, &md->uevent_list);
2491 spin_unlock_irqrestore(&md->uevent_lock, flags);
2492}
2493
2494/*
2495 * The gendisk is only valid as long as you have a reference
2496 * count on 'md'.
2497 */
2498struct gendisk *dm_disk(struct mapped_device *md)
2499{
2500 return md->disk;
2501}
2502EXPORT_SYMBOL_GPL(dm_disk);
2503
2504struct kobject *dm_kobject(struct mapped_device *md)
2505{
2506 return &md->kobj_holder.kobj;
2507}
2508
2509struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2510{
2511 struct mapped_device *md;
2512
2513 md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
2514
2515 if (test_bit(DMF_FREEING, &md->flags) ||
2516 dm_deleting_md(md))
2517 return NULL;
2518
2519 dm_get(md);
2520 return md;
2521}
2522
2523int dm_suspended_md(struct mapped_device *md)
2524{
2525 return test_bit(DMF_SUSPENDED, &md->flags);
2526}
2527
2528int dm_suspended_internally_md(struct mapped_device *md)
2529{
2530 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2531}
2532
2533int dm_test_deferred_remove_flag(struct mapped_device *md)
2534{
2535 return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2536}
2537
2538int dm_suspended(struct dm_target *ti)
2539{
2540 return dm_suspended_md(dm_table_get_md(ti->table));
2541}
2542EXPORT_SYMBOL_GPL(dm_suspended);
2543
2544int dm_noflush_suspending(struct dm_target *ti)
2545{
2546 return __noflush_suspending(dm_table_get_md(ti->table));
2547}
2548EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2549
2550struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, unsigned type,
2551 unsigned integrity, unsigned per_io_data_size)
2552{
2553 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
2554 struct kmem_cache *cachep = NULL;
2555 unsigned int pool_size = 0;
2556 unsigned int front_pad;
2557
2558 if (!pools)
2559 return NULL;
2560
2561 switch (type) {
2562 case DM_TYPE_BIO_BASED:
2563 case DM_TYPE_DAX_BIO_BASED:
2564 cachep = _io_cache;
2565 pool_size = dm_get_reserved_bio_based_ios();
2566 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + offsetof(struct dm_target_io, clone);
2567 break;
2568 case DM_TYPE_REQUEST_BASED:
2569 cachep = _rq_tio_cache;
2570 pool_size = dm_get_reserved_rq_based_ios();
2571 pools->rq_pool = mempool_create_slab_pool(pool_size, _rq_cache);
2572 if (!pools->rq_pool)
2573 goto out;
2574 /* fall through to setup remaining rq-based pools */
2575 case DM_TYPE_MQ_REQUEST_BASED:
2576 if (!pool_size)
2577 pool_size = dm_get_reserved_rq_based_ios();
2578 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2579 /* per_io_data_size is used for blk-mq pdu at queue allocation */
2580 break;
2581 default:
2582 BUG();
2583 }
2584
2585 if (cachep) {
2586 pools->io_pool = mempool_create_slab_pool(pool_size, cachep);
2587 if (!pools->io_pool)
2588 goto out;
2589 }
2590
2591 pools->bs = bioset_create_nobvec(pool_size, front_pad);
2592 if (!pools->bs)
2593 goto out;
2594
2595 if (integrity && bioset_integrity_create(pools->bs, pool_size))
2596 goto out;
2597
2598 return pools;
2599
2600out:
2601 dm_free_md_mempools(pools);
2602
2603 return NULL;
2604}
2605
2606void dm_free_md_mempools(struct dm_md_mempools *pools)
2607{
2608 if (!pools)
2609 return;
2610
2611 mempool_destroy(pools->io_pool);
2612 mempool_destroy(pools->rq_pool);
2613
2614 if (pools->bs)
2615 bioset_free(pools->bs);
2616
2617 kfree(pools);
2618}
2619
2620struct dm_pr {
2621 u64 old_key;
2622 u64 new_key;
2623 u32 flags;
2624 bool fail_early;
2625};
2626
2627static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
2628 void *data)
2629{
2630 struct mapped_device *md = bdev->bd_disk->private_data;
2631 struct dm_table *table;
2632 struct dm_target *ti;
2633 int ret = -ENOTTY, srcu_idx;
2634
2635 table = dm_get_live_table(md, &srcu_idx);
2636 if (!table || !dm_table_get_size(table))
2637 goto out;
2638
2639 /* We only support devices that have a single target */
2640 if (dm_table_get_num_targets(table) != 1)
2641 goto out;
2642 ti = dm_table_get_target(table, 0);
2643
2644 ret = -EINVAL;
2645 if (!ti->type->iterate_devices)
2646 goto out;
2647
2648 ret = ti->type->iterate_devices(ti, fn, data);
2649out:
2650 dm_put_live_table(md, srcu_idx);
2651 return ret;
2652}
2653
2654/*
2655 * For register / unregister we need to manually call out to every path.
2656 */
2657static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
2658 sector_t start, sector_t len, void *data)
2659{
2660 struct dm_pr *pr = data;
2661 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
2662
2663 if (!ops || !ops->pr_register)
2664 return -EOPNOTSUPP;
2665 return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
2666}
2667
2668static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
2669 u32 flags)
2670{
2671 struct dm_pr pr = {
2672 .old_key = old_key,
2673 .new_key = new_key,
2674 .flags = flags,
2675 .fail_early = true,
2676 };
2677 int ret;
2678
2679 ret = dm_call_pr(bdev, __dm_pr_register, &pr);
2680 if (ret && new_key) {
2681 /* unregister all paths if we failed to register any path */
2682 pr.old_key = new_key;
2683 pr.new_key = 0;
2684 pr.flags = 0;
2685 pr.fail_early = false;
2686 dm_call_pr(bdev, __dm_pr_register, &pr);
2687 }
2688
2689 return ret;
2690}
2691
2692static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
2693 u32 flags)
2694{
2695 struct mapped_device *md = bdev->bd_disk->private_data;
2696 const struct pr_ops *ops;
2697 fmode_t mode;
2698 int r;
2699
2700 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
2701 if (r < 0)
2702 return r;
2703
2704 ops = bdev->bd_disk->fops->pr_ops;
2705 if (ops && ops->pr_reserve)
2706 r = ops->pr_reserve(bdev, key, type, flags);
2707 else
2708 r = -EOPNOTSUPP;
2709
2710 bdput(bdev);
2711 return r;
2712}
2713
2714static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2715{
2716 struct mapped_device *md = bdev->bd_disk->private_data;
2717 const struct pr_ops *ops;
2718 fmode_t mode;
2719 int r;
2720
2721 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
2722 if (r < 0)
2723 return r;
2724
2725 ops = bdev->bd_disk->fops->pr_ops;
2726 if (ops && ops->pr_release)
2727 r = ops->pr_release(bdev, key, type);
2728 else
2729 r = -EOPNOTSUPP;
2730
2731 bdput(bdev);
2732 return r;
2733}
2734
2735static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
2736 enum pr_type type, bool abort)
2737{
2738 struct mapped_device *md = bdev->bd_disk->private_data;
2739 const struct pr_ops *ops;
2740 fmode_t mode;
2741 int r;
2742
2743 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
2744 if (r < 0)
2745 return r;
2746
2747 ops = bdev->bd_disk->fops->pr_ops;
2748 if (ops && ops->pr_preempt)
2749 r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
2750 else
2751 r = -EOPNOTSUPP;
2752
2753 bdput(bdev);
2754 return r;
2755}
2756
2757static int dm_pr_clear(struct block_device *bdev, u64 key)
2758{
2759 struct mapped_device *md = bdev->bd_disk->private_data;
2760 const struct pr_ops *ops;
2761 fmode_t mode;
2762 int r;
2763
2764 r = dm_grab_bdev_for_ioctl(md, &bdev, &mode);
2765 if (r < 0)
2766 return r;
2767
2768 ops = bdev->bd_disk->fops->pr_ops;
2769 if (ops && ops->pr_clear)
2770 r = ops->pr_clear(bdev, key);
2771 else
2772 r = -EOPNOTSUPP;
2773
2774 bdput(bdev);
2775 return r;
2776}
2777
2778static const struct pr_ops dm_pr_ops = {
2779 .pr_register = dm_pr_register,
2780 .pr_reserve = dm_pr_reserve,
2781 .pr_release = dm_pr_release,
2782 .pr_preempt = dm_pr_preempt,
2783 .pr_clear = dm_pr_clear,
2784};
2785
2786static const struct block_device_operations dm_blk_dops = {
2787 .open = dm_blk_open,
2788 .release = dm_blk_close,
2789 .ioctl = dm_blk_ioctl,
2790 .direct_access = dm_blk_direct_access,
2791 .getgeo = dm_blk_getgeo,
2792 .pr_ops = &dm_pr_ops,
2793 .owner = THIS_MODULE
2794};
2795
2796/*
2797 * module hooks
2798 */
2799module_init(dm_init);
2800module_exit(dm_exit);
2801
2802module_param(major, uint, 0);
2803MODULE_PARM_DESC(major, "The major number of the device mapper");
2804
2805module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
2806MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
2807
2808module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
2809MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
2810
2811MODULE_DESCRIPTION(DM_NAME " driver");
2812MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
2813MODULE_LICENSE("GPL");
1/*
2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-core.h"
9#include "dm-rq.h"
10#include "dm-uevent.h"
11
12#include <linux/init.h>
13#include <linux/module.h>
14#include <linux/mutex.h>
15#include <linux/sched/mm.h>
16#include <linux/sched/signal.h>
17#include <linux/blkpg.h>
18#include <linux/bio.h>
19#include <linux/mempool.h>
20#include <linux/dax.h>
21#include <linux/slab.h>
22#include <linux/idr.h>
23#include <linux/uio.h>
24#include <linux/hdreg.h>
25#include <linux/delay.h>
26#include <linux/wait.h>
27#include <linux/pr.h>
28#include <linux/refcount.h>
29#include <linux/part_stat.h>
30#include <linux/blk-crypto.h>
31#include <linux/keyslot-manager.h>
32
33#define DM_MSG_PREFIX "core"
34
35/*
36 * Cookies are numeric values sent with CHANGE and REMOVE
37 * uevents while resuming, removing or renaming the device.
38 */
39#define DM_COOKIE_ENV_VAR_NAME "DM_COOKIE"
40#define DM_COOKIE_LENGTH 24
41
42static const char *_name = DM_NAME;
43
44static unsigned int major = 0;
45static unsigned int _major = 0;
46
47static DEFINE_IDR(_minor_idr);
48
49static DEFINE_SPINLOCK(_minor_lock);
50
51static void do_deferred_remove(struct work_struct *w);
52
53static DECLARE_WORK(deferred_remove_work, do_deferred_remove);
54
55static struct workqueue_struct *deferred_remove_workqueue;
56
57atomic_t dm_global_event_nr = ATOMIC_INIT(0);
58DECLARE_WAIT_QUEUE_HEAD(dm_global_eventq);
59
60void dm_issue_global_event(void)
61{
62 atomic_inc(&dm_global_event_nr);
63 wake_up(&dm_global_eventq);
64}
65
66/*
67 * One of these is allocated (on-stack) per original bio.
68 */
69struct clone_info {
70 struct dm_table *map;
71 struct bio *bio;
72 struct dm_io *io;
73 sector_t sector;
74 unsigned sector_count;
75};
76
77#define DM_TARGET_IO_BIO_OFFSET (offsetof(struct dm_target_io, clone))
78#define DM_IO_BIO_OFFSET \
79 (offsetof(struct dm_target_io, clone) + offsetof(struct dm_io, tio))
80
81void *dm_per_bio_data(struct bio *bio, size_t data_size)
82{
83 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
84 if (!tio->inside_dm_io)
85 return (char *)bio - DM_TARGET_IO_BIO_OFFSET - data_size;
86 return (char *)bio - DM_IO_BIO_OFFSET - data_size;
87}
88EXPORT_SYMBOL_GPL(dm_per_bio_data);
89
90struct bio *dm_bio_from_per_bio_data(void *data, size_t data_size)
91{
92 struct dm_io *io = (struct dm_io *)((char *)data + data_size);
93 if (io->magic == DM_IO_MAGIC)
94 return (struct bio *)((char *)io + DM_IO_BIO_OFFSET);
95 BUG_ON(io->magic != DM_TIO_MAGIC);
96 return (struct bio *)((char *)io + DM_TARGET_IO_BIO_OFFSET);
97}
98EXPORT_SYMBOL_GPL(dm_bio_from_per_bio_data);
99
100unsigned dm_bio_get_target_bio_nr(const struct bio *bio)
101{
102 return container_of(bio, struct dm_target_io, clone)->target_bio_nr;
103}
104EXPORT_SYMBOL_GPL(dm_bio_get_target_bio_nr);
105
106#define MINOR_ALLOCED ((void *)-1)
107
108#define DM_NUMA_NODE NUMA_NO_NODE
109static int dm_numa_node = DM_NUMA_NODE;
110
111#define DEFAULT_SWAP_BIOS (8 * 1048576 / PAGE_SIZE)
112static int swap_bios = DEFAULT_SWAP_BIOS;
113static int get_swap_bios(void)
114{
115 int latch = READ_ONCE(swap_bios);
116 if (unlikely(latch <= 0))
117 latch = DEFAULT_SWAP_BIOS;
118 return latch;
119}
120
121/*
122 * For mempools pre-allocation at the table loading time.
123 */
124struct dm_md_mempools {
125 struct bio_set bs;
126 struct bio_set io_bs;
127};
128
129struct table_device {
130 struct list_head list;
131 refcount_t count;
132 struct dm_dev dm_dev;
133};
134
135/*
136 * Bio-based DM's mempools' reserved IOs set by the user.
137 */
138#define RESERVED_BIO_BASED_IOS 16
139static unsigned reserved_bio_based_ios = RESERVED_BIO_BASED_IOS;
140
141static int __dm_get_module_param_int(int *module_param, int min, int max)
142{
143 int param = READ_ONCE(*module_param);
144 int modified_param = 0;
145 bool modified = true;
146
147 if (param < min)
148 modified_param = min;
149 else if (param > max)
150 modified_param = max;
151 else
152 modified = false;
153
154 if (modified) {
155 (void)cmpxchg(module_param, param, modified_param);
156 param = modified_param;
157 }
158
159 return param;
160}
161
162unsigned __dm_get_module_param(unsigned *module_param,
163 unsigned def, unsigned max)
164{
165 unsigned param = READ_ONCE(*module_param);
166 unsigned modified_param = 0;
167
168 if (!param)
169 modified_param = def;
170 else if (param > max)
171 modified_param = max;
172
173 if (modified_param) {
174 (void)cmpxchg(module_param, param, modified_param);
175 param = modified_param;
176 }
177
178 return param;
179}
180
181unsigned dm_get_reserved_bio_based_ios(void)
182{
183 return __dm_get_module_param(&reserved_bio_based_ios,
184 RESERVED_BIO_BASED_IOS, DM_RESERVED_MAX_IOS);
185}
186EXPORT_SYMBOL_GPL(dm_get_reserved_bio_based_ios);
187
188static unsigned dm_get_numa_node(void)
189{
190 return __dm_get_module_param_int(&dm_numa_node,
191 DM_NUMA_NODE, num_online_nodes() - 1);
192}
193
194static int __init local_init(void)
195{
196 int r;
197
198 r = dm_uevent_init();
199 if (r)
200 return r;
201
202 deferred_remove_workqueue = alloc_workqueue("kdmremove", WQ_UNBOUND, 1);
203 if (!deferred_remove_workqueue) {
204 r = -ENOMEM;
205 goto out_uevent_exit;
206 }
207
208 _major = major;
209 r = register_blkdev(_major, _name);
210 if (r < 0)
211 goto out_free_workqueue;
212
213 if (!_major)
214 _major = r;
215
216 return 0;
217
218out_free_workqueue:
219 destroy_workqueue(deferred_remove_workqueue);
220out_uevent_exit:
221 dm_uevent_exit();
222
223 return r;
224}
225
226static void local_exit(void)
227{
228 flush_scheduled_work();
229 destroy_workqueue(deferred_remove_workqueue);
230
231 unregister_blkdev(_major, _name);
232 dm_uevent_exit();
233
234 _major = 0;
235
236 DMINFO("cleaned up");
237}
238
239static int (*_inits[])(void) __initdata = {
240 local_init,
241 dm_target_init,
242 dm_linear_init,
243 dm_stripe_init,
244 dm_io_init,
245 dm_kcopyd_init,
246 dm_interface_init,
247 dm_statistics_init,
248};
249
250static void (*_exits[])(void) = {
251 local_exit,
252 dm_target_exit,
253 dm_linear_exit,
254 dm_stripe_exit,
255 dm_io_exit,
256 dm_kcopyd_exit,
257 dm_interface_exit,
258 dm_statistics_exit,
259};
260
261static int __init dm_init(void)
262{
263 const int count = ARRAY_SIZE(_inits);
264
265 int r, i;
266
267 for (i = 0; i < count; i++) {
268 r = _inits[i]();
269 if (r)
270 goto bad;
271 }
272
273 return 0;
274
275 bad:
276 while (i--)
277 _exits[i]();
278
279 return r;
280}
281
282static void __exit dm_exit(void)
283{
284 int i = ARRAY_SIZE(_exits);
285
286 while (i--)
287 _exits[i]();
288
289 /*
290 * Should be empty by this point.
291 */
292 idr_destroy(&_minor_idr);
293}
294
295/*
296 * Block device functions
297 */
298int dm_deleting_md(struct mapped_device *md)
299{
300 return test_bit(DMF_DELETING, &md->flags);
301}
302
303static int dm_blk_open(struct block_device *bdev, fmode_t mode)
304{
305 struct mapped_device *md;
306
307 spin_lock(&_minor_lock);
308
309 md = bdev->bd_disk->private_data;
310 if (!md)
311 goto out;
312
313 if (test_bit(DMF_FREEING, &md->flags) ||
314 dm_deleting_md(md)) {
315 md = NULL;
316 goto out;
317 }
318
319 dm_get(md);
320 atomic_inc(&md->open_count);
321out:
322 spin_unlock(&_minor_lock);
323
324 return md ? 0 : -ENXIO;
325}
326
327static void dm_blk_close(struct gendisk *disk, fmode_t mode)
328{
329 struct mapped_device *md;
330
331 spin_lock(&_minor_lock);
332
333 md = disk->private_data;
334 if (WARN_ON(!md))
335 goto out;
336
337 if (atomic_dec_and_test(&md->open_count) &&
338 (test_bit(DMF_DEFERRED_REMOVE, &md->flags)))
339 queue_work(deferred_remove_workqueue, &deferred_remove_work);
340
341 dm_put(md);
342out:
343 spin_unlock(&_minor_lock);
344}
345
346int dm_open_count(struct mapped_device *md)
347{
348 return atomic_read(&md->open_count);
349}
350
351/*
352 * Guarantees nothing is using the device before it's deleted.
353 */
354int dm_lock_for_deletion(struct mapped_device *md, bool mark_deferred, bool only_deferred)
355{
356 int r = 0;
357
358 spin_lock(&_minor_lock);
359
360 if (dm_open_count(md)) {
361 r = -EBUSY;
362 if (mark_deferred)
363 set_bit(DMF_DEFERRED_REMOVE, &md->flags);
364 } else if (only_deferred && !test_bit(DMF_DEFERRED_REMOVE, &md->flags))
365 r = -EEXIST;
366 else
367 set_bit(DMF_DELETING, &md->flags);
368
369 spin_unlock(&_minor_lock);
370
371 return r;
372}
373
374int dm_cancel_deferred_remove(struct mapped_device *md)
375{
376 int r = 0;
377
378 spin_lock(&_minor_lock);
379
380 if (test_bit(DMF_DELETING, &md->flags))
381 r = -EBUSY;
382 else
383 clear_bit(DMF_DEFERRED_REMOVE, &md->flags);
384
385 spin_unlock(&_minor_lock);
386
387 return r;
388}
389
390static void do_deferred_remove(struct work_struct *w)
391{
392 dm_deferred_remove();
393}
394
395static int dm_blk_getgeo(struct block_device *bdev, struct hd_geometry *geo)
396{
397 struct mapped_device *md = bdev->bd_disk->private_data;
398
399 return dm_get_geometry(md, geo);
400}
401
402static int dm_prepare_ioctl(struct mapped_device *md, int *srcu_idx,
403 struct block_device **bdev)
404{
405 struct dm_target *tgt;
406 struct dm_table *map;
407 int r;
408
409retry:
410 r = -ENOTTY;
411 map = dm_get_live_table(md, srcu_idx);
412 if (!map || !dm_table_get_size(map))
413 return r;
414
415 /* We only support devices that have a single target */
416 if (dm_table_get_num_targets(map) != 1)
417 return r;
418
419 tgt = dm_table_get_target(map, 0);
420 if (!tgt->type->prepare_ioctl)
421 return r;
422
423 if (dm_suspended_md(md))
424 return -EAGAIN;
425
426 r = tgt->type->prepare_ioctl(tgt, bdev);
427 if (r == -ENOTCONN && !fatal_signal_pending(current)) {
428 dm_put_live_table(md, *srcu_idx);
429 msleep(10);
430 goto retry;
431 }
432
433 return r;
434}
435
436static void dm_unprepare_ioctl(struct mapped_device *md, int srcu_idx)
437{
438 dm_put_live_table(md, srcu_idx);
439}
440
441static int dm_blk_ioctl(struct block_device *bdev, fmode_t mode,
442 unsigned int cmd, unsigned long arg)
443{
444 struct mapped_device *md = bdev->bd_disk->private_data;
445 int r, srcu_idx;
446
447 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
448 if (r < 0)
449 goto out;
450
451 if (r > 0) {
452 /*
453 * Target determined this ioctl is being issued against a
454 * subset of the parent bdev; require extra privileges.
455 */
456 if (!capable(CAP_SYS_RAWIO)) {
457 DMDEBUG_LIMIT(
458 "%s: sending ioctl %x to DM device without required privilege.",
459 current->comm, cmd);
460 r = -ENOIOCTLCMD;
461 goto out;
462 }
463 }
464
465 if (!bdev->bd_disk->fops->ioctl)
466 r = -ENOTTY;
467 else
468 r = bdev->bd_disk->fops->ioctl(bdev, mode, cmd, arg);
469out:
470 dm_unprepare_ioctl(md, srcu_idx);
471 return r;
472}
473
474u64 dm_start_time_ns_from_clone(struct bio *bio)
475{
476 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
477 struct dm_io *io = tio->io;
478
479 return jiffies_to_nsecs(io->start_time);
480}
481EXPORT_SYMBOL_GPL(dm_start_time_ns_from_clone);
482
483static void start_io_acct(struct dm_io *io)
484{
485 struct mapped_device *md = io->md;
486 struct bio *bio = io->orig_bio;
487
488 io->start_time = bio_start_io_acct(bio);
489 if (unlikely(dm_stats_used(&md->stats)))
490 dm_stats_account_io(&md->stats, bio_data_dir(bio),
491 bio->bi_iter.bi_sector, bio_sectors(bio),
492 false, 0, &io->stats_aux);
493}
494
495static void end_io_acct(struct mapped_device *md, struct bio *bio,
496 unsigned long start_time, struct dm_stats_aux *stats_aux)
497{
498 unsigned long duration = jiffies - start_time;
499
500 bio_end_io_acct(bio, start_time);
501
502 if (unlikely(dm_stats_used(&md->stats)))
503 dm_stats_account_io(&md->stats, bio_data_dir(bio),
504 bio->bi_iter.bi_sector, bio_sectors(bio),
505 true, duration, stats_aux);
506
507 /* nudge anyone waiting on suspend queue */
508 if (unlikely(wq_has_sleeper(&md->wait)))
509 wake_up(&md->wait);
510}
511
512static struct dm_io *alloc_io(struct mapped_device *md, struct bio *bio)
513{
514 struct dm_io *io;
515 struct dm_target_io *tio;
516 struct bio *clone;
517
518 clone = bio_alloc_bioset(GFP_NOIO, 0, &md->io_bs);
519 if (!clone)
520 return NULL;
521
522 tio = container_of(clone, struct dm_target_io, clone);
523 tio->inside_dm_io = true;
524 tio->io = NULL;
525
526 io = container_of(tio, struct dm_io, tio);
527 io->magic = DM_IO_MAGIC;
528 io->status = 0;
529 atomic_set(&io->io_count, 1);
530 io->orig_bio = bio;
531 io->md = md;
532 spin_lock_init(&io->endio_lock);
533
534 start_io_acct(io);
535
536 return io;
537}
538
539static void free_io(struct mapped_device *md, struct dm_io *io)
540{
541 bio_put(&io->tio.clone);
542}
543
544static struct dm_target_io *alloc_tio(struct clone_info *ci, struct dm_target *ti,
545 unsigned target_bio_nr, gfp_t gfp_mask)
546{
547 struct dm_target_io *tio;
548
549 if (!ci->io->tio.io) {
550 /* the dm_target_io embedded in ci->io is available */
551 tio = &ci->io->tio;
552 } else {
553 struct bio *clone = bio_alloc_bioset(gfp_mask, 0, &ci->io->md->bs);
554 if (!clone)
555 return NULL;
556
557 tio = container_of(clone, struct dm_target_io, clone);
558 tio->inside_dm_io = false;
559 }
560
561 tio->magic = DM_TIO_MAGIC;
562 tio->io = ci->io;
563 tio->ti = ti;
564 tio->target_bio_nr = target_bio_nr;
565
566 return tio;
567}
568
569static void free_tio(struct dm_target_io *tio)
570{
571 if (tio->inside_dm_io)
572 return;
573 bio_put(&tio->clone);
574}
575
576/*
577 * Add the bio to the list of deferred io.
578 */
579static void queue_io(struct mapped_device *md, struct bio *bio)
580{
581 unsigned long flags;
582
583 spin_lock_irqsave(&md->deferred_lock, flags);
584 bio_list_add(&md->deferred, bio);
585 spin_unlock_irqrestore(&md->deferred_lock, flags);
586 queue_work(md->wq, &md->work);
587}
588
589/*
590 * Everyone (including functions in this file), should use this
591 * function to access the md->map field, and make sure they call
592 * dm_put_live_table() when finished.
593 */
594struct dm_table *dm_get_live_table(struct mapped_device *md, int *srcu_idx) __acquires(md->io_barrier)
595{
596 *srcu_idx = srcu_read_lock(&md->io_barrier);
597
598 return srcu_dereference(md->map, &md->io_barrier);
599}
600
601void dm_put_live_table(struct mapped_device *md, int srcu_idx) __releases(md->io_barrier)
602{
603 srcu_read_unlock(&md->io_barrier, srcu_idx);
604}
605
606void dm_sync_table(struct mapped_device *md)
607{
608 synchronize_srcu(&md->io_barrier);
609 synchronize_rcu_expedited();
610}
611
612/*
613 * A fast alternative to dm_get_live_table/dm_put_live_table.
614 * The caller must not block between these two functions.
615 */
616static struct dm_table *dm_get_live_table_fast(struct mapped_device *md) __acquires(RCU)
617{
618 rcu_read_lock();
619 return rcu_dereference(md->map);
620}
621
622static void dm_put_live_table_fast(struct mapped_device *md) __releases(RCU)
623{
624 rcu_read_unlock();
625}
626
627static char *_dm_claim_ptr = "I belong to device-mapper";
628
629/*
630 * Open a table device so we can use it as a map destination.
631 */
632static int open_table_device(struct table_device *td, dev_t dev,
633 struct mapped_device *md)
634{
635 struct block_device *bdev;
636
637 int r;
638
639 BUG_ON(td->dm_dev.bdev);
640
641 bdev = blkdev_get_by_dev(dev, td->dm_dev.mode | FMODE_EXCL, _dm_claim_ptr);
642 if (IS_ERR(bdev))
643 return PTR_ERR(bdev);
644
645 r = bd_link_disk_holder(bdev, dm_disk(md));
646 if (r) {
647 blkdev_put(bdev, td->dm_dev.mode | FMODE_EXCL);
648 return r;
649 }
650
651 td->dm_dev.bdev = bdev;
652 td->dm_dev.dax_dev = dax_get_by_host(bdev->bd_disk->disk_name);
653 return 0;
654}
655
656/*
657 * Close a table device that we've been using.
658 */
659static void close_table_device(struct table_device *td, struct mapped_device *md)
660{
661 if (!td->dm_dev.bdev)
662 return;
663
664 bd_unlink_disk_holder(td->dm_dev.bdev, dm_disk(md));
665 blkdev_put(td->dm_dev.bdev, td->dm_dev.mode | FMODE_EXCL);
666 put_dax(td->dm_dev.dax_dev);
667 td->dm_dev.bdev = NULL;
668 td->dm_dev.dax_dev = NULL;
669}
670
671static struct table_device *find_table_device(struct list_head *l, dev_t dev,
672 fmode_t mode)
673{
674 struct table_device *td;
675
676 list_for_each_entry(td, l, list)
677 if (td->dm_dev.bdev->bd_dev == dev && td->dm_dev.mode == mode)
678 return td;
679
680 return NULL;
681}
682
683int dm_get_table_device(struct mapped_device *md, dev_t dev, fmode_t mode,
684 struct dm_dev **result)
685{
686 int r;
687 struct table_device *td;
688
689 mutex_lock(&md->table_devices_lock);
690 td = find_table_device(&md->table_devices, dev, mode);
691 if (!td) {
692 td = kmalloc_node(sizeof(*td), GFP_KERNEL, md->numa_node_id);
693 if (!td) {
694 mutex_unlock(&md->table_devices_lock);
695 return -ENOMEM;
696 }
697
698 td->dm_dev.mode = mode;
699 td->dm_dev.bdev = NULL;
700
701 if ((r = open_table_device(td, dev, md))) {
702 mutex_unlock(&md->table_devices_lock);
703 kfree(td);
704 return r;
705 }
706
707 format_dev_t(td->dm_dev.name, dev);
708
709 refcount_set(&td->count, 1);
710 list_add(&td->list, &md->table_devices);
711 } else {
712 refcount_inc(&td->count);
713 }
714 mutex_unlock(&md->table_devices_lock);
715
716 *result = &td->dm_dev;
717 return 0;
718}
719
720void dm_put_table_device(struct mapped_device *md, struct dm_dev *d)
721{
722 struct table_device *td = container_of(d, struct table_device, dm_dev);
723
724 mutex_lock(&md->table_devices_lock);
725 if (refcount_dec_and_test(&td->count)) {
726 close_table_device(td, md);
727 list_del(&td->list);
728 kfree(td);
729 }
730 mutex_unlock(&md->table_devices_lock);
731}
732
733static void free_table_devices(struct list_head *devices)
734{
735 struct list_head *tmp, *next;
736
737 list_for_each_safe(tmp, next, devices) {
738 struct table_device *td = list_entry(tmp, struct table_device, list);
739
740 DMWARN("dm_destroy: %s still exists with %d references",
741 td->dm_dev.name, refcount_read(&td->count));
742 kfree(td);
743 }
744}
745
746/*
747 * Get the geometry associated with a dm device
748 */
749int dm_get_geometry(struct mapped_device *md, struct hd_geometry *geo)
750{
751 *geo = md->geometry;
752
753 return 0;
754}
755
756/*
757 * Set the geometry of a device.
758 */
759int dm_set_geometry(struct mapped_device *md, struct hd_geometry *geo)
760{
761 sector_t sz = (sector_t)geo->cylinders * geo->heads * geo->sectors;
762
763 if (geo->start > sz) {
764 DMWARN("Start sector is beyond the geometry limits.");
765 return -EINVAL;
766 }
767
768 md->geometry = *geo;
769
770 return 0;
771}
772
773static int __noflush_suspending(struct mapped_device *md)
774{
775 return test_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
776}
777
778/*
779 * Decrements the number of outstanding ios that a bio has been
780 * cloned into, completing the original io if necc.
781 */
782void dm_io_dec_pending(struct dm_io *io, blk_status_t error)
783{
784 unsigned long flags;
785 blk_status_t io_error;
786 struct bio *bio;
787 struct mapped_device *md = io->md;
788 unsigned long start_time = 0;
789 struct dm_stats_aux stats_aux;
790
791 /* Push-back supersedes any I/O errors */
792 if (unlikely(error)) {
793 spin_lock_irqsave(&io->endio_lock, flags);
794 if (!(io->status == BLK_STS_DM_REQUEUE && __noflush_suspending(md)))
795 io->status = error;
796 spin_unlock_irqrestore(&io->endio_lock, flags);
797 }
798
799 if (atomic_dec_and_test(&io->io_count)) {
800 bio = io->orig_bio;
801 if (io->status == BLK_STS_DM_REQUEUE) {
802 /*
803 * Target requested pushing back the I/O.
804 */
805 spin_lock_irqsave(&md->deferred_lock, flags);
806 if (__noflush_suspending(md) &&
807 !WARN_ON_ONCE(dm_is_zone_write(md, bio))) {
808 /* NOTE early return due to BLK_STS_DM_REQUEUE below */
809 bio_list_add_head(&md->deferred, bio);
810 } else {
811 /*
812 * noflush suspend was interrupted or this is
813 * a write to a zoned target.
814 */
815 io->status = BLK_STS_IOERR;
816 }
817 spin_unlock_irqrestore(&md->deferred_lock, flags);
818 }
819
820 io_error = io->status;
821 start_time = io->start_time;
822 stats_aux = io->stats_aux;
823 free_io(md, io);
824 end_io_acct(md, bio, start_time, &stats_aux);
825
826 if (io_error == BLK_STS_DM_REQUEUE)
827 return;
828
829 if ((bio->bi_opf & REQ_PREFLUSH) && bio->bi_iter.bi_size) {
830 /*
831 * Preflush done for flush with data, reissue
832 * without REQ_PREFLUSH.
833 */
834 bio->bi_opf &= ~REQ_PREFLUSH;
835 queue_io(md, bio);
836 } else {
837 /* done with normal IO or empty flush */
838 if (io_error)
839 bio->bi_status = io_error;
840 bio_endio(bio);
841 }
842 }
843}
844
845void disable_discard(struct mapped_device *md)
846{
847 struct queue_limits *limits = dm_get_queue_limits(md);
848
849 /* device doesn't really support DISCARD, disable it */
850 limits->max_discard_sectors = 0;
851 blk_queue_flag_clear(QUEUE_FLAG_DISCARD, md->queue);
852}
853
854void disable_write_same(struct mapped_device *md)
855{
856 struct queue_limits *limits = dm_get_queue_limits(md);
857
858 /* device doesn't really support WRITE SAME, disable it */
859 limits->max_write_same_sectors = 0;
860}
861
862void disable_write_zeroes(struct mapped_device *md)
863{
864 struct queue_limits *limits = dm_get_queue_limits(md);
865
866 /* device doesn't really support WRITE ZEROES, disable it */
867 limits->max_write_zeroes_sectors = 0;
868}
869
870static bool swap_bios_limit(struct dm_target *ti, struct bio *bio)
871{
872 return unlikely((bio->bi_opf & REQ_SWAP) != 0) && unlikely(ti->limit_swap_bios);
873}
874
875static void clone_endio(struct bio *bio)
876{
877 blk_status_t error = bio->bi_status;
878 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
879 struct dm_io *io = tio->io;
880 struct mapped_device *md = tio->io->md;
881 dm_endio_fn endio = tio->ti->type->end_io;
882 struct request_queue *q = bio->bi_bdev->bd_disk->queue;
883
884 if (unlikely(error == BLK_STS_TARGET)) {
885 if (bio_op(bio) == REQ_OP_DISCARD &&
886 !q->limits.max_discard_sectors)
887 disable_discard(md);
888 else if (bio_op(bio) == REQ_OP_WRITE_SAME &&
889 !q->limits.max_write_same_sectors)
890 disable_write_same(md);
891 else if (bio_op(bio) == REQ_OP_WRITE_ZEROES &&
892 !q->limits.max_write_zeroes_sectors)
893 disable_write_zeroes(md);
894 }
895
896 if (blk_queue_is_zoned(q))
897 dm_zone_endio(io, bio);
898
899 if (endio) {
900 int r = endio(tio->ti, bio, &error);
901 switch (r) {
902 case DM_ENDIO_REQUEUE:
903 /*
904 * Requeuing writes to a sequential zone of a zoned
905 * target will break the sequential write pattern:
906 * fail such IO.
907 */
908 if (WARN_ON_ONCE(dm_is_zone_write(md, bio)))
909 error = BLK_STS_IOERR;
910 else
911 error = BLK_STS_DM_REQUEUE;
912 fallthrough;
913 case DM_ENDIO_DONE:
914 break;
915 case DM_ENDIO_INCOMPLETE:
916 /* The target will handle the io */
917 return;
918 default:
919 DMWARN("unimplemented target endio return value: %d", r);
920 BUG();
921 }
922 }
923
924 if (unlikely(swap_bios_limit(tio->ti, bio))) {
925 struct mapped_device *md = io->md;
926 up(&md->swap_bios_semaphore);
927 }
928
929 free_tio(tio);
930 dm_io_dec_pending(io, error);
931}
932
933/*
934 * Return maximum size of I/O possible at the supplied sector up to the current
935 * target boundary.
936 */
937static inline sector_t max_io_len_target_boundary(struct dm_target *ti,
938 sector_t target_offset)
939{
940 return ti->len - target_offset;
941}
942
943static sector_t max_io_len(struct dm_target *ti, sector_t sector)
944{
945 sector_t target_offset = dm_target_offset(ti, sector);
946 sector_t len = max_io_len_target_boundary(ti, target_offset);
947 sector_t max_len;
948
949 /*
950 * Does the target need to split IO even further?
951 * - varied (per target) IO splitting is a tenet of DM; this
952 * explains why stacked chunk_sectors based splitting via
953 * blk_max_size_offset() isn't possible here. So pass in
954 * ti->max_io_len to override stacked chunk_sectors.
955 */
956 if (ti->max_io_len) {
957 max_len = blk_max_size_offset(ti->table->md->queue,
958 target_offset, ti->max_io_len);
959 if (len > max_len)
960 len = max_len;
961 }
962
963 return len;
964}
965
966int dm_set_target_max_io_len(struct dm_target *ti, sector_t len)
967{
968 if (len > UINT_MAX) {
969 DMERR("Specified maximum size of target IO (%llu) exceeds limit (%u)",
970 (unsigned long long)len, UINT_MAX);
971 ti->error = "Maximum size of target IO is too large";
972 return -EINVAL;
973 }
974
975 ti->max_io_len = (uint32_t) len;
976
977 return 0;
978}
979EXPORT_SYMBOL_GPL(dm_set_target_max_io_len);
980
981static struct dm_target *dm_dax_get_live_target(struct mapped_device *md,
982 sector_t sector, int *srcu_idx)
983 __acquires(md->io_barrier)
984{
985 struct dm_table *map;
986 struct dm_target *ti;
987
988 map = dm_get_live_table(md, srcu_idx);
989 if (!map)
990 return NULL;
991
992 ti = dm_table_find_target(map, sector);
993 if (!ti)
994 return NULL;
995
996 return ti;
997}
998
999static long dm_dax_direct_access(struct dax_device *dax_dev, pgoff_t pgoff,
1000 long nr_pages, void **kaddr, pfn_t *pfn)
1001{
1002 struct mapped_device *md = dax_get_private(dax_dev);
1003 sector_t sector = pgoff * PAGE_SECTORS;
1004 struct dm_target *ti;
1005 long len, ret = -EIO;
1006 int srcu_idx;
1007
1008 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1009
1010 if (!ti)
1011 goto out;
1012 if (!ti->type->direct_access)
1013 goto out;
1014 len = max_io_len(ti, sector) / PAGE_SECTORS;
1015 if (len < 1)
1016 goto out;
1017 nr_pages = min(len, nr_pages);
1018 ret = ti->type->direct_access(ti, pgoff, nr_pages, kaddr, pfn);
1019
1020 out:
1021 dm_put_live_table(md, srcu_idx);
1022
1023 return ret;
1024}
1025
1026static bool dm_dax_supported(struct dax_device *dax_dev, struct block_device *bdev,
1027 int blocksize, sector_t start, sector_t len)
1028{
1029 struct mapped_device *md = dax_get_private(dax_dev);
1030 struct dm_table *map;
1031 bool ret = false;
1032 int srcu_idx;
1033
1034 map = dm_get_live_table(md, &srcu_idx);
1035 if (!map)
1036 goto out;
1037
1038 ret = dm_table_supports_dax(map, device_not_dax_capable, &blocksize);
1039
1040out:
1041 dm_put_live_table(md, srcu_idx);
1042
1043 return ret;
1044}
1045
1046static size_t dm_dax_copy_from_iter(struct dax_device *dax_dev, pgoff_t pgoff,
1047 void *addr, size_t bytes, struct iov_iter *i)
1048{
1049 struct mapped_device *md = dax_get_private(dax_dev);
1050 sector_t sector = pgoff * PAGE_SECTORS;
1051 struct dm_target *ti;
1052 long ret = 0;
1053 int srcu_idx;
1054
1055 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1056
1057 if (!ti)
1058 goto out;
1059 if (!ti->type->dax_copy_from_iter) {
1060 ret = copy_from_iter(addr, bytes, i);
1061 goto out;
1062 }
1063 ret = ti->type->dax_copy_from_iter(ti, pgoff, addr, bytes, i);
1064 out:
1065 dm_put_live_table(md, srcu_idx);
1066
1067 return ret;
1068}
1069
1070static size_t dm_dax_copy_to_iter(struct dax_device *dax_dev, pgoff_t pgoff,
1071 void *addr, size_t bytes, struct iov_iter *i)
1072{
1073 struct mapped_device *md = dax_get_private(dax_dev);
1074 sector_t sector = pgoff * PAGE_SECTORS;
1075 struct dm_target *ti;
1076 long ret = 0;
1077 int srcu_idx;
1078
1079 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1080
1081 if (!ti)
1082 goto out;
1083 if (!ti->type->dax_copy_to_iter) {
1084 ret = copy_to_iter(addr, bytes, i);
1085 goto out;
1086 }
1087 ret = ti->type->dax_copy_to_iter(ti, pgoff, addr, bytes, i);
1088 out:
1089 dm_put_live_table(md, srcu_idx);
1090
1091 return ret;
1092}
1093
1094static int dm_dax_zero_page_range(struct dax_device *dax_dev, pgoff_t pgoff,
1095 size_t nr_pages)
1096{
1097 struct mapped_device *md = dax_get_private(dax_dev);
1098 sector_t sector = pgoff * PAGE_SECTORS;
1099 struct dm_target *ti;
1100 int ret = -EIO;
1101 int srcu_idx;
1102
1103 ti = dm_dax_get_live_target(md, sector, &srcu_idx);
1104
1105 if (!ti)
1106 goto out;
1107 if (WARN_ON(!ti->type->dax_zero_page_range)) {
1108 /*
1109 * ->zero_page_range() is mandatory dax operation. If we are
1110 * here, something is wrong.
1111 */
1112 goto out;
1113 }
1114 ret = ti->type->dax_zero_page_range(ti, pgoff, nr_pages);
1115 out:
1116 dm_put_live_table(md, srcu_idx);
1117
1118 return ret;
1119}
1120
1121/*
1122 * A target may call dm_accept_partial_bio only from the map routine. It is
1123 * allowed for all bio types except REQ_PREFLUSH, REQ_OP_ZONE_* zone management
1124 * operations and REQ_OP_ZONE_APPEND (zone append writes).
1125 *
1126 * dm_accept_partial_bio informs the dm that the target only wants to process
1127 * additional n_sectors sectors of the bio and the rest of the data should be
1128 * sent in a next bio.
1129 *
1130 * A diagram that explains the arithmetics:
1131 * +--------------------+---------------+-------+
1132 * | 1 | 2 | 3 |
1133 * +--------------------+---------------+-------+
1134 *
1135 * <-------------- *tio->len_ptr --------------->
1136 * <------- bi_size ------->
1137 * <-- n_sectors -->
1138 *
1139 * Region 1 was already iterated over with bio_advance or similar function.
1140 * (it may be empty if the target doesn't use bio_advance)
1141 * Region 2 is the remaining bio size that the target wants to process.
1142 * (it may be empty if region 1 is non-empty, although there is no reason
1143 * to make it empty)
1144 * The target requires that region 3 is to be sent in the next bio.
1145 *
1146 * If the target wants to receive multiple copies of the bio (via num_*bios, etc),
1147 * the partially processed part (the sum of regions 1+2) must be the same for all
1148 * copies of the bio.
1149 */
1150void dm_accept_partial_bio(struct bio *bio, unsigned n_sectors)
1151{
1152 struct dm_target_io *tio = container_of(bio, struct dm_target_io, clone);
1153 unsigned bi_size = bio->bi_iter.bi_size >> SECTOR_SHIFT;
1154
1155 BUG_ON(bio->bi_opf & REQ_PREFLUSH);
1156 BUG_ON(op_is_zone_mgmt(bio_op(bio)));
1157 BUG_ON(bio_op(bio) == REQ_OP_ZONE_APPEND);
1158 BUG_ON(bi_size > *tio->len_ptr);
1159 BUG_ON(n_sectors > bi_size);
1160
1161 *tio->len_ptr -= bi_size - n_sectors;
1162 bio->bi_iter.bi_size = n_sectors << SECTOR_SHIFT;
1163}
1164EXPORT_SYMBOL_GPL(dm_accept_partial_bio);
1165
1166static noinline void __set_swap_bios_limit(struct mapped_device *md, int latch)
1167{
1168 mutex_lock(&md->swap_bios_lock);
1169 while (latch < md->swap_bios) {
1170 cond_resched();
1171 down(&md->swap_bios_semaphore);
1172 md->swap_bios--;
1173 }
1174 while (latch > md->swap_bios) {
1175 cond_resched();
1176 up(&md->swap_bios_semaphore);
1177 md->swap_bios++;
1178 }
1179 mutex_unlock(&md->swap_bios_lock);
1180}
1181
1182static blk_qc_t __map_bio(struct dm_target_io *tio)
1183{
1184 int r;
1185 sector_t sector;
1186 struct bio *clone = &tio->clone;
1187 struct dm_io *io = tio->io;
1188 struct dm_target *ti = tio->ti;
1189 blk_qc_t ret = BLK_QC_T_NONE;
1190
1191 clone->bi_end_io = clone_endio;
1192
1193 /*
1194 * Map the clone. If r == 0 we don't need to do
1195 * anything, the target has assumed ownership of
1196 * this io.
1197 */
1198 dm_io_inc_pending(io);
1199 sector = clone->bi_iter.bi_sector;
1200
1201 if (unlikely(swap_bios_limit(ti, clone))) {
1202 struct mapped_device *md = io->md;
1203 int latch = get_swap_bios();
1204 if (unlikely(latch != md->swap_bios))
1205 __set_swap_bios_limit(md, latch);
1206 down(&md->swap_bios_semaphore);
1207 }
1208
1209 /*
1210 * Check if the IO needs a special mapping due to zone append emulation
1211 * on zoned target. In this case, dm_zone_map_bio() calls the target
1212 * map operation.
1213 */
1214 if (dm_emulate_zone_append(io->md))
1215 r = dm_zone_map_bio(tio);
1216 else
1217 r = ti->type->map(ti, clone);
1218
1219 switch (r) {
1220 case DM_MAPIO_SUBMITTED:
1221 break;
1222 case DM_MAPIO_REMAPPED:
1223 /* the bio has been remapped so dispatch it */
1224 trace_block_bio_remap(clone, bio_dev(io->orig_bio), sector);
1225 ret = submit_bio_noacct(clone);
1226 break;
1227 case DM_MAPIO_KILL:
1228 if (unlikely(swap_bios_limit(ti, clone))) {
1229 struct mapped_device *md = io->md;
1230 up(&md->swap_bios_semaphore);
1231 }
1232 free_tio(tio);
1233 dm_io_dec_pending(io, BLK_STS_IOERR);
1234 break;
1235 case DM_MAPIO_REQUEUE:
1236 if (unlikely(swap_bios_limit(ti, clone))) {
1237 struct mapped_device *md = io->md;
1238 up(&md->swap_bios_semaphore);
1239 }
1240 free_tio(tio);
1241 dm_io_dec_pending(io, BLK_STS_DM_REQUEUE);
1242 break;
1243 default:
1244 DMWARN("unimplemented target map return value: %d", r);
1245 BUG();
1246 }
1247
1248 return ret;
1249}
1250
1251static void bio_setup_sector(struct bio *bio, sector_t sector, unsigned len)
1252{
1253 bio->bi_iter.bi_sector = sector;
1254 bio->bi_iter.bi_size = to_bytes(len);
1255}
1256
1257/*
1258 * Creates a bio that consists of range of complete bvecs.
1259 */
1260static int clone_bio(struct dm_target_io *tio, struct bio *bio,
1261 sector_t sector, unsigned len)
1262{
1263 struct bio *clone = &tio->clone;
1264 int r;
1265
1266 __bio_clone_fast(clone, bio);
1267
1268 r = bio_crypt_clone(clone, bio, GFP_NOIO);
1269 if (r < 0)
1270 return r;
1271
1272 if (bio_integrity(bio)) {
1273 if (unlikely(!dm_target_has_integrity(tio->ti->type) &&
1274 !dm_target_passes_integrity(tio->ti->type))) {
1275 DMWARN("%s: the target %s doesn't support integrity data.",
1276 dm_device_name(tio->io->md),
1277 tio->ti->type->name);
1278 return -EIO;
1279 }
1280
1281 r = bio_integrity_clone(clone, bio, GFP_NOIO);
1282 if (r < 0)
1283 return r;
1284 }
1285
1286 bio_advance(clone, to_bytes(sector - clone->bi_iter.bi_sector));
1287 clone->bi_iter.bi_size = to_bytes(len);
1288
1289 if (bio_integrity(bio))
1290 bio_integrity_trim(clone);
1291
1292 return 0;
1293}
1294
1295static void alloc_multiple_bios(struct bio_list *blist, struct clone_info *ci,
1296 struct dm_target *ti, unsigned num_bios)
1297{
1298 struct dm_target_io *tio;
1299 int try;
1300
1301 if (!num_bios)
1302 return;
1303
1304 if (num_bios == 1) {
1305 tio = alloc_tio(ci, ti, 0, GFP_NOIO);
1306 bio_list_add(blist, &tio->clone);
1307 return;
1308 }
1309
1310 for (try = 0; try < 2; try++) {
1311 int bio_nr;
1312 struct bio *bio;
1313
1314 if (try)
1315 mutex_lock(&ci->io->md->table_devices_lock);
1316 for (bio_nr = 0; bio_nr < num_bios; bio_nr++) {
1317 tio = alloc_tio(ci, ti, bio_nr, try ? GFP_NOIO : GFP_NOWAIT);
1318 if (!tio)
1319 break;
1320
1321 bio_list_add(blist, &tio->clone);
1322 }
1323 if (try)
1324 mutex_unlock(&ci->io->md->table_devices_lock);
1325 if (bio_nr == num_bios)
1326 return;
1327
1328 while ((bio = bio_list_pop(blist))) {
1329 tio = container_of(bio, struct dm_target_io, clone);
1330 free_tio(tio);
1331 }
1332 }
1333}
1334
1335static blk_qc_t __clone_and_map_simple_bio(struct clone_info *ci,
1336 struct dm_target_io *tio, unsigned *len)
1337{
1338 struct bio *clone = &tio->clone;
1339
1340 tio->len_ptr = len;
1341
1342 __bio_clone_fast(clone, ci->bio);
1343 if (len)
1344 bio_setup_sector(clone, ci->sector, *len);
1345
1346 return __map_bio(tio);
1347}
1348
1349static void __send_duplicate_bios(struct clone_info *ci, struct dm_target *ti,
1350 unsigned num_bios, unsigned *len)
1351{
1352 struct bio_list blist = BIO_EMPTY_LIST;
1353 struct bio *bio;
1354 struct dm_target_io *tio;
1355
1356 alloc_multiple_bios(&blist, ci, ti, num_bios);
1357
1358 while ((bio = bio_list_pop(&blist))) {
1359 tio = container_of(bio, struct dm_target_io, clone);
1360 (void) __clone_and_map_simple_bio(ci, tio, len);
1361 }
1362}
1363
1364static int __send_empty_flush(struct clone_info *ci)
1365{
1366 unsigned target_nr = 0;
1367 struct dm_target *ti;
1368 struct bio flush_bio;
1369
1370 /*
1371 * Use an on-stack bio for this, it's safe since we don't
1372 * need to reference it after submit. It's just used as
1373 * the basis for the clone(s).
1374 */
1375 bio_init(&flush_bio, NULL, 0);
1376 flush_bio.bi_opf = REQ_OP_WRITE | REQ_PREFLUSH | REQ_SYNC;
1377 bio_set_dev(&flush_bio, ci->io->md->disk->part0);
1378
1379 ci->bio = &flush_bio;
1380 ci->sector_count = 0;
1381
1382 BUG_ON(bio_has_data(ci->bio));
1383 while ((ti = dm_table_get_target(ci->map, target_nr++)))
1384 __send_duplicate_bios(ci, ti, ti->num_flush_bios, NULL);
1385
1386 bio_uninit(ci->bio);
1387 return 0;
1388}
1389
1390static int __clone_and_map_data_bio(struct clone_info *ci, struct dm_target *ti,
1391 sector_t sector, unsigned *len)
1392{
1393 struct bio *bio = ci->bio;
1394 struct dm_target_io *tio;
1395 int r;
1396
1397 tio = alloc_tio(ci, ti, 0, GFP_NOIO);
1398 tio->len_ptr = len;
1399 r = clone_bio(tio, bio, sector, *len);
1400 if (r < 0) {
1401 free_tio(tio);
1402 return r;
1403 }
1404 (void) __map_bio(tio);
1405
1406 return 0;
1407}
1408
1409static int __send_changing_extent_only(struct clone_info *ci, struct dm_target *ti,
1410 unsigned num_bios)
1411{
1412 unsigned len;
1413
1414 /*
1415 * Even though the device advertised support for this type of
1416 * request, that does not mean every target supports it, and
1417 * reconfiguration might also have changed that since the
1418 * check was performed.
1419 */
1420 if (!num_bios)
1421 return -EOPNOTSUPP;
1422
1423 len = min_t(sector_t, ci->sector_count,
1424 max_io_len_target_boundary(ti, dm_target_offset(ti, ci->sector)));
1425
1426 __send_duplicate_bios(ci, ti, num_bios, &len);
1427
1428 ci->sector += len;
1429 ci->sector_count -= len;
1430
1431 return 0;
1432}
1433
1434static bool is_abnormal_io(struct bio *bio)
1435{
1436 bool r = false;
1437
1438 switch (bio_op(bio)) {
1439 case REQ_OP_DISCARD:
1440 case REQ_OP_SECURE_ERASE:
1441 case REQ_OP_WRITE_SAME:
1442 case REQ_OP_WRITE_ZEROES:
1443 r = true;
1444 break;
1445 }
1446
1447 return r;
1448}
1449
1450static bool __process_abnormal_io(struct clone_info *ci, struct dm_target *ti,
1451 int *result)
1452{
1453 struct bio *bio = ci->bio;
1454 unsigned num_bios = 0;
1455
1456 switch (bio_op(bio)) {
1457 case REQ_OP_DISCARD:
1458 num_bios = ti->num_discard_bios;
1459 break;
1460 case REQ_OP_SECURE_ERASE:
1461 num_bios = ti->num_secure_erase_bios;
1462 break;
1463 case REQ_OP_WRITE_SAME:
1464 num_bios = ti->num_write_same_bios;
1465 break;
1466 case REQ_OP_WRITE_ZEROES:
1467 num_bios = ti->num_write_zeroes_bios;
1468 break;
1469 default:
1470 return false;
1471 }
1472
1473 *result = __send_changing_extent_only(ci, ti, num_bios);
1474 return true;
1475}
1476
1477/*
1478 * Select the correct strategy for processing a non-flush bio.
1479 */
1480static int __split_and_process_non_flush(struct clone_info *ci)
1481{
1482 struct dm_target *ti;
1483 unsigned len;
1484 int r;
1485
1486 ti = dm_table_find_target(ci->map, ci->sector);
1487 if (!ti)
1488 return -EIO;
1489
1490 if (__process_abnormal_io(ci, ti, &r))
1491 return r;
1492
1493 len = min_t(sector_t, max_io_len(ti, ci->sector), ci->sector_count);
1494
1495 r = __clone_and_map_data_bio(ci, ti, ci->sector, &len);
1496 if (r < 0)
1497 return r;
1498
1499 ci->sector += len;
1500 ci->sector_count -= len;
1501
1502 return 0;
1503}
1504
1505static void init_clone_info(struct clone_info *ci, struct mapped_device *md,
1506 struct dm_table *map, struct bio *bio)
1507{
1508 ci->map = map;
1509 ci->io = alloc_io(md, bio);
1510 ci->sector = bio->bi_iter.bi_sector;
1511}
1512
1513#define __dm_part_stat_sub(part, field, subnd) \
1514 (part_stat_get(part, field) -= (subnd))
1515
1516/*
1517 * Entry point to split a bio into clones and submit them to the targets.
1518 */
1519static blk_qc_t __split_and_process_bio(struct mapped_device *md,
1520 struct dm_table *map, struct bio *bio)
1521{
1522 struct clone_info ci;
1523 blk_qc_t ret = BLK_QC_T_NONE;
1524 int error = 0;
1525
1526 init_clone_info(&ci, md, map, bio);
1527
1528 if (bio->bi_opf & REQ_PREFLUSH) {
1529 error = __send_empty_flush(&ci);
1530 /* dm_io_dec_pending submits any data associated with flush */
1531 } else if (op_is_zone_mgmt(bio_op(bio))) {
1532 ci.bio = bio;
1533 ci.sector_count = 0;
1534 error = __split_and_process_non_flush(&ci);
1535 } else {
1536 ci.bio = bio;
1537 ci.sector_count = bio_sectors(bio);
1538 error = __split_and_process_non_flush(&ci);
1539 if (ci.sector_count && !error) {
1540 /*
1541 * Remainder must be passed to submit_bio_noacct()
1542 * so that it gets handled *after* bios already submitted
1543 * have been completely processed.
1544 * We take a clone of the original to store in
1545 * ci.io->orig_bio to be used by end_io_acct() and
1546 * for dec_pending to use for completion handling.
1547 */
1548 struct bio *b = bio_split(bio, bio_sectors(bio) - ci.sector_count,
1549 GFP_NOIO, &md->queue->bio_split);
1550 ci.io->orig_bio = b;
1551
1552 /*
1553 * Adjust IO stats for each split, otherwise upon queue
1554 * reentry there will be redundant IO accounting.
1555 * NOTE: this is a stop-gap fix, a proper fix involves
1556 * significant refactoring of DM core's bio splitting
1557 * (by eliminating DM's splitting and just using bio_split)
1558 */
1559 part_stat_lock();
1560 __dm_part_stat_sub(dm_disk(md)->part0,
1561 sectors[op_stat_group(bio_op(bio))], ci.sector_count);
1562 part_stat_unlock();
1563
1564 bio_chain(b, bio);
1565 trace_block_split(b, bio->bi_iter.bi_sector);
1566 ret = submit_bio_noacct(bio);
1567 }
1568 }
1569
1570 /* drop the extra reference count */
1571 dm_io_dec_pending(ci.io, errno_to_blk_status(error));
1572 return ret;
1573}
1574
1575static blk_qc_t dm_submit_bio(struct bio *bio)
1576{
1577 struct mapped_device *md = bio->bi_bdev->bd_disk->private_data;
1578 blk_qc_t ret = BLK_QC_T_NONE;
1579 int srcu_idx;
1580 struct dm_table *map;
1581
1582 map = dm_get_live_table(md, &srcu_idx);
1583 if (unlikely(!map)) {
1584 DMERR_LIMIT("%s: mapping table unavailable, erroring io",
1585 dm_device_name(md));
1586 bio_io_error(bio);
1587 goto out;
1588 }
1589
1590 /* If suspended, queue this IO for later */
1591 if (unlikely(test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags))) {
1592 if (bio->bi_opf & REQ_NOWAIT)
1593 bio_wouldblock_error(bio);
1594 else if (bio->bi_opf & REQ_RAHEAD)
1595 bio_io_error(bio);
1596 else
1597 queue_io(md, bio);
1598 goto out;
1599 }
1600
1601 /*
1602 * Use blk_queue_split() for abnormal IO (e.g. discard, writesame, etc)
1603 * otherwise associated queue_limits won't be imposed.
1604 */
1605 if (is_abnormal_io(bio))
1606 blk_queue_split(&bio);
1607
1608 ret = __split_and_process_bio(md, map, bio);
1609out:
1610 dm_put_live_table(md, srcu_idx);
1611 return ret;
1612}
1613
1614/*-----------------------------------------------------------------
1615 * An IDR is used to keep track of allocated minor numbers.
1616 *---------------------------------------------------------------*/
1617static void free_minor(int minor)
1618{
1619 spin_lock(&_minor_lock);
1620 idr_remove(&_minor_idr, minor);
1621 spin_unlock(&_minor_lock);
1622}
1623
1624/*
1625 * See if the device with a specific minor # is free.
1626 */
1627static int specific_minor(int minor)
1628{
1629 int r;
1630
1631 if (minor >= (1 << MINORBITS))
1632 return -EINVAL;
1633
1634 idr_preload(GFP_KERNEL);
1635 spin_lock(&_minor_lock);
1636
1637 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, minor, minor + 1, GFP_NOWAIT);
1638
1639 spin_unlock(&_minor_lock);
1640 idr_preload_end();
1641 if (r < 0)
1642 return r == -ENOSPC ? -EBUSY : r;
1643 return 0;
1644}
1645
1646static int next_free_minor(int *minor)
1647{
1648 int r;
1649
1650 idr_preload(GFP_KERNEL);
1651 spin_lock(&_minor_lock);
1652
1653 r = idr_alloc(&_minor_idr, MINOR_ALLOCED, 0, 1 << MINORBITS, GFP_NOWAIT);
1654
1655 spin_unlock(&_minor_lock);
1656 idr_preload_end();
1657 if (r < 0)
1658 return r;
1659 *minor = r;
1660 return 0;
1661}
1662
1663static const struct block_device_operations dm_blk_dops;
1664static const struct block_device_operations dm_rq_blk_dops;
1665static const struct dax_operations dm_dax_ops;
1666
1667static void dm_wq_work(struct work_struct *work);
1668
1669#ifdef CONFIG_BLK_INLINE_ENCRYPTION
1670static void dm_queue_destroy_keyslot_manager(struct request_queue *q)
1671{
1672 dm_destroy_keyslot_manager(q->ksm);
1673}
1674
1675#else /* CONFIG_BLK_INLINE_ENCRYPTION */
1676
1677static inline void dm_queue_destroy_keyslot_manager(struct request_queue *q)
1678{
1679}
1680#endif /* !CONFIG_BLK_INLINE_ENCRYPTION */
1681
1682static void cleanup_mapped_device(struct mapped_device *md)
1683{
1684 if (md->wq)
1685 destroy_workqueue(md->wq);
1686 bioset_exit(&md->bs);
1687 bioset_exit(&md->io_bs);
1688
1689 if (md->dax_dev) {
1690 kill_dax(md->dax_dev);
1691 put_dax(md->dax_dev);
1692 md->dax_dev = NULL;
1693 }
1694
1695 if (md->disk) {
1696 spin_lock(&_minor_lock);
1697 md->disk->private_data = NULL;
1698 spin_unlock(&_minor_lock);
1699 del_gendisk(md->disk);
1700 }
1701
1702 if (md->queue)
1703 dm_queue_destroy_keyslot_manager(md->queue);
1704
1705 if (md->disk)
1706 blk_cleanup_disk(md->disk);
1707
1708 cleanup_srcu_struct(&md->io_barrier);
1709
1710 mutex_destroy(&md->suspend_lock);
1711 mutex_destroy(&md->type_lock);
1712 mutex_destroy(&md->table_devices_lock);
1713 mutex_destroy(&md->swap_bios_lock);
1714
1715 dm_mq_cleanup_mapped_device(md);
1716 dm_cleanup_zoned_dev(md);
1717}
1718
1719/*
1720 * Allocate and initialise a blank device with a given minor.
1721 */
1722static struct mapped_device *alloc_dev(int minor)
1723{
1724 int r, numa_node_id = dm_get_numa_node();
1725 struct mapped_device *md;
1726 void *old_md;
1727
1728 md = kvzalloc_node(sizeof(*md), GFP_KERNEL, numa_node_id);
1729 if (!md) {
1730 DMWARN("unable to allocate device, out of memory.");
1731 return NULL;
1732 }
1733
1734 if (!try_module_get(THIS_MODULE))
1735 goto bad_module_get;
1736
1737 /* get a minor number for the dev */
1738 if (minor == DM_ANY_MINOR)
1739 r = next_free_minor(&minor);
1740 else
1741 r = specific_minor(minor);
1742 if (r < 0)
1743 goto bad_minor;
1744
1745 r = init_srcu_struct(&md->io_barrier);
1746 if (r < 0)
1747 goto bad_io_barrier;
1748
1749 md->numa_node_id = numa_node_id;
1750 md->init_tio_pdu = false;
1751 md->type = DM_TYPE_NONE;
1752 mutex_init(&md->suspend_lock);
1753 mutex_init(&md->type_lock);
1754 mutex_init(&md->table_devices_lock);
1755 spin_lock_init(&md->deferred_lock);
1756 atomic_set(&md->holders, 1);
1757 atomic_set(&md->open_count, 0);
1758 atomic_set(&md->event_nr, 0);
1759 atomic_set(&md->uevent_seq, 0);
1760 INIT_LIST_HEAD(&md->uevent_list);
1761 INIT_LIST_HEAD(&md->table_devices);
1762 spin_lock_init(&md->uevent_lock);
1763
1764 /*
1765 * default to bio-based until DM table is loaded and md->type
1766 * established. If request-based table is loaded: blk-mq will
1767 * override accordingly.
1768 */
1769 md->disk = blk_alloc_disk(md->numa_node_id);
1770 if (!md->disk)
1771 goto bad;
1772 md->queue = md->disk->queue;
1773
1774 init_waitqueue_head(&md->wait);
1775 INIT_WORK(&md->work, dm_wq_work);
1776 init_waitqueue_head(&md->eventq);
1777 init_completion(&md->kobj_holder.completion);
1778
1779 md->swap_bios = get_swap_bios();
1780 sema_init(&md->swap_bios_semaphore, md->swap_bios);
1781 mutex_init(&md->swap_bios_lock);
1782
1783 md->disk->major = _major;
1784 md->disk->first_minor = minor;
1785 md->disk->minors = 1;
1786 md->disk->fops = &dm_blk_dops;
1787 md->disk->queue = md->queue;
1788 md->disk->private_data = md;
1789 sprintf(md->disk->disk_name, "dm-%d", minor);
1790
1791 if (IS_ENABLED(CONFIG_DAX_DRIVER)) {
1792 md->dax_dev = alloc_dax(md, md->disk->disk_name,
1793 &dm_dax_ops, 0);
1794 if (IS_ERR(md->dax_dev))
1795 goto bad;
1796 }
1797
1798 add_disk_no_queue_reg(md->disk);
1799 format_dev_t(md->name, MKDEV(_major, minor));
1800
1801 md->wq = alloc_workqueue("kdmflush", WQ_MEM_RECLAIM, 0);
1802 if (!md->wq)
1803 goto bad;
1804
1805 dm_stats_init(&md->stats);
1806
1807 /* Populate the mapping, nobody knows we exist yet */
1808 spin_lock(&_minor_lock);
1809 old_md = idr_replace(&_minor_idr, md, minor);
1810 spin_unlock(&_minor_lock);
1811
1812 BUG_ON(old_md != MINOR_ALLOCED);
1813
1814 return md;
1815
1816bad:
1817 cleanup_mapped_device(md);
1818bad_io_barrier:
1819 free_minor(minor);
1820bad_minor:
1821 module_put(THIS_MODULE);
1822bad_module_get:
1823 kvfree(md);
1824 return NULL;
1825}
1826
1827static void unlock_fs(struct mapped_device *md);
1828
1829static void free_dev(struct mapped_device *md)
1830{
1831 int minor = MINOR(disk_devt(md->disk));
1832
1833 unlock_fs(md);
1834
1835 cleanup_mapped_device(md);
1836
1837 free_table_devices(&md->table_devices);
1838 dm_stats_cleanup(&md->stats);
1839 free_minor(minor);
1840
1841 module_put(THIS_MODULE);
1842 kvfree(md);
1843}
1844
1845static int __bind_mempools(struct mapped_device *md, struct dm_table *t)
1846{
1847 struct dm_md_mempools *p = dm_table_get_md_mempools(t);
1848 int ret = 0;
1849
1850 if (dm_table_bio_based(t)) {
1851 /*
1852 * The md may already have mempools that need changing.
1853 * If so, reload bioset because front_pad may have changed
1854 * because a different table was loaded.
1855 */
1856 bioset_exit(&md->bs);
1857 bioset_exit(&md->io_bs);
1858
1859 } else if (bioset_initialized(&md->bs)) {
1860 /*
1861 * There's no need to reload with request-based dm
1862 * because the size of front_pad doesn't change.
1863 * Note for future: If you are to reload bioset,
1864 * prep-ed requests in the queue may refer
1865 * to bio from the old bioset, so you must walk
1866 * through the queue to unprep.
1867 */
1868 goto out;
1869 }
1870
1871 BUG_ON(!p ||
1872 bioset_initialized(&md->bs) ||
1873 bioset_initialized(&md->io_bs));
1874
1875 ret = bioset_init_from_src(&md->bs, &p->bs);
1876 if (ret)
1877 goto out;
1878 ret = bioset_init_from_src(&md->io_bs, &p->io_bs);
1879 if (ret)
1880 bioset_exit(&md->bs);
1881out:
1882 /* mempool bind completed, no longer need any mempools in the table */
1883 dm_table_free_md_mempools(t);
1884 return ret;
1885}
1886
1887/*
1888 * Bind a table to the device.
1889 */
1890static void event_callback(void *context)
1891{
1892 unsigned long flags;
1893 LIST_HEAD(uevents);
1894 struct mapped_device *md = (struct mapped_device *) context;
1895
1896 spin_lock_irqsave(&md->uevent_lock, flags);
1897 list_splice_init(&md->uevent_list, &uevents);
1898 spin_unlock_irqrestore(&md->uevent_lock, flags);
1899
1900 dm_send_uevents(&uevents, &disk_to_dev(md->disk)->kobj);
1901
1902 atomic_inc(&md->event_nr);
1903 wake_up(&md->eventq);
1904 dm_issue_global_event();
1905}
1906
1907/*
1908 * Returns old map, which caller must destroy.
1909 */
1910static struct dm_table *__bind(struct mapped_device *md, struct dm_table *t,
1911 struct queue_limits *limits)
1912{
1913 struct dm_table *old_map;
1914 struct request_queue *q = md->queue;
1915 bool request_based = dm_table_request_based(t);
1916 sector_t size;
1917 int ret;
1918
1919 lockdep_assert_held(&md->suspend_lock);
1920
1921 size = dm_table_get_size(t);
1922
1923 /*
1924 * Wipe any geometry if the size of the table changed.
1925 */
1926 if (size != dm_get_size(md))
1927 memset(&md->geometry, 0, sizeof(md->geometry));
1928
1929 if (!get_capacity(md->disk))
1930 set_capacity(md->disk, size);
1931 else
1932 set_capacity_and_notify(md->disk, size);
1933
1934 dm_table_event_callback(t, event_callback, md);
1935
1936 /*
1937 * The queue hasn't been stopped yet, if the old table type wasn't
1938 * for request-based during suspension. So stop it to prevent
1939 * I/O mapping before resume.
1940 * This must be done before setting the queue restrictions,
1941 * because request-based dm may be run just after the setting.
1942 */
1943 if (request_based)
1944 dm_stop_queue(q);
1945
1946 if (request_based) {
1947 /*
1948 * Leverage the fact that request-based DM targets are
1949 * immutable singletons - used to optimize dm_mq_queue_rq.
1950 */
1951 md->immutable_target = dm_table_get_immutable_target(t);
1952 }
1953
1954 ret = __bind_mempools(md, t);
1955 if (ret) {
1956 old_map = ERR_PTR(ret);
1957 goto out;
1958 }
1959
1960 ret = dm_table_set_restrictions(t, q, limits);
1961 if (ret) {
1962 old_map = ERR_PTR(ret);
1963 goto out;
1964 }
1965
1966 old_map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
1967 rcu_assign_pointer(md->map, (void *)t);
1968 md->immutable_target_type = dm_table_get_immutable_target_type(t);
1969
1970 if (old_map)
1971 dm_sync_table(md);
1972
1973out:
1974 return old_map;
1975}
1976
1977/*
1978 * Returns unbound table for the caller to free.
1979 */
1980static struct dm_table *__unbind(struct mapped_device *md)
1981{
1982 struct dm_table *map = rcu_dereference_protected(md->map, 1);
1983
1984 if (!map)
1985 return NULL;
1986
1987 dm_table_event_callback(map, NULL, NULL);
1988 RCU_INIT_POINTER(md->map, NULL);
1989 dm_sync_table(md);
1990
1991 return map;
1992}
1993
1994/*
1995 * Constructor for a new device.
1996 */
1997int dm_create(int minor, struct mapped_device **result)
1998{
1999 int r;
2000 struct mapped_device *md;
2001
2002 md = alloc_dev(minor);
2003 if (!md)
2004 return -ENXIO;
2005
2006 r = dm_sysfs_init(md);
2007 if (r) {
2008 free_dev(md);
2009 return r;
2010 }
2011
2012 *result = md;
2013 return 0;
2014}
2015
2016/*
2017 * Functions to manage md->type.
2018 * All are required to hold md->type_lock.
2019 */
2020void dm_lock_md_type(struct mapped_device *md)
2021{
2022 mutex_lock(&md->type_lock);
2023}
2024
2025void dm_unlock_md_type(struct mapped_device *md)
2026{
2027 mutex_unlock(&md->type_lock);
2028}
2029
2030void dm_set_md_type(struct mapped_device *md, enum dm_queue_mode type)
2031{
2032 BUG_ON(!mutex_is_locked(&md->type_lock));
2033 md->type = type;
2034}
2035
2036enum dm_queue_mode dm_get_md_type(struct mapped_device *md)
2037{
2038 return md->type;
2039}
2040
2041struct target_type *dm_get_immutable_target_type(struct mapped_device *md)
2042{
2043 return md->immutable_target_type;
2044}
2045
2046/*
2047 * The queue_limits are only valid as long as you have a reference
2048 * count on 'md'.
2049 */
2050struct queue_limits *dm_get_queue_limits(struct mapped_device *md)
2051{
2052 BUG_ON(!atomic_read(&md->holders));
2053 return &md->queue->limits;
2054}
2055EXPORT_SYMBOL_GPL(dm_get_queue_limits);
2056
2057/*
2058 * Setup the DM device's queue based on md's type
2059 */
2060int dm_setup_md_queue(struct mapped_device *md, struct dm_table *t)
2061{
2062 int r;
2063 struct queue_limits limits;
2064 enum dm_queue_mode type = dm_get_md_type(md);
2065
2066 switch (type) {
2067 case DM_TYPE_REQUEST_BASED:
2068 md->disk->fops = &dm_rq_blk_dops;
2069 r = dm_mq_init_request_queue(md, t);
2070 if (r) {
2071 DMERR("Cannot initialize queue for request-based dm mapped device");
2072 return r;
2073 }
2074 break;
2075 case DM_TYPE_BIO_BASED:
2076 case DM_TYPE_DAX_BIO_BASED:
2077 break;
2078 case DM_TYPE_NONE:
2079 WARN_ON_ONCE(true);
2080 break;
2081 }
2082
2083 r = dm_calculate_queue_limits(t, &limits);
2084 if (r) {
2085 DMERR("Cannot calculate initial queue limits");
2086 return r;
2087 }
2088 r = dm_table_set_restrictions(t, md->queue, &limits);
2089 if (r)
2090 return r;
2091
2092 blk_register_queue(md->disk);
2093
2094 return 0;
2095}
2096
2097struct mapped_device *dm_get_md(dev_t dev)
2098{
2099 struct mapped_device *md;
2100 unsigned minor = MINOR(dev);
2101
2102 if (MAJOR(dev) != _major || minor >= (1 << MINORBITS))
2103 return NULL;
2104
2105 spin_lock(&_minor_lock);
2106
2107 md = idr_find(&_minor_idr, minor);
2108 if (!md || md == MINOR_ALLOCED || (MINOR(disk_devt(dm_disk(md))) != minor) ||
2109 test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2110 md = NULL;
2111 goto out;
2112 }
2113 dm_get(md);
2114out:
2115 spin_unlock(&_minor_lock);
2116
2117 return md;
2118}
2119EXPORT_SYMBOL_GPL(dm_get_md);
2120
2121void *dm_get_mdptr(struct mapped_device *md)
2122{
2123 return md->interface_ptr;
2124}
2125
2126void dm_set_mdptr(struct mapped_device *md, void *ptr)
2127{
2128 md->interface_ptr = ptr;
2129}
2130
2131void dm_get(struct mapped_device *md)
2132{
2133 atomic_inc(&md->holders);
2134 BUG_ON(test_bit(DMF_FREEING, &md->flags));
2135}
2136
2137int dm_hold(struct mapped_device *md)
2138{
2139 spin_lock(&_minor_lock);
2140 if (test_bit(DMF_FREEING, &md->flags)) {
2141 spin_unlock(&_minor_lock);
2142 return -EBUSY;
2143 }
2144 dm_get(md);
2145 spin_unlock(&_minor_lock);
2146 return 0;
2147}
2148EXPORT_SYMBOL_GPL(dm_hold);
2149
2150const char *dm_device_name(struct mapped_device *md)
2151{
2152 return md->name;
2153}
2154EXPORT_SYMBOL_GPL(dm_device_name);
2155
2156static void __dm_destroy(struct mapped_device *md, bool wait)
2157{
2158 struct dm_table *map;
2159 int srcu_idx;
2160
2161 might_sleep();
2162
2163 spin_lock(&_minor_lock);
2164 idr_replace(&_minor_idr, MINOR_ALLOCED, MINOR(disk_devt(dm_disk(md))));
2165 set_bit(DMF_FREEING, &md->flags);
2166 spin_unlock(&_minor_lock);
2167
2168 blk_set_queue_dying(md->queue);
2169
2170 /*
2171 * Take suspend_lock so that presuspend and postsuspend methods
2172 * do not race with internal suspend.
2173 */
2174 mutex_lock(&md->suspend_lock);
2175 map = dm_get_live_table(md, &srcu_idx);
2176 if (!dm_suspended_md(md)) {
2177 dm_table_presuspend_targets(map);
2178 set_bit(DMF_SUSPENDED, &md->flags);
2179 set_bit(DMF_POST_SUSPENDING, &md->flags);
2180 dm_table_postsuspend_targets(map);
2181 }
2182 /* dm_put_live_table must be before msleep, otherwise deadlock is possible */
2183 dm_put_live_table(md, srcu_idx);
2184 mutex_unlock(&md->suspend_lock);
2185
2186 /*
2187 * Rare, but there may be I/O requests still going to complete,
2188 * for example. Wait for all references to disappear.
2189 * No one should increment the reference count of the mapped_device,
2190 * after the mapped_device state becomes DMF_FREEING.
2191 */
2192 if (wait)
2193 while (atomic_read(&md->holders))
2194 msleep(1);
2195 else if (atomic_read(&md->holders))
2196 DMWARN("%s: Forcibly removing mapped_device still in use! (%d users)",
2197 dm_device_name(md), atomic_read(&md->holders));
2198
2199 dm_sysfs_exit(md);
2200 dm_table_destroy(__unbind(md));
2201 free_dev(md);
2202}
2203
2204void dm_destroy(struct mapped_device *md)
2205{
2206 __dm_destroy(md, true);
2207}
2208
2209void dm_destroy_immediate(struct mapped_device *md)
2210{
2211 __dm_destroy(md, false);
2212}
2213
2214void dm_put(struct mapped_device *md)
2215{
2216 atomic_dec(&md->holders);
2217}
2218EXPORT_SYMBOL_GPL(dm_put);
2219
2220static bool md_in_flight_bios(struct mapped_device *md)
2221{
2222 int cpu;
2223 struct block_device *part = dm_disk(md)->part0;
2224 long sum = 0;
2225
2226 for_each_possible_cpu(cpu) {
2227 sum += part_stat_local_read_cpu(part, in_flight[0], cpu);
2228 sum += part_stat_local_read_cpu(part, in_flight[1], cpu);
2229 }
2230
2231 return sum != 0;
2232}
2233
2234static int dm_wait_for_bios_completion(struct mapped_device *md, unsigned int task_state)
2235{
2236 int r = 0;
2237 DEFINE_WAIT(wait);
2238
2239 while (true) {
2240 prepare_to_wait(&md->wait, &wait, task_state);
2241
2242 if (!md_in_flight_bios(md))
2243 break;
2244
2245 if (signal_pending_state(task_state, current)) {
2246 r = -EINTR;
2247 break;
2248 }
2249
2250 io_schedule();
2251 }
2252 finish_wait(&md->wait, &wait);
2253
2254 return r;
2255}
2256
2257static int dm_wait_for_completion(struct mapped_device *md, unsigned int task_state)
2258{
2259 int r = 0;
2260
2261 if (!queue_is_mq(md->queue))
2262 return dm_wait_for_bios_completion(md, task_state);
2263
2264 while (true) {
2265 if (!blk_mq_queue_inflight(md->queue))
2266 break;
2267
2268 if (signal_pending_state(task_state, current)) {
2269 r = -EINTR;
2270 break;
2271 }
2272
2273 msleep(5);
2274 }
2275
2276 return r;
2277}
2278
2279/*
2280 * Process the deferred bios
2281 */
2282static void dm_wq_work(struct work_struct *work)
2283{
2284 struct mapped_device *md = container_of(work, struct mapped_device, work);
2285 struct bio *bio;
2286
2287 while (!test_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags)) {
2288 spin_lock_irq(&md->deferred_lock);
2289 bio = bio_list_pop(&md->deferred);
2290 spin_unlock_irq(&md->deferred_lock);
2291
2292 if (!bio)
2293 break;
2294
2295 submit_bio_noacct(bio);
2296 }
2297}
2298
2299static void dm_queue_flush(struct mapped_device *md)
2300{
2301 clear_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2302 smp_mb__after_atomic();
2303 queue_work(md->wq, &md->work);
2304}
2305
2306/*
2307 * Swap in a new table, returning the old one for the caller to destroy.
2308 */
2309struct dm_table *dm_swap_table(struct mapped_device *md, struct dm_table *table)
2310{
2311 struct dm_table *live_map = NULL, *map = ERR_PTR(-EINVAL);
2312 struct queue_limits limits;
2313 int r;
2314
2315 mutex_lock(&md->suspend_lock);
2316
2317 /* device must be suspended */
2318 if (!dm_suspended_md(md))
2319 goto out;
2320
2321 /*
2322 * If the new table has no data devices, retain the existing limits.
2323 * This helps multipath with queue_if_no_path if all paths disappear,
2324 * then new I/O is queued based on these limits, and then some paths
2325 * reappear.
2326 */
2327 if (dm_table_has_no_data_devices(table)) {
2328 live_map = dm_get_live_table_fast(md);
2329 if (live_map)
2330 limits = md->queue->limits;
2331 dm_put_live_table_fast(md);
2332 }
2333
2334 if (!live_map) {
2335 r = dm_calculate_queue_limits(table, &limits);
2336 if (r) {
2337 map = ERR_PTR(r);
2338 goto out;
2339 }
2340 }
2341
2342 map = __bind(md, table, &limits);
2343 dm_issue_global_event();
2344
2345out:
2346 mutex_unlock(&md->suspend_lock);
2347 return map;
2348}
2349
2350/*
2351 * Functions to lock and unlock any filesystem running on the
2352 * device.
2353 */
2354static int lock_fs(struct mapped_device *md)
2355{
2356 int r;
2357
2358 WARN_ON(test_bit(DMF_FROZEN, &md->flags));
2359
2360 r = freeze_bdev(md->disk->part0);
2361 if (!r)
2362 set_bit(DMF_FROZEN, &md->flags);
2363 return r;
2364}
2365
2366static void unlock_fs(struct mapped_device *md)
2367{
2368 if (!test_bit(DMF_FROZEN, &md->flags))
2369 return;
2370 thaw_bdev(md->disk->part0);
2371 clear_bit(DMF_FROZEN, &md->flags);
2372}
2373
2374/*
2375 * @suspend_flags: DM_SUSPEND_LOCKFS_FLAG and/or DM_SUSPEND_NOFLUSH_FLAG
2376 * @task_state: e.g. TASK_INTERRUPTIBLE or TASK_UNINTERRUPTIBLE
2377 * @dmf_suspended_flag: DMF_SUSPENDED or DMF_SUSPENDED_INTERNALLY
2378 *
2379 * If __dm_suspend returns 0, the device is completely quiescent
2380 * now. There is no request-processing activity. All new requests
2381 * are being added to md->deferred list.
2382 */
2383static int __dm_suspend(struct mapped_device *md, struct dm_table *map,
2384 unsigned suspend_flags, unsigned int task_state,
2385 int dmf_suspended_flag)
2386{
2387 bool do_lockfs = suspend_flags & DM_SUSPEND_LOCKFS_FLAG;
2388 bool noflush = suspend_flags & DM_SUSPEND_NOFLUSH_FLAG;
2389 int r;
2390
2391 lockdep_assert_held(&md->suspend_lock);
2392
2393 /*
2394 * DMF_NOFLUSH_SUSPENDING must be set before presuspend.
2395 * This flag is cleared before dm_suspend returns.
2396 */
2397 if (noflush)
2398 set_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2399 else
2400 DMDEBUG("%s: suspending with flush", dm_device_name(md));
2401
2402 /*
2403 * This gets reverted if there's an error later and the targets
2404 * provide the .presuspend_undo hook.
2405 */
2406 dm_table_presuspend_targets(map);
2407
2408 /*
2409 * Flush I/O to the device.
2410 * Any I/O submitted after lock_fs() may not be flushed.
2411 * noflush takes precedence over do_lockfs.
2412 * (lock_fs() flushes I/Os and waits for them to complete.)
2413 */
2414 if (!noflush && do_lockfs) {
2415 r = lock_fs(md);
2416 if (r) {
2417 dm_table_presuspend_undo_targets(map);
2418 return r;
2419 }
2420 }
2421
2422 /*
2423 * Here we must make sure that no processes are submitting requests
2424 * to target drivers i.e. no one may be executing
2425 * __split_and_process_bio from dm_submit_bio.
2426 *
2427 * To get all processes out of __split_and_process_bio in dm_submit_bio,
2428 * we take the write lock. To prevent any process from reentering
2429 * __split_and_process_bio from dm_submit_bio and quiesce the thread
2430 * (dm_wq_work), we set DMF_BLOCK_IO_FOR_SUSPEND and call
2431 * flush_workqueue(md->wq).
2432 */
2433 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2434 if (map)
2435 synchronize_srcu(&md->io_barrier);
2436
2437 /*
2438 * Stop md->queue before flushing md->wq in case request-based
2439 * dm defers requests to md->wq from md->queue.
2440 */
2441 if (dm_request_based(md))
2442 dm_stop_queue(md->queue);
2443
2444 flush_workqueue(md->wq);
2445
2446 /*
2447 * At this point no more requests are entering target request routines.
2448 * We call dm_wait_for_completion to wait for all existing requests
2449 * to finish.
2450 */
2451 r = dm_wait_for_completion(md, task_state);
2452 if (!r)
2453 set_bit(dmf_suspended_flag, &md->flags);
2454
2455 if (noflush)
2456 clear_bit(DMF_NOFLUSH_SUSPENDING, &md->flags);
2457 if (map)
2458 synchronize_srcu(&md->io_barrier);
2459
2460 /* were we interrupted ? */
2461 if (r < 0) {
2462 dm_queue_flush(md);
2463
2464 if (dm_request_based(md))
2465 dm_start_queue(md->queue);
2466
2467 unlock_fs(md);
2468 dm_table_presuspend_undo_targets(map);
2469 /* pushback list is already flushed, so skip flush */
2470 }
2471
2472 return r;
2473}
2474
2475/*
2476 * We need to be able to change a mapping table under a mounted
2477 * filesystem. For example we might want to move some data in
2478 * the background. Before the table can be swapped with
2479 * dm_bind_table, dm_suspend must be called to flush any in
2480 * flight bios and ensure that any further io gets deferred.
2481 */
2482/*
2483 * Suspend mechanism in request-based dm.
2484 *
2485 * 1. Flush all I/Os by lock_fs() if needed.
2486 * 2. Stop dispatching any I/O by stopping the request_queue.
2487 * 3. Wait for all in-flight I/Os to be completed or requeued.
2488 *
2489 * To abort suspend, start the request_queue.
2490 */
2491int dm_suspend(struct mapped_device *md, unsigned suspend_flags)
2492{
2493 struct dm_table *map = NULL;
2494 int r = 0;
2495
2496retry:
2497 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2498
2499 if (dm_suspended_md(md)) {
2500 r = -EINVAL;
2501 goto out_unlock;
2502 }
2503
2504 if (dm_suspended_internally_md(md)) {
2505 /* already internally suspended, wait for internal resume */
2506 mutex_unlock(&md->suspend_lock);
2507 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2508 if (r)
2509 return r;
2510 goto retry;
2511 }
2512
2513 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2514
2515 r = __dm_suspend(md, map, suspend_flags, TASK_INTERRUPTIBLE, DMF_SUSPENDED);
2516 if (r)
2517 goto out_unlock;
2518
2519 set_bit(DMF_POST_SUSPENDING, &md->flags);
2520 dm_table_postsuspend_targets(map);
2521 clear_bit(DMF_POST_SUSPENDING, &md->flags);
2522
2523out_unlock:
2524 mutex_unlock(&md->suspend_lock);
2525 return r;
2526}
2527
2528static int __dm_resume(struct mapped_device *md, struct dm_table *map)
2529{
2530 if (map) {
2531 int r = dm_table_resume_targets(map);
2532 if (r)
2533 return r;
2534 }
2535
2536 dm_queue_flush(md);
2537
2538 /*
2539 * Flushing deferred I/Os must be done after targets are resumed
2540 * so that mapping of targets can work correctly.
2541 * Request-based dm is queueing the deferred I/Os in its request_queue.
2542 */
2543 if (dm_request_based(md))
2544 dm_start_queue(md->queue);
2545
2546 unlock_fs(md);
2547
2548 return 0;
2549}
2550
2551int dm_resume(struct mapped_device *md)
2552{
2553 int r;
2554 struct dm_table *map = NULL;
2555
2556retry:
2557 r = -EINVAL;
2558 mutex_lock_nested(&md->suspend_lock, SINGLE_DEPTH_NESTING);
2559
2560 if (!dm_suspended_md(md))
2561 goto out;
2562
2563 if (dm_suspended_internally_md(md)) {
2564 /* already internally suspended, wait for internal resume */
2565 mutex_unlock(&md->suspend_lock);
2566 r = wait_on_bit(&md->flags, DMF_SUSPENDED_INTERNALLY, TASK_INTERRUPTIBLE);
2567 if (r)
2568 return r;
2569 goto retry;
2570 }
2571
2572 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2573 if (!map || !dm_table_get_size(map))
2574 goto out;
2575
2576 r = __dm_resume(md, map);
2577 if (r)
2578 goto out;
2579
2580 clear_bit(DMF_SUSPENDED, &md->flags);
2581out:
2582 mutex_unlock(&md->suspend_lock);
2583
2584 return r;
2585}
2586
2587/*
2588 * Internal suspend/resume works like userspace-driven suspend. It waits
2589 * until all bios finish and prevents issuing new bios to the target drivers.
2590 * It may be used only from the kernel.
2591 */
2592
2593static void __dm_internal_suspend(struct mapped_device *md, unsigned suspend_flags)
2594{
2595 struct dm_table *map = NULL;
2596
2597 lockdep_assert_held(&md->suspend_lock);
2598
2599 if (md->internal_suspend_count++)
2600 return; /* nested internal suspend */
2601
2602 if (dm_suspended_md(md)) {
2603 set_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2604 return; /* nest suspend */
2605 }
2606
2607 map = rcu_dereference_protected(md->map, lockdep_is_held(&md->suspend_lock));
2608
2609 /*
2610 * Using TASK_UNINTERRUPTIBLE because only NOFLUSH internal suspend is
2611 * supported. Properly supporting a TASK_INTERRUPTIBLE internal suspend
2612 * would require changing .presuspend to return an error -- avoid this
2613 * until there is a need for more elaborate variants of internal suspend.
2614 */
2615 (void) __dm_suspend(md, map, suspend_flags, TASK_UNINTERRUPTIBLE,
2616 DMF_SUSPENDED_INTERNALLY);
2617
2618 set_bit(DMF_POST_SUSPENDING, &md->flags);
2619 dm_table_postsuspend_targets(map);
2620 clear_bit(DMF_POST_SUSPENDING, &md->flags);
2621}
2622
2623static void __dm_internal_resume(struct mapped_device *md)
2624{
2625 BUG_ON(!md->internal_suspend_count);
2626
2627 if (--md->internal_suspend_count)
2628 return; /* resume from nested internal suspend */
2629
2630 if (dm_suspended_md(md))
2631 goto done; /* resume from nested suspend */
2632
2633 /*
2634 * NOTE: existing callers don't need to call dm_table_resume_targets
2635 * (which may fail -- so best to avoid it for now by passing NULL map)
2636 */
2637 (void) __dm_resume(md, NULL);
2638
2639done:
2640 clear_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2641 smp_mb__after_atomic();
2642 wake_up_bit(&md->flags, DMF_SUSPENDED_INTERNALLY);
2643}
2644
2645void dm_internal_suspend_noflush(struct mapped_device *md)
2646{
2647 mutex_lock(&md->suspend_lock);
2648 __dm_internal_suspend(md, DM_SUSPEND_NOFLUSH_FLAG);
2649 mutex_unlock(&md->suspend_lock);
2650}
2651EXPORT_SYMBOL_GPL(dm_internal_suspend_noflush);
2652
2653void dm_internal_resume(struct mapped_device *md)
2654{
2655 mutex_lock(&md->suspend_lock);
2656 __dm_internal_resume(md);
2657 mutex_unlock(&md->suspend_lock);
2658}
2659EXPORT_SYMBOL_GPL(dm_internal_resume);
2660
2661/*
2662 * Fast variants of internal suspend/resume hold md->suspend_lock,
2663 * which prevents interaction with userspace-driven suspend.
2664 */
2665
2666void dm_internal_suspend_fast(struct mapped_device *md)
2667{
2668 mutex_lock(&md->suspend_lock);
2669 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2670 return;
2671
2672 set_bit(DMF_BLOCK_IO_FOR_SUSPEND, &md->flags);
2673 synchronize_srcu(&md->io_barrier);
2674 flush_workqueue(md->wq);
2675 dm_wait_for_completion(md, TASK_UNINTERRUPTIBLE);
2676}
2677EXPORT_SYMBOL_GPL(dm_internal_suspend_fast);
2678
2679void dm_internal_resume_fast(struct mapped_device *md)
2680{
2681 if (dm_suspended_md(md) || dm_suspended_internally_md(md))
2682 goto done;
2683
2684 dm_queue_flush(md);
2685
2686done:
2687 mutex_unlock(&md->suspend_lock);
2688}
2689EXPORT_SYMBOL_GPL(dm_internal_resume_fast);
2690
2691/*-----------------------------------------------------------------
2692 * Event notification.
2693 *---------------------------------------------------------------*/
2694int dm_kobject_uevent(struct mapped_device *md, enum kobject_action action,
2695 unsigned cookie)
2696{
2697 int r;
2698 unsigned noio_flag;
2699 char udev_cookie[DM_COOKIE_LENGTH];
2700 char *envp[] = { udev_cookie, NULL };
2701
2702 noio_flag = memalloc_noio_save();
2703
2704 if (!cookie)
2705 r = kobject_uevent(&disk_to_dev(md->disk)->kobj, action);
2706 else {
2707 snprintf(udev_cookie, DM_COOKIE_LENGTH, "%s=%u",
2708 DM_COOKIE_ENV_VAR_NAME, cookie);
2709 r = kobject_uevent_env(&disk_to_dev(md->disk)->kobj,
2710 action, envp);
2711 }
2712
2713 memalloc_noio_restore(noio_flag);
2714
2715 return r;
2716}
2717
2718uint32_t dm_next_uevent_seq(struct mapped_device *md)
2719{
2720 return atomic_add_return(1, &md->uevent_seq);
2721}
2722
2723uint32_t dm_get_event_nr(struct mapped_device *md)
2724{
2725 return atomic_read(&md->event_nr);
2726}
2727
2728int dm_wait_event(struct mapped_device *md, int event_nr)
2729{
2730 return wait_event_interruptible(md->eventq,
2731 (event_nr != atomic_read(&md->event_nr)));
2732}
2733
2734void dm_uevent_add(struct mapped_device *md, struct list_head *elist)
2735{
2736 unsigned long flags;
2737
2738 spin_lock_irqsave(&md->uevent_lock, flags);
2739 list_add(elist, &md->uevent_list);
2740 spin_unlock_irqrestore(&md->uevent_lock, flags);
2741}
2742
2743/*
2744 * The gendisk is only valid as long as you have a reference
2745 * count on 'md'.
2746 */
2747struct gendisk *dm_disk(struct mapped_device *md)
2748{
2749 return md->disk;
2750}
2751EXPORT_SYMBOL_GPL(dm_disk);
2752
2753struct kobject *dm_kobject(struct mapped_device *md)
2754{
2755 return &md->kobj_holder.kobj;
2756}
2757
2758struct mapped_device *dm_get_from_kobject(struct kobject *kobj)
2759{
2760 struct mapped_device *md;
2761
2762 md = container_of(kobj, struct mapped_device, kobj_holder.kobj);
2763
2764 spin_lock(&_minor_lock);
2765 if (test_bit(DMF_FREEING, &md->flags) || dm_deleting_md(md)) {
2766 md = NULL;
2767 goto out;
2768 }
2769 dm_get(md);
2770out:
2771 spin_unlock(&_minor_lock);
2772
2773 return md;
2774}
2775
2776int dm_suspended_md(struct mapped_device *md)
2777{
2778 return test_bit(DMF_SUSPENDED, &md->flags);
2779}
2780
2781static int dm_post_suspending_md(struct mapped_device *md)
2782{
2783 return test_bit(DMF_POST_SUSPENDING, &md->flags);
2784}
2785
2786int dm_suspended_internally_md(struct mapped_device *md)
2787{
2788 return test_bit(DMF_SUSPENDED_INTERNALLY, &md->flags);
2789}
2790
2791int dm_test_deferred_remove_flag(struct mapped_device *md)
2792{
2793 return test_bit(DMF_DEFERRED_REMOVE, &md->flags);
2794}
2795
2796int dm_suspended(struct dm_target *ti)
2797{
2798 return dm_suspended_md(ti->table->md);
2799}
2800EXPORT_SYMBOL_GPL(dm_suspended);
2801
2802int dm_post_suspending(struct dm_target *ti)
2803{
2804 return dm_post_suspending_md(ti->table->md);
2805}
2806EXPORT_SYMBOL_GPL(dm_post_suspending);
2807
2808int dm_noflush_suspending(struct dm_target *ti)
2809{
2810 return __noflush_suspending(ti->table->md);
2811}
2812EXPORT_SYMBOL_GPL(dm_noflush_suspending);
2813
2814struct dm_md_mempools *dm_alloc_md_mempools(struct mapped_device *md, enum dm_queue_mode type,
2815 unsigned integrity, unsigned per_io_data_size,
2816 unsigned min_pool_size)
2817{
2818 struct dm_md_mempools *pools = kzalloc_node(sizeof(*pools), GFP_KERNEL, md->numa_node_id);
2819 unsigned int pool_size = 0;
2820 unsigned int front_pad, io_front_pad;
2821 int ret;
2822
2823 if (!pools)
2824 return NULL;
2825
2826 switch (type) {
2827 case DM_TYPE_BIO_BASED:
2828 case DM_TYPE_DAX_BIO_BASED:
2829 pool_size = max(dm_get_reserved_bio_based_ios(), min_pool_size);
2830 front_pad = roundup(per_io_data_size, __alignof__(struct dm_target_io)) + DM_TARGET_IO_BIO_OFFSET;
2831 io_front_pad = roundup(per_io_data_size, __alignof__(struct dm_io)) + DM_IO_BIO_OFFSET;
2832 ret = bioset_init(&pools->io_bs, pool_size, io_front_pad, 0);
2833 if (ret)
2834 goto out;
2835 if (integrity && bioset_integrity_create(&pools->io_bs, pool_size))
2836 goto out;
2837 break;
2838 case DM_TYPE_REQUEST_BASED:
2839 pool_size = max(dm_get_reserved_rq_based_ios(), min_pool_size);
2840 front_pad = offsetof(struct dm_rq_clone_bio_info, clone);
2841 /* per_io_data_size is used for blk-mq pdu at queue allocation */
2842 break;
2843 default:
2844 BUG();
2845 }
2846
2847 ret = bioset_init(&pools->bs, pool_size, front_pad, 0);
2848 if (ret)
2849 goto out;
2850
2851 if (integrity && bioset_integrity_create(&pools->bs, pool_size))
2852 goto out;
2853
2854 return pools;
2855
2856out:
2857 dm_free_md_mempools(pools);
2858
2859 return NULL;
2860}
2861
2862void dm_free_md_mempools(struct dm_md_mempools *pools)
2863{
2864 if (!pools)
2865 return;
2866
2867 bioset_exit(&pools->bs);
2868 bioset_exit(&pools->io_bs);
2869
2870 kfree(pools);
2871}
2872
2873struct dm_pr {
2874 u64 old_key;
2875 u64 new_key;
2876 u32 flags;
2877 bool fail_early;
2878};
2879
2880static int dm_call_pr(struct block_device *bdev, iterate_devices_callout_fn fn,
2881 void *data)
2882{
2883 struct mapped_device *md = bdev->bd_disk->private_data;
2884 struct dm_table *table;
2885 struct dm_target *ti;
2886 int ret = -ENOTTY, srcu_idx;
2887
2888 table = dm_get_live_table(md, &srcu_idx);
2889 if (!table || !dm_table_get_size(table))
2890 goto out;
2891
2892 /* We only support devices that have a single target */
2893 if (dm_table_get_num_targets(table) != 1)
2894 goto out;
2895 ti = dm_table_get_target(table, 0);
2896
2897 ret = -EINVAL;
2898 if (!ti->type->iterate_devices)
2899 goto out;
2900
2901 ret = ti->type->iterate_devices(ti, fn, data);
2902out:
2903 dm_put_live_table(md, srcu_idx);
2904 return ret;
2905}
2906
2907/*
2908 * For register / unregister we need to manually call out to every path.
2909 */
2910static int __dm_pr_register(struct dm_target *ti, struct dm_dev *dev,
2911 sector_t start, sector_t len, void *data)
2912{
2913 struct dm_pr *pr = data;
2914 const struct pr_ops *ops = dev->bdev->bd_disk->fops->pr_ops;
2915
2916 if (!ops || !ops->pr_register)
2917 return -EOPNOTSUPP;
2918 return ops->pr_register(dev->bdev, pr->old_key, pr->new_key, pr->flags);
2919}
2920
2921static int dm_pr_register(struct block_device *bdev, u64 old_key, u64 new_key,
2922 u32 flags)
2923{
2924 struct dm_pr pr = {
2925 .old_key = old_key,
2926 .new_key = new_key,
2927 .flags = flags,
2928 .fail_early = true,
2929 };
2930 int ret;
2931
2932 ret = dm_call_pr(bdev, __dm_pr_register, &pr);
2933 if (ret && new_key) {
2934 /* unregister all paths if we failed to register any path */
2935 pr.old_key = new_key;
2936 pr.new_key = 0;
2937 pr.flags = 0;
2938 pr.fail_early = false;
2939 dm_call_pr(bdev, __dm_pr_register, &pr);
2940 }
2941
2942 return ret;
2943}
2944
2945static int dm_pr_reserve(struct block_device *bdev, u64 key, enum pr_type type,
2946 u32 flags)
2947{
2948 struct mapped_device *md = bdev->bd_disk->private_data;
2949 const struct pr_ops *ops;
2950 int r, srcu_idx;
2951
2952 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
2953 if (r < 0)
2954 goto out;
2955
2956 ops = bdev->bd_disk->fops->pr_ops;
2957 if (ops && ops->pr_reserve)
2958 r = ops->pr_reserve(bdev, key, type, flags);
2959 else
2960 r = -EOPNOTSUPP;
2961out:
2962 dm_unprepare_ioctl(md, srcu_idx);
2963 return r;
2964}
2965
2966static int dm_pr_release(struct block_device *bdev, u64 key, enum pr_type type)
2967{
2968 struct mapped_device *md = bdev->bd_disk->private_data;
2969 const struct pr_ops *ops;
2970 int r, srcu_idx;
2971
2972 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
2973 if (r < 0)
2974 goto out;
2975
2976 ops = bdev->bd_disk->fops->pr_ops;
2977 if (ops && ops->pr_release)
2978 r = ops->pr_release(bdev, key, type);
2979 else
2980 r = -EOPNOTSUPP;
2981out:
2982 dm_unprepare_ioctl(md, srcu_idx);
2983 return r;
2984}
2985
2986static int dm_pr_preempt(struct block_device *bdev, u64 old_key, u64 new_key,
2987 enum pr_type type, bool abort)
2988{
2989 struct mapped_device *md = bdev->bd_disk->private_data;
2990 const struct pr_ops *ops;
2991 int r, srcu_idx;
2992
2993 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
2994 if (r < 0)
2995 goto out;
2996
2997 ops = bdev->bd_disk->fops->pr_ops;
2998 if (ops && ops->pr_preempt)
2999 r = ops->pr_preempt(bdev, old_key, new_key, type, abort);
3000 else
3001 r = -EOPNOTSUPP;
3002out:
3003 dm_unprepare_ioctl(md, srcu_idx);
3004 return r;
3005}
3006
3007static int dm_pr_clear(struct block_device *bdev, u64 key)
3008{
3009 struct mapped_device *md = bdev->bd_disk->private_data;
3010 const struct pr_ops *ops;
3011 int r, srcu_idx;
3012
3013 r = dm_prepare_ioctl(md, &srcu_idx, &bdev);
3014 if (r < 0)
3015 goto out;
3016
3017 ops = bdev->bd_disk->fops->pr_ops;
3018 if (ops && ops->pr_clear)
3019 r = ops->pr_clear(bdev, key);
3020 else
3021 r = -EOPNOTSUPP;
3022out:
3023 dm_unprepare_ioctl(md, srcu_idx);
3024 return r;
3025}
3026
3027static const struct pr_ops dm_pr_ops = {
3028 .pr_register = dm_pr_register,
3029 .pr_reserve = dm_pr_reserve,
3030 .pr_release = dm_pr_release,
3031 .pr_preempt = dm_pr_preempt,
3032 .pr_clear = dm_pr_clear,
3033};
3034
3035static const struct block_device_operations dm_blk_dops = {
3036 .submit_bio = dm_submit_bio,
3037 .open = dm_blk_open,
3038 .release = dm_blk_close,
3039 .ioctl = dm_blk_ioctl,
3040 .getgeo = dm_blk_getgeo,
3041 .report_zones = dm_blk_report_zones,
3042 .pr_ops = &dm_pr_ops,
3043 .owner = THIS_MODULE
3044};
3045
3046static const struct block_device_operations dm_rq_blk_dops = {
3047 .open = dm_blk_open,
3048 .release = dm_blk_close,
3049 .ioctl = dm_blk_ioctl,
3050 .getgeo = dm_blk_getgeo,
3051 .pr_ops = &dm_pr_ops,
3052 .owner = THIS_MODULE
3053};
3054
3055static const struct dax_operations dm_dax_ops = {
3056 .direct_access = dm_dax_direct_access,
3057 .dax_supported = dm_dax_supported,
3058 .copy_from_iter = dm_dax_copy_from_iter,
3059 .copy_to_iter = dm_dax_copy_to_iter,
3060 .zero_page_range = dm_dax_zero_page_range,
3061};
3062
3063/*
3064 * module hooks
3065 */
3066module_init(dm_init);
3067module_exit(dm_exit);
3068
3069module_param(major, uint, 0);
3070MODULE_PARM_DESC(major, "The major number of the device mapper");
3071
3072module_param(reserved_bio_based_ios, uint, S_IRUGO | S_IWUSR);
3073MODULE_PARM_DESC(reserved_bio_based_ios, "Reserved IOs in bio-based mempools");
3074
3075module_param(dm_numa_node, int, S_IRUGO | S_IWUSR);
3076MODULE_PARM_DESC(dm_numa_node, "NUMA node for DM device memory allocations");
3077
3078module_param(swap_bios, int, S_IRUGO | S_IWUSR);
3079MODULE_PARM_DESC(swap_bios, "Maximum allowed inflight swap IOs");
3080
3081MODULE_DESCRIPTION(DM_NAME " driver");
3082MODULE_AUTHOR("Joe Thornber <dm-devel@redhat.com>");
3083MODULE_LICENSE("GPL");