Loading...
Note: File does not exist in v3.5.6.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2019 Arrikto, Inc. All Rights Reserved.
4 */
5
6#include <linux/mm.h>
7#include <linux/bio.h>
8#include <linux/err.h>
9#include <linux/hash.h>
10#include <linux/list.h>
11#include <linux/log2.h>
12#include <linux/init.h>
13#include <linux/slab.h>
14#include <linux/wait.h>
15#include <linux/dm-io.h>
16#include <linux/mutex.h>
17#include <linux/atomic.h>
18#include <linux/bitops.h>
19#include <linux/blkdev.h>
20#include <linux/kdev_t.h>
21#include <linux/kernel.h>
22#include <linux/module.h>
23#include <linux/jiffies.h>
24#include <linux/mempool.h>
25#include <linux/spinlock.h>
26#include <linux/blk_types.h>
27#include <linux/dm-kcopyd.h>
28#include <linux/workqueue.h>
29#include <linux/backing-dev.h>
30#include <linux/device-mapper.h>
31
32#include "dm.h"
33#include "dm-clone-metadata.h"
34
35#define DM_MSG_PREFIX "clone"
36
37/*
38 * Minimum and maximum allowed region sizes
39 */
40#define MIN_REGION_SIZE (1 << 3) /* 4KB */
41#define MAX_REGION_SIZE (1 << 21) /* 1GB */
42
43#define MIN_HYDRATIONS 256 /* Size of hydration mempool */
44#define DEFAULT_HYDRATION_THRESHOLD 1 /* 1 region */
45#define DEFAULT_HYDRATION_BATCH_SIZE 1 /* Hydrate in batches of 1 region */
46
47#define COMMIT_PERIOD HZ /* 1 sec */
48
49/*
50 * Hydration hash table size: 1 << HASH_TABLE_BITS
51 */
52#define HASH_TABLE_BITS 15
53
54DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(clone_hydration_throttle,
55 "A percentage of time allocated for hydrating regions");
56
57/* Slab cache for struct dm_clone_region_hydration */
58static struct kmem_cache *_hydration_cache;
59
60/* dm-clone metadata modes */
61enum clone_metadata_mode {
62 CM_WRITE, /* metadata may be changed */
63 CM_READ_ONLY, /* metadata may not be changed */
64 CM_FAIL, /* all metadata I/O fails */
65};
66
67struct hash_table_bucket;
68
69struct clone {
70 struct dm_target *ti;
71
72 struct dm_dev *metadata_dev;
73 struct dm_dev *dest_dev;
74 struct dm_dev *source_dev;
75
76 unsigned long nr_regions;
77 sector_t region_size;
78 unsigned int region_shift;
79
80 /*
81 * A metadata commit and the actions taken in case it fails should run
82 * as a single atomic step.
83 */
84 struct mutex commit_lock;
85
86 struct dm_clone_metadata *cmd;
87
88 /* Region hydration hash table */
89 struct hash_table_bucket *ht;
90
91 atomic_t ios_in_flight;
92
93 wait_queue_head_t hydration_stopped;
94
95 mempool_t hydration_pool;
96
97 unsigned long last_commit_jiffies;
98
99 /*
100 * We defer incoming WRITE bios for regions that are not hydrated,
101 * until after these regions have been hydrated.
102 *
103 * Also, we defer REQ_FUA and REQ_PREFLUSH bios, until after the
104 * metadata have been committed.
105 */
106 spinlock_t lock;
107 struct bio_list deferred_bios;
108 struct bio_list deferred_discard_bios;
109 struct bio_list deferred_flush_bios;
110 struct bio_list deferred_flush_completions;
111
112 /* Maximum number of regions being copied during background hydration. */
113 unsigned int hydration_threshold;
114
115 /* Number of regions to batch together during background hydration. */
116 unsigned int hydration_batch_size;
117
118 /* Which region to hydrate next */
119 unsigned long hydration_offset;
120
121 atomic_t hydrations_in_flight;
122
123 /*
124 * Save a copy of the table line rather than reconstructing it for the
125 * status.
126 */
127 unsigned int nr_ctr_args;
128 const char **ctr_args;
129
130 struct workqueue_struct *wq;
131 struct work_struct worker;
132 struct delayed_work waker;
133
134 struct dm_kcopyd_client *kcopyd_client;
135
136 enum clone_metadata_mode mode;
137 unsigned long flags;
138};
139
140/*
141 * dm-clone flags
142 */
143#define DM_CLONE_DISCARD_PASSDOWN 0
144#define DM_CLONE_HYDRATION_ENABLED 1
145#define DM_CLONE_HYDRATION_SUSPENDED 2
146
147/*---------------------------------------------------------------------------*/
148
149/*
150 * Metadata failure handling.
151 */
152static enum clone_metadata_mode get_clone_mode(struct clone *clone)
153{
154 return READ_ONCE(clone->mode);
155}
156
157static const char *clone_device_name(struct clone *clone)
158{
159 return dm_table_device_name(clone->ti->table);
160}
161
162static void __set_clone_mode(struct clone *clone, enum clone_metadata_mode new_mode)
163{
164 static const char * const descs[] = {
165 "read-write",
166 "read-only",
167 "fail"
168 };
169
170 enum clone_metadata_mode old_mode = get_clone_mode(clone);
171
172 /* Never move out of fail mode */
173 if (old_mode == CM_FAIL)
174 new_mode = CM_FAIL;
175
176 switch (new_mode) {
177 case CM_FAIL:
178 case CM_READ_ONLY:
179 dm_clone_metadata_set_read_only(clone->cmd);
180 break;
181
182 case CM_WRITE:
183 dm_clone_metadata_set_read_write(clone->cmd);
184 break;
185 }
186
187 WRITE_ONCE(clone->mode, new_mode);
188
189 if (new_mode != old_mode) {
190 dm_table_event(clone->ti->table);
191 DMINFO("%s: Switching to %s mode", clone_device_name(clone),
192 descs[(int)new_mode]);
193 }
194}
195
196static void __abort_transaction(struct clone *clone)
197{
198 const char *dev_name = clone_device_name(clone);
199
200 if (get_clone_mode(clone) >= CM_READ_ONLY)
201 return;
202
203 DMERR("%s: Aborting current metadata transaction", dev_name);
204 if (dm_clone_metadata_abort(clone->cmd)) {
205 DMERR("%s: Failed to abort metadata transaction", dev_name);
206 __set_clone_mode(clone, CM_FAIL);
207 }
208}
209
210static void __reload_in_core_bitset(struct clone *clone)
211{
212 const char *dev_name = clone_device_name(clone);
213
214 if (get_clone_mode(clone) == CM_FAIL)
215 return;
216
217 /* Reload the on-disk bitset */
218 DMINFO("%s: Reloading on-disk bitmap", dev_name);
219 if (dm_clone_reload_in_core_bitset(clone->cmd)) {
220 DMERR("%s: Failed to reload on-disk bitmap", dev_name);
221 __set_clone_mode(clone, CM_FAIL);
222 }
223}
224
225static void __metadata_operation_failed(struct clone *clone, const char *op, int r)
226{
227 DMERR("%s: Metadata operation `%s' failed: error = %d",
228 clone_device_name(clone), op, r);
229
230 __abort_transaction(clone);
231 __set_clone_mode(clone, CM_READ_ONLY);
232
233 /*
234 * dm_clone_reload_in_core_bitset() may run concurrently with either
235 * dm_clone_set_region_hydrated() or dm_clone_cond_set_range(), but
236 * it's safe as we have already set the metadata to read-only mode.
237 */
238 __reload_in_core_bitset(clone);
239}
240
241/*---------------------------------------------------------------------------*/
242
243/* Wake up anyone waiting for region hydrations to stop */
244static inline void wakeup_hydration_waiters(struct clone *clone)
245{
246 wake_up_all(&clone->hydration_stopped);
247}
248
249static inline void wake_worker(struct clone *clone)
250{
251 queue_work(clone->wq, &clone->worker);
252}
253
254/*---------------------------------------------------------------------------*/
255
256/*
257 * bio helper functions.
258 */
259static inline void remap_to_source(struct clone *clone, struct bio *bio)
260{
261 bio_set_dev(bio, clone->source_dev->bdev);
262}
263
264static inline void remap_to_dest(struct clone *clone, struct bio *bio)
265{
266 bio_set_dev(bio, clone->dest_dev->bdev);
267}
268
269static bool bio_triggers_commit(struct clone *clone, struct bio *bio)
270{
271 return op_is_flush(bio->bi_opf) &&
272 dm_clone_changed_this_transaction(clone->cmd);
273}
274
275/* Get the address of the region in sectors */
276static inline sector_t region_to_sector(struct clone *clone, unsigned long region_nr)
277{
278 return ((sector_t)region_nr << clone->region_shift);
279}
280
281/* Get the region number of the bio */
282static inline unsigned long bio_to_region(struct clone *clone, struct bio *bio)
283{
284 return (bio->bi_iter.bi_sector >> clone->region_shift);
285}
286
287/* Get the region range covered by the bio */
288static void bio_region_range(struct clone *clone, struct bio *bio,
289 unsigned long *rs, unsigned long *nr_regions)
290{
291 unsigned long end;
292
293 *rs = dm_sector_div_up(bio->bi_iter.bi_sector, clone->region_size);
294 end = bio_end_sector(bio) >> clone->region_shift;
295
296 if (*rs >= end)
297 *nr_regions = 0;
298 else
299 *nr_regions = end - *rs;
300}
301
302/* Check whether a bio overwrites a region */
303static inline bool is_overwrite_bio(struct clone *clone, struct bio *bio)
304{
305 return (bio_data_dir(bio) == WRITE && bio_sectors(bio) == clone->region_size);
306}
307
308static void fail_bios(struct bio_list *bios, blk_status_t status)
309{
310 struct bio *bio;
311
312 while ((bio = bio_list_pop(bios))) {
313 bio->bi_status = status;
314 bio_endio(bio);
315 }
316}
317
318static void submit_bios(struct bio_list *bios)
319{
320 struct bio *bio;
321 struct blk_plug plug;
322
323 blk_start_plug(&plug);
324
325 while ((bio = bio_list_pop(bios)))
326 submit_bio_noacct(bio);
327
328 blk_finish_plug(&plug);
329}
330
331/*
332 * Submit bio to the underlying device.
333 *
334 * If the bio triggers a commit, delay it, until after the metadata have been
335 * committed.
336 *
337 * NOTE: The bio remapping must be performed by the caller.
338 */
339static void issue_bio(struct clone *clone, struct bio *bio)
340{
341 if (!bio_triggers_commit(clone, bio)) {
342 submit_bio_noacct(bio);
343 return;
344 }
345
346 /*
347 * If the metadata mode is RO or FAIL we won't be able to commit the
348 * metadata, so we complete the bio with an error.
349 */
350 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
351 bio_io_error(bio);
352 return;
353 }
354
355 /*
356 * Batch together any bios that trigger commits and then issue a single
357 * commit for them in process_deferred_flush_bios().
358 */
359 spin_lock_irq(&clone->lock);
360 bio_list_add(&clone->deferred_flush_bios, bio);
361 spin_unlock_irq(&clone->lock);
362
363 wake_worker(clone);
364}
365
366/*
367 * Remap bio to the destination device and submit it.
368 *
369 * If the bio triggers a commit, delay it, until after the metadata have been
370 * committed.
371 */
372static void remap_and_issue(struct clone *clone, struct bio *bio)
373{
374 remap_to_dest(clone, bio);
375 issue_bio(clone, bio);
376}
377
378/*
379 * Issue bios that have been deferred until after their region has finished
380 * hydrating.
381 *
382 * We delegate the bio submission to the worker thread, so this is safe to call
383 * from interrupt context.
384 */
385static void issue_deferred_bios(struct clone *clone, struct bio_list *bios)
386{
387 struct bio *bio;
388 unsigned long flags;
389 struct bio_list flush_bios = BIO_EMPTY_LIST;
390 struct bio_list normal_bios = BIO_EMPTY_LIST;
391
392 if (bio_list_empty(bios))
393 return;
394
395 while ((bio = bio_list_pop(bios))) {
396 if (bio_triggers_commit(clone, bio))
397 bio_list_add(&flush_bios, bio);
398 else
399 bio_list_add(&normal_bios, bio);
400 }
401
402 spin_lock_irqsave(&clone->lock, flags);
403 bio_list_merge(&clone->deferred_bios, &normal_bios);
404 bio_list_merge(&clone->deferred_flush_bios, &flush_bios);
405 spin_unlock_irqrestore(&clone->lock, flags);
406
407 wake_worker(clone);
408}
409
410static void complete_overwrite_bio(struct clone *clone, struct bio *bio)
411{
412 unsigned long flags;
413
414 /*
415 * If the bio has the REQ_FUA flag set we must commit the metadata
416 * before signaling its completion.
417 *
418 * complete_overwrite_bio() is only called by hydration_complete(),
419 * after having successfully updated the metadata. This means we don't
420 * need to call dm_clone_changed_this_transaction() to check if the
421 * metadata has changed and thus we can avoid taking the metadata spin
422 * lock.
423 */
424 if (!(bio->bi_opf & REQ_FUA)) {
425 bio_endio(bio);
426 return;
427 }
428
429 /*
430 * If the metadata mode is RO or FAIL we won't be able to commit the
431 * metadata, so we complete the bio with an error.
432 */
433 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
434 bio_io_error(bio);
435 return;
436 }
437
438 /*
439 * Batch together any bios that trigger commits and then issue a single
440 * commit for them in process_deferred_flush_bios().
441 */
442 spin_lock_irqsave(&clone->lock, flags);
443 bio_list_add(&clone->deferred_flush_completions, bio);
444 spin_unlock_irqrestore(&clone->lock, flags);
445
446 wake_worker(clone);
447}
448
449static void trim_bio(struct bio *bio, sector_t sector, unsigned int len)
450{
451 bio->bi_iter.bi_sector = sector;
452 bio->bi_iter.bi_size = to_bytes(len);
453}
454
455static void complete_discard_bio(struct clone *clone, struct bio *bio, bool success)
456{
457 unsigned long rs, nr_regions;
458
459 /*
460 * If the destination device supports discards, remap and trim the
461 * discard bio and pass it down. Otherwise complete the bio
462 * immediately.
463 */
464 if (test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags) && success) {
465 remap_to_dest(clone, bio);
466 bio_region_range(clone, bio, &rs, &nr_regions);
467 trim_bio(bio, region_to_sector(clone, rs),
468 nr_regions << clone->region_shift);
469 submit_bio_noacct(bio);
470 } else
471 bio_endio(bio);
472}
473
474static void process_discard_bio(struct clone *clone, struct bio *bio)
475{
476 unsigned long rs, nr_regions;
477
478 bio_region_range(clone, bio, &rs, &nr_regions);
479 if (!nr_regions) {
480 bio_endio(bio);
481 return;
482 }
483
484 if (WARN_ON(rs >= clone->nr_regions || (rs + nr_regions) < rs ||
485 (rs + nr_regions) > clone->nr_regions)) {
486 DMERR("%s: Invalid range (%lu + %lu, total regions %lu) for discard (%llu + %u)",
487 clone_device_name(clone), rs, nr_regions,
488 clone->nr_regions,
489 (unsigned long long)bio->bi_iter.bi_sector,
490 bio_sectors(bio));
491 bio_endio(bio);
492 return;
493 }
494
495 /*
496 * The covered regions are already hydrated so we just need to pass
497 * down the discard.
498 */
499 if (dm_clone_is_range_hydrated(clone->cmd, rs, nr_regions)) {
500 complete_discard_bio(clone, bio, true);
501 return;
502 }
503
504 /*
505 * If the metadata mode is RO or FAIL we won't be able to update the
506 * metadata for the regions covered by the discard so we just ignore
507 * it.
508 */
509 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
510 bio_endio(bio);
511 return;
512 }
513
514 /*
515 * Defer discard processing.
516 */
517 spin_lock_irq(&clone->lock);
518 bio_list_add(&clone->deferred_discard_bios, bio);
519 spin_unlock_irq(&clone->lock);
520
521 wake_worker(clone);
522}
523
524/*---------------------------------------------------------------------------*/
525
526/*
527 * dm-clone region hydrations.
528 */
529struct dm_clone_region_hydration {
530 struct clone *clone;
531 unsigned long region_nr;
532
533 struct bio *overwrite_bio;
534 bio_end_io_t *overwrite_bio_end_io;
535
536 struct bio_list deferred_bios;
537
538 blk_status_t status;
539
540 /* Used by hydration batching */
541 struct list_head list;
542
543 /* Used by hydration hash table */
544 struct hlist_node h;
545};
546
547/*
548 * Hydration hash table implementation.
549 *
550 * Ideally we would like to use list_bl, which uses bit spin locks and employs
551 * the least significant bit of the list head to lock the corresponding bucket,
552 * reducing the memory overhead for the locks. But, currently, list_bl and bit
553 * spin locks don't support IRQ safe versions. Since we have to take the lock
554 * in both process and interrupt context, we must fall back to using regular
555 * spin locks; one per hash table bucket.
556 */
557struct hash_table_bucket {
558 struct hlist_head head;
559
560 /* Spinlock protecting the bucket */
561 spinlock_t lock;
562};
563
564#define bucket_lock_irqsave(bucket, flags) \
565 spin_lock_irqsave(&(bucket)->lock, flags)
566
567#define bucket_unlock_irqrestore(bucket, flags) \
568 spin_unlock_irqrestore(&(bucket)->lock, flags)
569
570#define bucket_lock_irq(bucket) \
571 spin_lock_irq(&(bucket)->lock)
572
573#define bucket_unlock_irq(bucket) \
574 spin_unlock_irq(&(bucket)->lock)
575
576static int hash_table_init(struct clone *clone)
577{
578 unsigned int i, sz;
579 struct hash_table_bucket *bucket;
580
581 sz = 1 << HASH_TABLE_BITS;
582
583 clone->ht = kvmalloc(sz * sizeof(struct hash_table_bucket), GFP_KERNEL);
584 if (!clone->ht)
585 return -ENOMEM;
586
587 for (i = 0; i < sz; i++) {
588 bucket = clone->ht + i;
589
590 INIT_HLIST_HEAD(&bucket->head);
591 spin_lock_init(&bucket->lock);
592 }
593
594 return 0;
595}
596
597static void hash_table_exit(struct clone *clone)
598{
599 kvfree(clone->ht);
600}
601
602static struct hash_table_bucket *get_hash_table_bucket(struct clone *clone,
603 unsigned long region_nr)
604{
605 return &clone->ht[hash_long(region_nr, HASH_TABLE_BITS)];
606}
607
608/*
609 * Search hash table for a hydration with hd->region_nr == region_nr
610 *
611 * NOTE: Must be called with the bucket lock held
612 */
613static struct dm_clone_region_hydration *__hash_find(struct hash_table_bucket *bucket,
614 unsigned long region_nr)
615{
616 struct dm_clone_region_hydration *hd;
617
618 hlist_for_each_entry(hd, &bucket->head, h) {
619 if (hd->region_nr == region_nr)
620 return hd;
621 }
622
623 return NULL;
624}
625
626/*
627 * Insert a hydration into the hash table.
628 *
629 * NOTE: Must be called with the bucket lock held.
630 */
631static inline void __insert_region_hydration(struct hash_table_bucket *bucket,
632 struct dm_clone_region_hydration *hd)
633{
634 hlist_add_head(&hd->h, &bucket->head);
635}
636
637/*
638 * This function inserts a hydration into the hash table, unless someone else
639 * managed to insert a hydration for the same region first. In the latter case
640 * it returns the existing hydration descriptor for this region.
641 *
642 * NOTE: Must be called with the hydration hash table lock held.
643 */
644static struct dm_clone_region_hydration *
645__find_or_insert_region_hydration(struct hash_table_bucket *bucket,
646 struct dm_clone_region_hydration *hd)
647{
648 struct dm_clone_region_hydration *hd2;
649
650 hd2 = __hash_find(bucket, hd->region_nr);
651 if (hd2)
652 return hd2;
653
654 __insert_region_hydration(bucket, hd);
655
656 return hd;
657}
658
659/*---------------------------------------------------------------------------*/
660
661/* Allocate a hydration */
662static struct dm_clone_region_hydration *alloc_hydration(struct clone *clone)
663{
664 struct dm_clone_region_hydration *hd;
665
666 /*
667 * Allocate a hydration from the hydration mempool.
668 * This might block but it can't fail.
669 */
670 hd = mempool_alloc(&clone->hydration_pool, GFP_NOIO);
671 hd->clone = clone;
672
673 return hd;
674}
675
676static inline void free_hydration(struct dm_clone_region_hydration *hd)
677{
678 mempool_free(hd, &hd->clone->hydration_pool);
679}
680
681/* Initialize a hydration */
682static void hydration_init(struct dm_clone_region_hydration *hd, unsigned long region_nr)
683{
684 hd->region_nr = region_nr;
685 hd->overwrite_bio = NULL;
686 bio_list_init(&hd->deferred_bios);
687 hd->status = 0;
688
689 INIT_LIST_HEAD(&hd->list);
690 INIT_HLIST_NODE(&hd->h);
691}
692
693/*---------------------------------------------------------------------------*/
694
695/*
696 * Update dm-clone's metadata after a region has finished hydrating and remove
697 * hydration from the hash table.
698 */
699static int hydration_update_metadata(struct dm_clone_region_hydration *hd)
700{
701 int r = 0;
702 unsigned long flags;
703 struct hash_table_bucket *bucket;
704 struct clone *clone = hd->clone;
705
706 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
707 r = -EPERM;
708
709 /* Update the metadata */
710 if (likely(!r) && hd->status == BLK_STS_OK)
711 r = dm_clone_set_region_hydrated(clone->cmd, hd->region_nr);
712
713 bucket = get_hash_table_bucket(clone, hd->region_nr);
714
715 /* Remove hydration from hash table */
716 bucket_lock_irqsave(bucket, flags);
717 hlist_del(&hd->h);
718 bucket_unlock_irqrestore(bucket, flags);
719
720 return r;
721}
722
723/*
724 * Complete a region's hydration:
725 *
726 * 1. Update dm-clone's metadata.
727 * 2. Remove hydration from hash table.
728 * 3. Complete overwrite bio.
729 * 4. Issue deferred bios.
730 * 5. If this was the last hydration, wake up anyone waiting for
731 * hydrations to finish.
732 */
733static void hydration_complete(struct dm_clone_region_hydration *hd)
734{
735 int r;
736 blk_status_t status;
737 struct clone *clone = hd->clone;
738
739 r = hydration_update_metadata(hd);
740
741 if (hd->status == BLK_STS_OK && likely(!r)) {
742 if (hd->overwrite_bio)
743 complete_overwrite_bio(clone, hd->overwrite_bio);
744
745 issue_deferred_bios(clone, &hd->deferred_bios);
746 } else {
747 status = r ? BLK_STS_IOERR : hd->status;
748
749 if (hd->overwrite_bio)
750 bio_list_add(&hd->deferred_bios, hd->overwrite_bio);
751
752 fail_bios(&hd->deferred_bios, status);
753 }
754
755 free_hydration(hd);
756
757 if (atomic_dec_and_test(&clone->hydrations_in_flight))
758 wakeup_hydration_waiters(clone);
759}
760
761static void hydration_kcopyd_callback(int read_err, unsigned long write_err, void *context)
762{
763 blk_status_t status;
764
765 struct dm_clone_region_hydration *tmp, *hd = context;
766 struct clone *clone = hd->clone;
767
768 LIST_HEAD(batched_hydrations);
769
770 if (read_err || write_err) {
771 DMERR_LIMIT("%s: hydration failed", clone_device_name(clone));
772 status = BLK_STS_IOERR;
773 } else {
774 status = BLK_STS_OK;
775 }
776 list_splice_tail(&hd->list, &batched_hydrations);
777
778 hd->status = status;
779 hydration_complete(hd);
780
781 /* Complete batched hydrations */
782 list_for_each_entry_safe(hd, tmp, &batched_hydrations, list) {
783 hd->status = status;
784 hydration_complete(hd);
785 }
786
787 /* Continue background hydration, if there is no I/O in-flight */
788 if (test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) &&
789 !atomic_read(&clone->ios_in_flight))
790 wake_worker(clone);
791}
792
793static void hydration_copy(struct dm_clone_region_hydration *hd, unsigned int nr_regions)
794{
795 unsigned long region_start, region_end;
796 sector_t tail_size, region_size, total_size;
797 struct dm_io_region from, to;
798 struct clone *clone = hd->clone;
799
800 if (WARN_ON(!nr_regions))
801 return;
802
803 region_size = clone->region_size;
804 region_start = hd->region_nr;
805 region_end = region_start + nr_regions - 1;
806
807 total_size = region_to_sector(clone, nr_regions - 1);
808
809 if (region_end == clone->nr_regions - 1) {
810 /*
811 * The last region of the target might be smaller than
812 * region_size.
813 */
814 tail_size = clone->ti->len & (region_size - 1);
815 if (!tail_size)
816 tail_size = region_size;
817 } else {
818 tail_size = region_size;
819 }
820
821 total_size += tail_size;
822
823 from.bdev = clone->source_dev->bdev;
824 from.sector = region_to_sector(clone, region_start);
825 from.count = total_size;
826
827 to.bdev = clone->dest_dev->bdev;
828 to.sector = from.sector;
829 to.count = from.count;
830
831 /* Issue copy */
832 atomic_add(nr_regions, &clone->hydrations_in_flight);
833 dm_kcopyd_copy(clone->kcopyd_client, &from, 1, &to, 0,
834 hydration_kcopyd_callback, hd);
835}
836
837static void overwrite_endio(struct bio *bio)
838{
839 struct dm_clone_region_hydration *hd = bio->bi_private;
840
841 bio->bi_end_io = hd->overwrite_bio_end_io;
842 hd->status = bio->bi_status;
843
844 hydration_complete(hd);
845}
846
847static void hydration_overwrite(struct dm_clone_region_hydration *hd, struct bio *bio)
848{
849 /*
850 * We don't need to save and restore bio->bi_private because device
851 * mapper core generates a new bio for us to use, with clean
852 * bi_private.
853 */
854 hd->overwrite_bio = bio;
855 hd->overwrite_bio_end_io = bio->bi_end_io;
856
857 bio->bi_end_io = overwrite_endio;
858 bio->bi_private = hd;
859
860 atomic_inc(&hd->clone->hydrations_in_flight);
861 submit_bio_noacct(bio);
862}
863
864/*
865 * Hydrate bio's region.
866 *
867 * This function starts the hydration of the bio's region and puts the bio in
868 * the list of deferred bios for this region. In case, by the time this
869 * function is called, the region has finished hydrating it's submitted to the
870 * destination device.
871 *
872 * NOTE: The bio remapping must be performed by the caller.
873 */
874static void hydrate_bio_region(struct clone *clone, struct bio *bio)
875{
876 unsigned long region_nr;
877 struct hash_table_bucket *bucket;
878 struct dm_clone_region_hydration *hd, *hd2;
879
880 region_nr = bio_to_region(clone, bio);
881 bucket = get_hash_table_bucket(clone, region_nr);
882
883 bucket_lock_irq(bucket);
884
885 hd = __hash_find(bucket, region_nr);
886 if (hd) {
887 /* Someone else is hydrating the region */
888 bio_list_add(&hd->deferred_bios, bio);
889 bucket_unlock_irq(bucket);
890 return;
891 }
892
893 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
894 /* The region has been hydrated */
895 bucket_unlock_irq(bucket);
896 issue_bio(clone, bio);
897 return;
898 }
899
900 /*
901 * We must allocate a hydration descriptor and start the hydration of
902 * the corresponding region.
903 */
904 bucket_unlock_irq(bucket);
905
906 hd = alloc_hydration(clone);
907 hydration_init(hd, region_nr);
908
909 bucket_lock_irq(bucket);
910
911 /* Check if the region has been hydrated in the meantime. */
912 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
913 bucket_unlock_irq(bucket);
914 free_hydration(hd);
915 issue_bio(clone, bio);
916 return;
917 }
918
919 hd2 = __find_or_insert_region_hydration(bucket, hd);
920 if (hd2 != hd) {
921 /* Someone else started the region's hydration. */
922 bio_list_add(&hd2->deferred_bios, bio);
923 bucket_unlock_irq(bucket);
924 free_hydration(hd);
925 return;
926 }
927
928 /*
929 * If the metadata mode is RO or FAIL then there is no point starting a
930 * hydration, since we will not be able to update the metadata when the
931 * hydration finishes.
932 */
933 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
934 hlist_del(&hd->h);
935 bucket_unlock_irq(bucket);
936 free_hydration(hd);
937 bio_io_error(bio);
938 return;
939 }
940
941 /*
942 * Start region hydration.
943 *
944 * If a bio overwrites a region, i.e., its size is equal to the
945 * region's size, then we don't need to copy the region from the source
946 * to the destination device.
947 */
948 if (is_overwrite_bio(clone, bio)) {
949 bucket_unlock_irq(bucket);
950 hydration_overwrite(hd, bio);
951 } else {
952 bio_list_add(&hd->deferred_bios, bio);
953 bucket_unlock_irq(bucket);
954 hydration_copy(hd, 1);
955 }
956}
957
958/*---------------------------------------------------------------------------*/
959
960/*
961 * Background hydrations.
962 */
963
964/*
965 * Batch region hydrations.
966 *
967 * To better utilize device bandwidth we batch together the hydration of
968 * adjacent regions. This allows us to use small region sizes, e.g., 4KB, which
969 * is good for small, random write performance (because of the overwriting of
970 * un-hydrated regions) and at the same time issue big copy requests to kcopyd
971 * to achieve high hydration bandwidth.
972 */
973struct batch_info {
974 struct dm_clone_region_hydration *head;
975 unsigned int nr_batched_regions;
976};
977
978static void __batch_hydration(struct batch_info *batch,
979 struct dm_clone_region_hydration *hd)
980{
981 struct clone *clone = hd->clone;
982 unsigned int max_batch_size = READ_ONCE(clone->hydration_batch_size);
983
984 if (batch->head) {
985 /* Try to extend the current batch */
986 if (batch->nr_batched_regions < max_batch_size &&
987 (batch->head->region_nr + batch->nr_batched_regions) == hd->region_nr) {
988 list_add_tail(&hd->list, &batch->head->list);
989 batch->nr_batched_regions++;
990 hd = NULL;
991 }
992
993 /* Check if we should issue the current batch */
994 if (batch->nr_batched_regions >= max_batch_size || hd) {
995 hydration_copy(batch->head, batch->nr_batched_regions);
996 batch->head = NULL;
997 batch->nr_batched_regions = 0;
998 }
999 }
1000
1001 if (!hd)
1002 return;
1003
1004 /* We treat max batch sizes of zero and one equivalently */
1005 if (max_batch_size <= 1) {
1006 hydration_copy(hd, 1);
1007 return;
1008 }
1009
1010 /* Start a new batch */
1011 BUG_ON(!list_empty(&hd->list));
1012 batch->head = hd;
1013 batch->nr_batched_regions = 1;
1014}
1015
1016static unsigned long __start_next_hydration(struct clone *clone,
1017 unsigned long offset,
1018 struct batch_info *batch)
1019{
1020 struct hash_table_bucket *bucket;
1021 struct dm_clone_region_hydration *hd;
1022 unsigned long nr_regions = clone->nr_regions;
1023
1024 hd = alloc_hydration(clone);
1025
1026 /* Try to find a region to hydrate. */
1027 do {
1028 offset = dm_clone_find_next_unhydrated_region(clone->cmd, offset);
1029 if (offset == nr_regions)
1030 break;
1031
1032 bucket = get_hash_table_bucket(clone, offset);
1033 bucket_lock_irq(bucket);
1034
1035 if (!dm_clone_is_region_hydrated(clone->cmd, offset) &&
1036 !__hash_find(bucket, offset)) {
1037 hydration_init(hd, offset);
1038 __insert_region_hydration(bucket, hd);
1039 bucket_unlock_irq(bucket);
1040
1041 /* Batch hydration */
1042 __batch_hydration(batch, hd);
1043
1044 return (offset + 1);
1045 }
1046
1047 bucket_unlock_irq(bucket);
1048
1049 } while (++offset < nr_regions);
1050
1051 if (hd)
1052 free_hydration(hd);
1053
1054 return offset;
1055}
1056
1057/*
1058 * This function searches for regions that still reside in the source device
1059 * and starts their hydration.
1060 */
1061static void do_hydration(struct clone *clone)
1062{
1063 unsigned int current_volume;
1064 unsigned long offset, nr_regions = clone->nr_regions;
1065
1066 struct batch_info batch = {
1067 .head = NULL,
1068 .nr_batched_regions = 0,
1069 };
1070
1071 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
1072 return;
1073
1074 if (dm_clone_is_hydration_done(clone->cmd))
1075 return;
1076
1077 /*
1078 * Avoid race with device suspension.
1079 */
1080 atomic_inc(&clone->hydrations_in_flight);
1081
1082 /*
1083 * Make sure atomic_inc() is ordered before test_bit(), otherwise we
1084 * might race with clone_postsuspend() and start a region hydration
1085 * after the target has been suspended.
1086 *
1087 * This is paired with the smp_mb__after_atomic() in
1088 * clone_postsuspend().
1089 */
1090 smp_mb__after_atomic();
1091
1092 offset = clone->hydration_offset;
1093 while (likely(!test_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags)) &&
1094 !atomic_read(&clone->ios_in_flight) &&
1095 test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) &&
1096 offset < nr_regions) {
1097 current_volume = atomic_read(&clone->hydrations_in_flight);
1098 current_volume += batch.nr_batched_regions;
1099
1100 if (current_volume > READ_ONCE(clone->hydration_threshold))
1101 break;
1102
1103 offset = __start_next_hydration(clone, offset, &batch);
1104 }
1105
1106 if (batch.head)
1107 hydration_copy(batch.head, batch.nr_batched_regions);
1108
1109 if (offset >= nr_regions)
1110 offset = 0;
1111
1112 clone->hydration_offset = offset;
1113
1114 if (atomic_dec_and_test(&clone->hydrations_in_flight))
1115 wakeup_hydration_waiters(clone);
1116}
1117
1118/*---------------------------------------------------------------------------*/
1119
1120static bool need_commit_due_to_time(struct clone *clone)
1121{
1122 return !time_in_range(jiffies, clone->last_commit_jiffies,
1123 clone->last_commit_jiffies + COMMIT_PERIOD);
1124}
1125
1126/*
1127 * A non-zero return indicates read-only or fail mode.
1128 */
1129static int commit_metadata(struct clone *clone, bool *dest_dev_flushed)
1130{
1131 int r = 0;
1132
1133 if (dest_dev_flushed)
1134 *dest_dev_flushed = false;
1135
1136 mutex_lock(&clone->commit_lock);
1137
1138 if (!dm_clone_changed_this_transaction(clone->cmd))
1139 goto out;
1140
1141 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
1142 r = -EPERM;
1143 goto out;
1144 }
1145
1146 r = dm_clone_metadata_pre_commit(clone->cmd);
1147 if (unlikely(r)) {
1148 __metadata_operation_failed(clone, "dm_clone_metadata_pre_commit", r);
1149 goto out;
1150 }
1151
1152 r = blkdev_issue_flush(clone->dest_dev->bdev);
1153 if (unlikely(r)) {
1154 __metadata_operation_failed(clone, "flush destination device", r);
1155 goto out;
1156 }
1157
1158 if (dest_dev_flushed)
1159 *dest_dev_flushed = true;
1160
1161 r = dm_clone_metadata_commit(clone->cmd);
1162 if (unlikely(r)) {
1163 __metadata_operation_failed(clone, "dm_clone_metadata_commit", r);
1164 goto out;
1165 }
1166
1167 if (dm_clone_is_hydration_done(clone->cmd))
1168 dm_table_event(clone->ti->table);
1169out:
1170 mutex_unlock(&clone->commit_lock);
1171
1172 return r;
1173}
1174
1175static void process_deferred_discards(struct clone *clone)
1176{
1177 int r = -EPERM;
1178 struct bio *bio;
1179 struct blk_plug plug;
1180 unsigned long rs, nr_regions;
1181 struct bio_list discards = BIO_EMPTY_LIST;
1182
1183 spin_lock_irq(&clone->lock);
1184 bio_list_merge(&discards, &clone->deferred_discard_bios);
1185 bio_list_init(&clone->deferred_discard_bios);
1186 spin_unlock_irq(&clone->lock);
1187
1188 if (bio_list_empty(&discards))
1189 return;
1190
1191 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
1192 goto out;
1193
1194 /* Update the metadata */
1195 bio_list_for_each(bio, &discards) {
1196 bio_region_range(clone, bio, &rs, &nr_regions);
1197 /*
1198 * A discard request might cover regions that have been already
1199 * hydrated. There is no need to update the metadata for these
1200 * regions.
1201 */
1202 r = dm_clone_cond_set_range(clone->cmd, rs, nr_regions);
1203 if (unlikely(r))
1204 break;
1205 }
1206out:
1207 blk_start_plug(&plug);
1208 while ((bio = bio_list_pop(&discards)))
1209 complete_discard_bio(clone, bio, r == 0);
1210 blk_finish_plug(&plug);
1211}
1212
1213static void process_deferred_bios(struct clone *clone)
1214{
1215 struct bio_list bios = BIO_EMPTY_LIST;
1216
1217 spin_lock_irq(&clone->lock);
1218 bio_list_merge(&bios, &clone->deferred_bios);
1219 bio_list_init(&clone->deferred_bios);
1220 spin_unlock_irq(&clone->lock);
1221
1222 if (bio_list_empty(&bios))
1223 return;
1224
1225 submit_bios(&bios);
1226}
1227
1228static void process_deferred_flush_bios(struct clone *clone)
1229{
1230 struct bio *bio;
1231 bool dest_dev_flushed;
1232 struct bio_list bios = BIO_EMPTY_LIST;
1233 struct bio_list bio_completions = BIO_EMPTY_LIST;
1234
1235 /*
1236 * If there are any deferred flush bios, we must commit the metadata
1237 * before issuing them or signaling their completion.
1238 */
1239 spin_lock_irq(&clone->lock);
1240 bio_list_merge(&bios, &clone->deferred_flush_bios);
1241 bio_list_init(&clone->deferred_flush_bios);
1242
1243 bio_list_merge(&bio_completions, &clone->deferred_flush_completions);
1244 bio_list_init(&clone->deferred_flush_completions);
1245 spin_unlock_irq(&clone->lock);
1246
1247 if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) &&
1248 !(dm_clone_changed_this_transaction(clone->cmd) && need_commit_due_to_time(clone)))
1249 return;
1250
1251 if (commit_metadata(clone, &dest_dev_flushed)) {
1252 bio_list_merge(&bios, &bio_completions);
1253
1254 while ((bio = bio_list_pop(&bios)))
1255 bio_io_error(bio);
1256
1257 return;
1258 }
1259
1260 clone->last_commit_jiffies = jiffies;
1261
1262 while ((bio = bio_list_pop(&bio_completions)))
1263 bio_endio(bio);
1264
1265 while ((bio = bio_list_pop(&bios))) {
1266 if ((bio->bi_opf & REQ_PREFLUSH) && dest_dev_flushed) {
1267 /* We just flushed the destination device as part of
1268 * the metadata commit, so there is no reason to send
1269 * another flush.
1270 */
1271 bio_endio(bio);
1272 } else {
1273 submit_bio_noacct(bio);
1274 }
1275 }
1276}
1277
1278static void do_worker(struct work_struct *work)
1279{
1280 struct clone *clone = container_of(work, typeof(*clone), worker);
1281
1282 process_deferred_bios(clone);
1283 process_deferred_discards(clone);
1284
1285 /*
1286 * process_deferred_flush_bios():
1287 *
1288 * - Commit metadata
1289 *
1290 * - Process deferred REQ_FUA completions
1291 *
1292 * - Process deferred REQ_PREFLUSH bios
1293 */
1294 process_deferred_flush_bios(clone);
1295
1296 /* Background hydration */
1297 do_hydration(clone);
1298}
1299
1300/*
1301 * Commit periodically so that not too much unwritten data builds up.
1302 *
1303 * Also, restart background hydration, if it has been stopped by in-flight I/O.
1304 */
1305static void do_waker(struct work_struct *work)
1306{
1307 struct clone *clone = container_of(to_delayed_work(work), struct clone, waker);
1308
1309 wake_worker(clone);
1310 queue_delayed_work(clone->wq, &clone->waker, COMMIT_PERIOD);
1311}
1312
1313/*---------------------------------------------------------------------------*/
1314
1315/*
1316 * Target methods
1317 */
1318static int clone_map(struct dm_target *ti, struct bio *bio)
1319{
1320 struct clone *clone = ti->private;
1321 unsigned long region_nr;
1322
1323 atomic_inc(&clone->ios_in_flight);
1324
1325 if (unlikely(get_clone_mode(clone) == CM_FAIL))
1326 return DM_MAPIO_KILL;
1327
1328 /*
1329 * REQ_PREFLUSH bios carry no data:
1330 *
1331 * - Commit metadata, if changed
1332 *
1333 * - Pass down to destination device
1334 */
1335 if (bio->bi_opf & REQ_PREFLUSH) {
1336 remap_and_issue(clone, bio);
1337 return DM_MAPIO_SUBMITTED;
1338 }
1339
1340 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1341
1342 /*
1343 * dm-clone interprets discards and performs a fast hydration of the
1344 * discarded regions, i.e., we skip the copy from the source device and
1345 * just mark the regions as hydrated.
1346 */
1347 if (bio_op(bio) == REQ_OP_DISCARD) {
1348 process_discard_bio(clone, bio);
1349 return DM_MAPIO_SUBMITTED;
1350 }
1351
1352 /*
1353 * If the bio's region is hydrated, redirect it to the destination
1354 * device.
1355 *
1356 * If the region is not hydrated and the bio is a READ, redirect it to
1357 * the source device.
1358 *
1359 * Else, defer WRITE bio until after its region has been hydrated and
1360 * start the region's hydration immediately.
1361 */
1362 region_nr = bio_to_region(clone, bio);
1363 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
1364 remap_and_issue(clone, bio);
1365 return DM_MAPIO_SUBMITTED;
1366 } else if (bio_data_dir(bio) == READ) {
1367 remap_to_source(clone, bio);
1368 return DM_MAPIO_REMAPPED;
1369 }
1370
1371 remap_to_dest(clone, bio);
1372 hydrate_bio_region(clone, bio);
1373
1374 return DM_MAPIO_SUBMITTED;
1375}
1376
1377static int clone_endio(struct dm_target *ti, struct bio *bio, blk_status_t *error)
1378{
1379 struct clone *clone = ti->private;
1380
1381 atomic_dec(&clone->ios_in_flight);
1382
1383 return DM_ENDIO_DONE;
1384}
1385
1386static void emit_flags(struct clone *clone, char *result, unsigned int maxlen,
1387 ssize_t *sz_ptr)
1388{
1389 ssize_t sz = *sz_ptr;
1390 unsigned int count;
1391
1392 count = !test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
1393 count += !test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
1394
1395 DMEMIT("%u ", count);
1396
1397 if (!test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags))
1398 DMEMIT("no_hydration ");
1399
1400 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags))
1401 DMEMIT("no_discard_passdown ");
1402
1403 *sz_ptr = sz;
1404}
1405
1406static void emit_core_args(struct clone *clone, char *result,
1407 unsigned int maxlen, ssize_t *sz_ptr)
1408{
1409 ssize_t sz = *sz_ptr;
1410 unsigned int count = 4;
1411
1412 DMEMIT("%u hydration_threshold %u hydration_batch_size %u ", count,
1413 READ_ONCE(clone->hydration_threshold),
1414 READ_ONCE(clone->hydration_batch_size));
1415
1416 *sz_ptr = sz;
1417}
1418
1419/*
1420 * Status format:
1421 *
1422 * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
1423 * <clone region size> <#hydrated regions>/<#total regions> <#hydrating regions>
1424 * <#features> <features>* <#core args> <core args>* <clone metadata mode>
1425 */
1426static void clone_status(struct dm_target *ti, status_type_t type,
1427 unsigned int status_flags, char *result,
1428 unsigned int maxlen)
1429{
1430 int r;
1431 unsigned int i;
1432 ssize_t sz = 0;
1433 dm_block_t nr_free_metadata_blocks = 0;
1434 dm_block_t nr_metadata_blocks = 0;
1435 char buf[BDEVNAME_SIZE];
1436 struct clone *clone = ti->private;
1437
1438 switch (type) {
1439 case STATUSTYPE_INFO:
1440 if (get_clone_mode(clone) == CM_FAIL) {
1441 DMEMIT("Fail");
1442 break;
1443 }
1444
1445 /* Commit to ensure statistics aren't out-of-date */
1446 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
1447 (void) commit_metadata(clone, NULL);
1448
1449 r = dm_clone_get_free_metadata_block_count(clone->cmd, &nr_free_metadata_blocks);
1450
1451 if (r) {
1452 DMERR("%s: dm_clone_get_free_metadata_block_count returned %d",
1453 clone_device_name(clone), r);
1454 goto error;
1455 }
1456
1457 r = dm_clone_get_metadata_dev_size(clone->cmd, &nr_metadata_blocks);
1458
1459 if (r) {
1460 DMERR("%s: dm_clone_get_metadata_dev_size returned %d",
1461 clone_device_name(clone), r);
1462 goto error;
1463 }
1464
1465 DMEMIT("%u %llu/%llu %llu %u/%lu %u ",
1466 DM_CLONE_METADATA_BLOCK_SIZE,
1467 (unsigned long long)(nr_metadata_blocks - nr_free_metadata_blocks),
1468 (unsigned long long)nr_metadata_blocks,
1469 (unsigned long long)clone->region_size,
1470 dm_clone_nr_of_hydrated_regions(clone->cmd),
1471 clone->nr_regions,
1472 atomic_read(&clone->hydrations_in_flight));
1473
1474 emit_flags(clone, result, maxlen, &sz);
1475 emit_core_args(clone, result, maxlen, &sz);
1476
1477 switch (get_clone_mode(clone)) {
1478 case CM_WRITE:
1479 DMEMIT("rw");
1480 break;
1481 case CM_READ_ONLY:
1482 DMEMIT("ro");
1483 break;
1484 case CM_FAIL:
1485 DMEMIT("Fail");
1486 }
1487
1488 break;
1489
1490 case STATUSTYPE_TABLE:
1491 format_dev_t(buf, clone->metadata_dev->bdev->bd_dev);
1492 DMEMIT("%s ", buf);
1493
1494 format_dev_t(buf, clone->dest_dev->bdev->bd_dev);
1495 DMEMIT("%s ", buf);
1496
1497 format_dev_t(buf, clone->source_dev->bdev->bd_dev);
1498 DMEMIT("%s", buf);
1499
1500 for (i = 0; i < clone->nr_ctr_args; i++)
1501 DMEMIT(" %s", clone->ctr_args[i]);
1502 break;
1503
1504 case STATUSTYPE_IMA:
1505 *result = '\0';
1506 break;
1507 }
1508
1509 return;
1510
1511error:
1512 DMEMIT("Error");
1513}
1514
1515static sector_t get_dev_size(struct dm_dev *dev)
1516{
1517 return bdev_nr_sectors(dev->bdev);
1518}
1519
1520/*---------------------------------------------------------------------------*/
1521
1522/*
1523 * Construct a clone device mapping:
1524 *
1525 * clone <metadata dev> <destination dev> <source dev> <region size>
1526 * [<#feature args> [<feature arg>]* [<#core args> [key value]*]]
1527 *
1528 * metadata dev: Fast device holding the persistent metadata
1529 * destination dev: The destination device, which will become a clone of the
1530 * source device
1531 * source dev: The read-only source device that gets cloned
1532 * region size: dm-clone unit size in sectors
1533 *
1534 * #feature args: Number of feature arguments passed
1535 * feature args: E.g. no_hydration, no_discard_passdown
1536 *
1537 * #core arguments: An even number of core arguments
1538 * core arguments: Key/value pairs for tuning the core
1539 * E.g. 'hydration_threshold 256'
1540 */
1541static int parse_feature_args(struct dm_arg_set *as, struct clone *clone)
1542{
1543 int r;
1544 unsigned int argc;
1545 const char *arg_name;
1546 struct dm_target *ti = clone->ti;
1547
1548 const struct dm_arg args = {
1549 .min = 0,
1550 .max = 2,
1551 .error = "Invalid number of feature arguments"
1552 };
1553
1554 /* No feature arguments supplied */
1555 if (!as->argc)
1556 return 0;
1557
1558 r = dm_read_arg_group(&args, as, &argc, &ti->error);
1559 if (r)
1560 return r;
1561
1562 while (argc) {
1563 arg_name = dm_shift_arg(as);
1564 argc--;
1565
1566 if (!strcasecmp(arg_name, "no_hydration")) {
1567 __clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
1568 } else if (!strcasecmp(arg_name, "no_discard_passdown")) {
1569 __clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
1570 } else {
1571 ti->error = "Invalid feature argument";
1572 return -EINVAL;
1573 }
1574 }
1575
1576 return 0;
1577}
1578
1579static int parse_core_args(struct dm_arg_set *as, struct clone *clone)
1580{
1581 int r;
1582 unsigned int argc;
1583 unsigned int value;
1584 const char *arg_name;
1585 struct dm_target *ti = clone->ti;
1586
1587 const struct dm_arg args = {
1588 .min = 0,
1589 .max = 4,
1590 .error = "Invalid number of core arguments"
1591 };
1592
1593 /* Initialize core arguments */
1594 clone->hydration_batch_size = DEFAULT_HYDRATION_BATCH_SIZE;
1595 clone->hydration_threshold = DEFAULT_HYDRATION_THRESHOLD;
1596
1597 /* No core arguments supplied */
1598 if (!as->argc)
1599 return 0;
1600
1601 r = dm_read_arg_group(&args, as, &argc, &ti->error);
1602 if (r)
1603 return r;
1604
1605 if (argc & 1) {
1606 ti->error = "Number of core arguments must be even";
1607 return -EINVAL;
1608 }
1609
1610 while (argc) {
1611 arg_name = dm_shift_arg(as);
1612 argc -= 2;
1613
1614 if (!strcasecmp(arg_name, "hydration_threshold")) {
1615 if (kstrtouint(dm_shift_arg(as), 10, &value)) {
1616 ti->error = "Invalid value for argument `hydration_threshold'";
1617 return -EINVAL;
1618 }
1619 clone->hydration_threshold = value;
1620 } else if (!strcasecmp(arg_name, "hydration_batch_size")) {
1621 if (kstrtouint(dm_shift_arg(as), 10, &value)) {
1622 ti->error = "Invalid value for argument `hydration_batch_size'";
1623 return -EINVAL;
1624 }
1625 clone->hydration_batch_size = value;
1626 } else {
1627 ti->error = "Invalid core argument";
1628 return -EINVAL;
1629 }
1630 }
1631
1632 return 0;
1633}
1634
1635static int parse_region_size(struct clone *clone, struct dm_arg_set *as, char **error)
1636{
1637 int r;
1638 unsigned int region_size;
1639 struct dm_arg arg;
1640
1641 arg.min = MIN_REGION_SIZE;
1642 arg.max = MAX_REGION_SIZE;
1643 arg.error = "Invalid region size";
1644
1645 r = dm_read_arg(&arg, as, ®ion_size, error);
1646 if (r)
1647 return r;
1648
1649 /* Check region size is a power of 2 */
1650 if (!is_power_of_2(region_size)) {
1651 *error = "Region size is not a power of 2";
1652 return -EINVAL;
1653 }
1654
1655 /* Validate the region size against the device logical block size */
1656 if (region_size % (bdev_logical_block_size(clone->source_dev->bdev) >> 9) ||
1657 region_size % (bdev_logical_block_size(clone->dest_dev->bdev) >> 9)) {
1658 *error = "Region size is not a multiple of device logical block size";
1659 return -EINVAL;
1660 }
1661
1662 clone->region_size = region_size;
1663
1664 return 0;
1665}
1666
1667static int validate_nr_regions(unsigned long n, char **error)
1668{
1669 /*
1670 * dm_bitset restricts us to 2^32 regions. test_bit & co. restrict us
1671 * further to 2^31 regions.
1672 */
1673 if (n > (1UL << 31)) {
1674 *error = "Too many regions. Consider increasing the region size";
1675 return -EINVAL;
1676 }
1677
1678 return 0;
1679}
1680
1681static int parse_metadata_dev(struct clone *clone, struct dm_arg_set *as, char **error)
1682{
1683 int r;
1684 sector_t metadata_dev_size;
1685
1686 r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
1687 &clone->metadata_dev);
1688 if (r) {
1689 *error = "Error opening metadata device";
1690 return r;
1691 }
1692
1693 metadata_dev_size = get_dev_size(clone->metadata_dev);
1694 if (metadata_dev_size > DM_CLONE_METADATA_MAX_SECTORS_WARNING)
1695 DMWARN("Metadata device %pg is larger than %u sectors: excess space will not be used.",
1696 clone->metadata_dev->bdev, DM_CLONE_METADATA_MAX_SECTORS);
1697
1698 return 0;
1699}
1700
1701static int parse_dest_dev(struct clone *clone, struct dm_arg_set *as, char **error)
1702{
1703 int r;
1704 sector_t dest_dev_size;
1705
1706 r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
1707 &clone->dest_dev);
1708 if (r) {
1709 *error = "Error opening destination device";
1710 return r;
1711 }
1712
1713 dest_dev_size = get_dev_size(clone->dest_dev);
1714 if (dest_dev_size < clone->ti->len) {
1715 dm_put_device(clone->ti, clone->dest_dev);
1716 *error = "Device size larger than destination device";
1717 return -EINVAL;
1718 }
1719
1720 return 0;
1721}
1722
1723static int parse_source_dev(struct clone *clone, struct dm_arg_set *as, char **error)
1724{
1725 int r;
1726 sector_t source_dev_size;
1727
1728 r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ,
1729 &clone->source_dev);
1730 if (r) {
1731 *error = "Error opening source device";
1732 return r;
1733 }
1734
1735 source_dev_size = get_dev_size(clone->source_dev);
1736 if (source_dev_size < clone->ti->len) {
1737 dm_put_device(clone->ti, clone->source_dev);
1738 *error = "Device size larger than source device";
1739 return -EINVAL;
1740 }
1741
1742 return 0;
1743}
1744
1745static int copy_ctr_args(struct clone *clone, int argc, const char **argv, char **error)
1746{
1747 unsigned int i;
1748 const char **copy;
1749
1750 copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
1751 if (!copy)
1752 goto error;
1753
1754 for (i = 0; i < argc; i++) {
1755 copy[i] = kstrdup(argv[i], GFP_KERNEL);
1756
1757 if (!copy[i]) {
1758 while (i--)
1759 kfree(copy[i]);
1760 kfree(copy);
1761 goto error;
1762 }
1763 }
1764
1765 clone->nr_ctr_args = argc;
1766 clone->ctr_args = copy;
1767 return 0;
1768
1769error:
1770 *error = "Failed to allocate memory for table line";
1771 return -ENOMEM;
1772}
1773
1774static int clone_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1775{
1776 int r;
1777 sector_t nr_regions;
1778 struct clone *clone;
1779 struct dm_arg_set as;
1780
1781 if (argc < 4) {
1782 ti->error = "Invalid number of arguments";
1783 return -EINVAL;
1784 }
1785
1786 as.argc = argc;
1787 as.argv = argv;
1788
1789 clone = kzalloc(sizeof(*clone), GFP_KERNEL);
1790 if (!clone) {
1791 ti->error = "Failed to allocate clone structure";
1792 return -ENOMEM;
1793 }
1794
1795 clone->ti = ti;
1796
1797 /* Initialize dm-clone flags */
1798 __set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
1799 __set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
1800 __set_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
1801
1802 r = parse_metadata_dev(clone, &as, &ti->error);
1803 if (r)
1804 goto out_with_clone;
1805
1806 r = parse_dest_dev(clone, &as, &ti->error);
1807 if (r)
1808 goto out_with_meta_dev;
1809
1810 r = parse_source_dev(clone, &as, &ti->error);
1811 if (r)
1812 goto out_with_dest_dev;
1813
1814 r = parse_region_size(clone, &as, &ti->error);
1815 if (r)
1816 goto out_with_source_dev;
1817
1818 clone->region_shift = __ffs(clone->region_size);
1819 nr_regions = dm_sector_div_up(ti->len, clone->region_size);
1820
1821 /* Check for overflow */
1822 if (nr_regions != (unsigned long)nr_regions) {
1823 ti->error = "Too many regions. Consider increasing the region size";
1824 r = -EOVERFLOW;
1825 goto out_with_source_dev;
1826 }
1827
1828 clone->nr_regions = nr_regions;
1829
1830 r = validate_nr_regions(clone->nr_regions, &ti->error);
1831 if (r)
1832 goto out_with_source_dev;
1833
1834 r = dm_set_target_max_io_len(ti, clone->region_size);
1835 if (r) {
1836 ti->error = "Failed to set max io len";
1837 goto out_with_source_dev;
1838 }
1839
1840 r = parse_feature_args(&as, clone);
1841 if (r)
1842 goto out_with_source_dev;
1843
1844 r = parse_core_args(&as, clone);
1845 if (r)
1846 goto out_with_source_dev;
1847
1848 /* Load metadata */
1849 clone->cmd = dm_clone_metadata_open(clone->metadata_dev->bdev, ti->len,
1850 clone->region_size);
1851 if (IS_ERR(clone->cmd)) {
1852 ti->error = "Failed to load metadata";
1853 r = PTR_ERR(clone->cmd);
1854 goto out_with_source_dev;
1855 }
1856
1857 __set_clone_mode(clone, CM_WRITE);
1858
1859 if (get_clone_mode(clone) != CM_WRITE) {
1860 ti->error = "Unable to get write access to metadata, please check/repair metadata";
1861 r = -EPERM;
1862 goto out_with_metadata;
1863 }
1864
1865 clone->last_commit_jiffies = jiffies;
1866
1867 /* Allocate hydration hash table */
1868 r = hash_table_init(clone);
1869 if (r) {
1870 ti->error = "Failed to allocate hydration hash table";
1871 goto out_with_metadata;
1872 }
1873
1874 atomic_set(&clone->ios_in_flight, 0);
1875 init_waitqueue_head(&clone->hydration_stopped);
1876 spin_lock_init(&clone->lock);
1877 bio_list_init(&clone->deferred_bios);
1878 bio_list_init(&clone->deferred_discard_bios);
1879 bio_list_init(&clone->deferred_flush_bios);
1880 bio_list_init(&clone->deferred_flush_completions);
1881 clone->hydration_offset = 0;
1882 atomic_set(&clone->hydrations_in_flight, 0);
1883
1884 clone->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
1885 if (!clone->wq) {
1886 ti->error = "Failed to allocate workqueue";
1887 r = -ENOMEM;
1888 goto out_with_ht;
1889 }
1890
1891 INIT_WORK(&clone->worker, do_worker);
1892 INIT_DELAYED_WORK(&clone->waker, do_waker);
1893
1894 clone->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle);
1895 if (IS_ERR(clone->kcopyd_client)) {
1896 r = PTR_ERR(clone->kcopyd_client);
1897 goto out_with_wq;
1898 }
1899
1900 r = mempool_init_slab_pool(&clone->hydration_pool, MIN_HYDRATIONS,
1901 _hydration_cache);
1902 if (r) {
1903 ti->error = "Failed to create dm_clone_region_hydration memory pool";
1904 goto out_with_kcopyd;
1905 }
1906
1907 /* Save a copy of the table line */
1908 r = copy_ctr_args(clone, argc - 3, (const char **)argv + 3, &ti->error);
1909 if (r)
1910 goto out_with_mempool;
1911
1912 mutex_init(&clone->commit_lock);
1913
1914 /* Enable flushes */
1915 ti->num_flush_bios = 1;
1916 ti->flush_supported = true;
1917
1918 /* Enable discards */
1919 ti->discards_supported = true;
1920 ti->num_discard_bios = 1;
1921
1922 ti->private = clone;
1923
1924 return 0;
1925
1926out_with_mempool:
1927 mempool_exit(&clone->hydration_pool);
1928out_with_kcopyd:
1929 dm_kcopyd_client_destroy(clone->kcopyd_client);
1930out_with_wq:
1931 destroy_workqueue(clone->wq);
1932out_with_ht:
1933 hash_table_exit(clone);
1934out_with_metadata:
1935 dm_clone_metadata_close(clone->cmd);
1936out_with_source_dev:
1937 dm_put_device(ti, clone->source_dev);
1938out_with_dest_dev:
1939 dm_put_device(ti, clone->dest_dev);
1940out_with_meta_dev:
1941 dm_put_device(ti, clone->metadata_dev);
1942out_with_clone:
1943 kfree(clone);
1944
1945 return r;
1946}
1947
1948static void clone_dtr(struct dm_target *ti)
1949{
1950 unsigned int i;
1951 struct clone *clone = ti->private;
1952
1953 mutex_destroy(&clone->commit_lock);
1954
1955 for (i = 0; i < clone->nr_ctr_args; i++)
1956 kfree(clone->ctr_args[i]);
1957 kfree(clone->ctr_args);
1958
1959 mempool_exit(&clone->hydration_pool);
1960 dm_kcopyd_client_destroy(clone->kcopyd_client);
1961 cancel_delayed_work_sync(&clone->waker);
1962 destroy_workqueue(clone->wq);
1963 hash_table_exit(clone);
1964 dm_clone_metadata_close(clone->cmd);
1965 dm_put_device(ti, clone->source_dev);
1966 dm_put_device(ti, clone->dest_dev);
1967 dm_put_device(ti, clone->metadata_dev);
1968
1969 kfree(clone);
1970}
1971
1972/*---------------------------------------------------------------------------*/
1973
1974static void clone_postsuspend(struct dm_target *ti)
1975{
1976 struct clone *clone = ti->private;
1977
1978 /*
1979 * To successfully suspend the device:
1980 *
1981 * - We cancel the delayed work for periodic commits and wait for
1982 * it to finish.
1983 *
1984 * - We stop the background hydration, i.e. we prevent new region
1985 * hydrations from starting.
1986 *
1987 * - We wait for any in-flight hydrations to finish.
1988 *
1989 * - We flush the workqueue.
1990 *
1991 * - We commit the metadata.
1992 */
1993 cancel_delayed_work_sync(&clone->waker);
1994
1995 set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
1996
1997 /*
1998 * Make sure set_bit() is ordered before atomic_read(), otherwise we
1999 * might race with do_hydration() and miss some started region
2000 * hydrations.
2001 *
2002 * This is paired with smp_mb__after_atomic() in do_hydration().
2003 */
2004 smp_mb__after_atomic();
2005
2006 wait_event(clone->hydration_stopped, !atomic_read(&clone->hydrations_in_flight));
2007 flush_workqueue(clone->wq);
2008
2009 (void) commit_metadata(clone, NULL);
2010}
2011
2012static void clone_resume(struct dm_target *ti)
2013{
2014 struct clone *clone = ti->private;
2015
2016 clear_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
2017 do_waker(&clone->waker.work);
2018}
2019
2020/*
2021 * If discard_passdown was enabled verify that the destination device supports
2022 * discards. Disable discard_passdown if not.
2023 */
2024static void disable_passdown_if_not_supported(struct clone *clone)
2025{
2026 struct block_device *dest_dev = clone->dest_dev->bdev;
2027 struct queue_limits *dest_limits = &bdev_get_queue(dest_dev)->limits;
2028 const char *reason = NULL;
2029
2030 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags))
2031 return;
2032
2033 if (!bdev_max_discard_sectors(dest_dev))
2034 reason = "discard unsupported";
2035 else if (dest_limits->max_discard_sectors < clone->region_size)
2036 reason = "max discard sectors smaller than a region";
2037
2038 if (reason) {
2039 DMWARN("Destination device (%pg) %s: Disabling discard passdown.",
2040 dest_dev, reason);
2041 clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
2042 }
2043}
2044
2045static void set_discard_limits(struct clone *clone, struct queue_limits *limits)
2046{
2047 struct block_device *dest_bdev = clone->dest_dev->bdev;
2048 struct queue_limits *dest_limits = &bdev_get_queue(dest_bdev)->limits;
2049
2050 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags)) {
2051 /* No passdown is done so we set our own virtual limits */
2052 limits->discard_granularity = clone->region_size << SECTOR_SHIFT;
2053 limits->max_discard_sectors = round_down(UINT_MAX >> SECTOR_SHIFT, clone->region_size);
2054 return;
2055 }
2056
2057 /*
2058 * clone_iterate_devices() is stacking both the source and destination
2059 * device limits but discards aren't passed to the source device, so
2060 * inherit destination's limits.
2061 */
2062 limits->max_discard_sectors = dest_limits->max_discard_sectors;
2063 limits->max_hw_discard_sectors = dest_limits->max_hw_discard_sectors;
2064 limits->discard_granularity = dest_limits->discard_granularity;
2065 limits->discard_alignment = dest_limits->discard_alignment;
2066 limits->discard_misaligned = dest_limits->discard_misaligned;
2067 limits->max_discard_segments = dest_limits->max_discard_segments;
2068}
2069
2070static void clone_io_hints(struct dm_target *ti, struct queue_limits *limits)
2071{
2072 struct clone *clone = ti->private;
2073 u64 io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
2074
2075 /*
2076 * If the system-determined stacked limits are compatible with
2077 * dm-clone's region size (io_opt is a factor) do not override them.
2078 */
2079 if (io_opt_sectors < clone->region_size ||
2080 do_div(io_opt_sectors, clone->region_size)) {
2081 blk_limits_io_min(limits, clone->region_size << SECTOR_SHIFT);
2082 blk_limits_io_opt(limits, clone->region_size << SECTOR_SHIFT);
2083 }
2084
2085 disable_passdown_if_not_supported(clone);
2086 set_discard_limits(clone, limits);
2087}
2088
2089static int clone_iterate_devices(struct dm_target *ti,
2090 iterate_devices_callout_fn fn, void *data)
2091{
2092 int ret;
2093 struct clone *clone = ti->private;
2094 struct dm_dev *dest_dev = clone->dest_dev;
2095 struct dm_dev *source_dev = clone->source_dev;
2096
2097 ret = fn(ti, source_dev, 0, ti->len, data);
2098 if (!ret)
2099 ret = fn(ti, dest_dev, 0, ti->len, data);
2100 return ret;
2101}
2102
2103/*
2104 * dm-clone message functions.
2105 */
2106static void set_hydration_threshold(struct clone *clone, unsigned int nr_regions)
2107{
2108 WRITE_ONCE(clone->hydration_threshold, nr_regions);
2109
2110 /*
2111 * If user space sets hydration_threshold to zero then the hydration
2112 * will stop. If at a later time the hydration_threshold is increased
2113 * we must restart the hydration process by waking up the worker.
2114 */
2115 wake_worker(clone);
2116}
2117
2118static void set_hydration_batch_size(struct clone *clone, unsigned int nr_regions)
2119{
2120 WRITE_ONCE(clone->hydration_batch_size, nr_regions);
2121}
2122
2123static void enable_hydration(struct clone *clone)
2124{
2125 if (!test_and_set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags))
2126 wake_worker(clone);
2127}
2128
2129static void disable_hydration(struct clone *clone)
2130{
2131 clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
2132}
2133
2134static int clone_message(struct dm_target *ti, unsigned int argc, char **argv,
2135 char *result, unsigned int maxlen)
2136{
2137 struct clone *clone = ti->private;
2138 unsigned int value;
2139
2140 if (!argc)
2141 return -EINVAL;
2142
2143 if (!strcasecmp(argv[0], "enable_hydration")) {
2144 enable_hydration(clone);
2145 return 0;
2146 }
2147
2148 if (!strcasecmp(argv[0], "disable_hydration")) {
2149 disable_hydration(clone);
2150 return 0;
2151 }
2152
2153 if (argc != 2)
2154 return -EINVAL;
2155
2156 if (!strcasecmp(argv[0], "hydration_threshold")) {
2157 if (kstrtouint(argv[1], 10, &value))
2158 return -EINVAL;
2159
2160 set_hydration_threshold(clone, value);
2161
2162 return 0;
2163 }
2164
2165 if (!strcasecmp(argv[0], "hydration_batch_size")) {
2166 if (kstrtouint(argv[1], 10, &value))
2167 return -EINVAL;
2168
2169 set_hydration_batch_size(clone, value);
2170
2171 return 0;
2172 }
2173
2174 DMERR("%s: Unsupported message `%s'", clone_device_name(clone), argv[0]);
2175 return -EINVAL;
2176}
2177
2178static struct target_type clone_target = {
2179 .name = "clone",
2180 .version = {1, 0, 0},
2181 .module = THIS_MODULE,
2182 .ctr = clone_ctr,
2183 .dtr = clone_dtr,
2184 .map = clone_map,
2185 .end_io = clone_endio,
2186 .postsuspend = clone_postsuspend,
2187 .resume = clone_resume,
2188 .status = clone_status,
2189 .message = clone_message,
2190 .io_hints = clone_io_hints,
2191 .iterate_devices = clone_iterate_devices,
2192};
2193
2194/*---------------------------------------------------------------------------*/
2195
2196/* Module functions */
2197static int __init dm_clone_init(void)
2198{
2199 int r;
2200
2201 _hydration_cache = KMEM_CACHE(dm_clone_region_hydration, 0);
2202 if (!_hydration_cache)
2203 return -ENOMEM;
2204
2205 r = dm_register_target(&clone_target);
2206 if (r < 0) {
2207 DMERR("Failed to register clone target");
2208 return r;
2209 }
2210
2211 return 0;
2212}
2213
2214static void __exit dm_clone_exit(void)
2215{
2216 dm_unregister_target(&clone_target);
2217
2218 kmem_cache_destroy(_hydration_cache);
2219 _hydration_cache = NULL;
2220}
2221
2222/* Module hooks */
2223module_init(dm_clone_init);
2224module_exit(dm_clone_exit);
2225
2226MODULE_DESCRIPTION(DM_NAME " clone target");
2227MODULE_AUTHOR("Nikos Tsironis <ntsironis@arrikto.com>");
2228MODULE_LICENSE("GPL");