Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2019 Arrikto, Inc. All Rights Reserved.
4 */
5
6#include <linux/mm.h>
7#include <linux/bio.h>
8#include <linux/err.h>
9#include <linux/hash.h>
10#include <linux/list.h>
11#include <linux/log2.h>
12#include <linux/init.h>
13#include <linux/slab.h>
14#include <linux/wait.h>
15#include <linux/dm-io.h>
16#include <linux/mutex.h>
17#include <linux/atomic.h>
18#include <linux/bitops.h>
19#include <linux/blkdev.h>
20#include <linux/kdev_t.h>
21#include <linux/kernel.h>
22#include <linux/module.h>
23#include <linux/jiffies.h>
24#include <linux/mempool.h>
25#include <linux/spinlock.h>
26#include <linux/blk_types.h>
27#include <linux/dm-kcopyd.h>
28#include <linux/workqueue.h>
29#include <linux/backing-dev.h>
30#include <linux/device-mapper.h>
31
32#include "dm.h"
33#include "dm-clone-metadata.h"
34
35#define DM_MSG_PREFIX "clone"
36
37/*
38 * Minimum and maximum allowed region sizes
39 */
40#define MIN_REGION_SIZE (1 << 3) /* 4KB */
41#define MAX_REGION_SIZE (1 << 21) /* 1GB */
42
43#define MIN_HYDRATIONS 256 /* Size of hydration mempool */
44#define DEFAULT_HYDRATION_THRESHOLD 1 /* 1 region */
45#define DEFAULT_HYDRATION_BATCH_SIZE 1 /* Hydrate in batches of 1 region */
46
47#define COMMIT_PERIOD HZ /* 1 sec */
48
49/*
50 * Hydration hash table size: 1 << HASH_TABLE_BITS
51 */
52#define HASH_TABLE_BITS 15
53
54DECLARE_DM_KCOPYD_THROTTLE_WITH_MODULE_PARM(clone_hydration_throttle,
55 "A percentage of time allocated for hydrating regions");
56
57/* Slab cache for struct dm_clone_region_hydration */
58static struct kmem_cache *_hydration_cache;
59
60/* dm-clone metadata modes */
61enum clone_metadata_mode {
62 CM_WRITE, /* metadata may be changed */
63 CM_READ_ONLY, /* metadata may not be changed */
64 CM_FAIL, /* all metadata I/O fails */
65};
66
67struct hash_table_bucket;
68
69struct clone {
70 struct dm_target *ti;
71 struct dm_target_callbacks callbacks;
72
73 struct dm_dev *metadata_dev;
74 struct dm_dev *dest_dev;
75 struct dm_dev *source_dev;
76
77 unsigned long nr_regions;
78 sector_t region_size;
79 unsigned int region_shift;
80
81 /*
82 * A metadata commit and the actions taken in case it fails should run
83 * as a single atomic step.
84 */
85 struct mutex commit_lock;
86
87 struct dm_clone_metadata *cmd;
88
89 /* Region hydration hash table */
90 struct hash_table_bucket *ht;
91
92 atomic_t ios_in_flight;
93
94 wait_queue_head_t hydration_stopped;
95
96 mempool_t hydration_pool;
97
98 unsigned long last_commit_jiffies;
99
100 /*
101 * We defer incoming WRITE bios for regions that are not hydrated,
102 * until after these regions have been hydrated.
103 *
104 * Also, we defer REQ_FUA and REQ_PREFLUSH bios, until after the
105 * metadata have been committed.
106 */
107 spinlock_t lock;
108 struct bio_list deferred_bios;
109 struct bio_list deferred_discard_bios;
110 struct bio_list deferred_flush_bios;
111 struct bio_list deferred_flush_completions;
112
113 /* Maximum number of regions being copied during background hydration. */
114 unsigned int hydration_threshold;
115
116 /* Number of regions to batch together during background hydration. */
117 unsigned int hydration_batch_size;
118
119 /* Which region to hydrate next */
120 unsigned long hydration_offset;
121
122 atomic_t hydrations_in_flight;
123
124 /*
125 * Save a copy of the table line rather than reconstructing it for the
126 * status.
127 */
128 unsigned int nr_ctr_args;
129 const char **ctr_args;
130
131 struct workqueue_struct *wq;
132 struct work_struct worker;
133 struct delayed_work waker;
134
135 struct dm_kcopyd_client *kcopyd_client;
136
137 enum clone_metadata_mode mode;
138 unsigned long flags;
139};
140
141/*
142 * dm-clone flags
143 */
144#define DM_CLONE_DISCARD_PASSDOWN 0
145#define DM_CLONE_HYDRATION_ENABLED 1
146#define DM_CLONE_HYDRATION_SUSPENDED 2
147
148/*---------------------------------------------------------------------------*/
149
150/*
151 * Metadata failure handling.
152 */
153static enum clone_metadata_mode get_clone_mode(struct clone *clone)
154{
155 return READ_ONCE(clone->mode);
156}
157
158static const char *clone_device_name(struct clone *clone)
159{
160 return dm_table_device_name(clone->ti->table);
161}
162
163static void __set_clone_mode(struct clone *clone, enum clone_metadata_mode new_mode)
164{
165 const char *descs[] = {
166 "read-write",
167 "read-only",
168 "fail"
169 };
170
171 enum clone_metadata_mode old_mode = get_clone_mode(clone);
172
173 /* Never move out of fail mode */
174 if (old_mode == CM_FAIL)
175 new_mode = CM_FAIL;
176
177 switch (new_mode) {
178 case CM_FAIL:
179 case CM_READ_ONLY:
180 dm_clone_metadata_set_read_only(clone->cmd);
181 break;
182
183 case CM_WRITE:
184 dm_clone_metadata_set_read_write(clone->cmd);
185 break;
186 }
187
188 WRITE_ONCE(clone->mode, new_mode);
189
190 if (new_mode != old_mode) {
191 dm_table_event(clone->ti->table);
192 DMINFO("%s: Switching to %s mode", clone_device_name(clone),
193 descs[(int)new_mode]);
194 }
195}
196
197static void __abort_transaction(struct clone *clone)
198{
199 const char *dev_name = clone_device_name(clone);
200
201 if (get_clone_mode(clone) >= CM_READ_ONLY)
202 return;
203
204 DMERR("%s: Aborting current metadata transaction", dev_name);
205 if (dm_clone_metadata_abort(clone->cmd)) {
206 DMERR("%s: Failed to abort metadata transaction", dev_name);
207 __set_clone_mode(clone, CM_FAIL);
208 }
209}
210
211static void __reload_in_core_bitset(struct clone *clone)
212{
213 const char *dev_name = clone_device_name(clone);
214
215 if (get_clone_mode(clone) == CM_FAIL)
216 return;
217
218 /* Reload the on-disk bitset */
219 DMINFO("%s: Reloading on-disk bitmap", dev_name);
220 if (dm_clone_reload_in_core_bitset(clone->cmd)) {
221 DMERR("%s: Failed to reload on-disk bitmap", dev_name);
222 __set_clone_mode(clone, CM_FAIL);
223 }
224}
225
226static void __metadata_operation_failed(struct clone *clone, const char *op, int r)
227{
228 DMERR("%s: Metadata operation `%s' failed: error = %d",
229 clone_device_name(clone), op, r);
230
231 __abort_transaction(clone);
232 __set_clone_mode(clone, CM_READ_ONLY);
233
234 /*
235 * dm_clone_reload_in_core_bitset() may run concurrently with either
236 * dm_clone_set_region_hydrated() or dm_clone_cond_set_range(), but
237 * it's safe as we have already set the metadata to read-only mode.
238 */
239 __reload_in_core_bitset(clone);
240}
241
242/*---------------------------------------------------------------------------*/
243
244/* Wake up anyone waiting for region hydrations to stop */
245static inline void wakeup_hydration_waiters(struct clone *clone)
246{
247 wake_up_all(&clone->hydration_stopped);
248}
249
250static inline void wake_worker(struct clone *clone)
251{
252 queue_work(clone->wq, &clone->worker);
253}
254
255/*---------------------------------------------------------------------------*/
256
257/*
258 * bio helper functions.
259 */
260static inline void remap_to_source(struct clone *clone, struct bio *bio)
261{
262 bio_set_dev(bio, clone->source_dev->bdev);
263}
264
265static inline void remap_to_dest(struct clone *clone, struct bio *bio)
266{
267 bio_set_dev(bio, clone->dest_dev->bdev);
268}
269
270static bool bio_triggers_commit(struct clone *clone, struct bio *bio)
271{
272 return op_is_flush(bio->bi_opf) &&
273 dm_clone_changed_this_transaction(clone->cmd);
274}
275
276/* Get the address of the region in sectors */
277static inline sector_t region_to_sector(struct clone *clone, unsigned long region_nr)
278{
279 return (region_nr << clone->region_shift);
280}
281
282/* Get the region number of the bio */
283static inline unsigned long bio_to_region(struct clone *clone, struct bio *bio)
284{
285 return (bio->bi_iter.bi_sector >> clone->region_shift);
286}
287
288/* Get the region range covered by the bio */
289static void bio_region_range(struct clone *clone, struct bio *bio,
290 unsigned long *rs, unsigned long *re)
291{
292 *rs = dm_sector_div_up(bio->bi_iter.bi_sector, clone->region_size);
293 *re = bio_end_sector(bio) >> clone->region_shift;
294}
295
296/* Check whether a bio overwrites a region */
297static inline bool is_overwrite_bio(struct clone *clone, struct bio *bio)
298{
299 return (bio_data_dir(bio) == WRITE && bio_sectors(bio) == clone->region_size);
300}
301
302static void fail_bios(struct bio_list *bios, blk_status_t status)
303{
304 struct bio *bio;
305
306 while ((bio = bio_list_pop(bios))) {
307 bio->bi_status = status;
308 bio_endio(bio);
309 }
310}
311
312static void submit_bios(struct bio_list *bios)
313{
314 struct bio *bio;
315 struct blk_plug plug;
316
317 blk_start_plug(&plug);
318
319 while ((bio = bio_list_pop(bios)))
320 generic_make_request(bio);
321
322 blk_finish_plug(&plug);
323}
324
325/*
326 * Submit bio to the underlying device.
327 *
328 * If the bio triggers a commit, delay it, until after the metadata have been
329 * committed.
330 *
331 * NOTE: The bio remapping must be performed by the caller.
332 */
333static void issue_bio(struct clone *clone, struct bio *bio)
334{
335 unsigned long flags;
336
337 if (!bio_triggers_commit(clone, bio)) {
338 generic_make_request(bio);
339 return;
340 }
341
342 /*
343 * If the metadata mode is RO or FAIL we won't be able to commit the
344 * metadata, so we complete the bio with an error.
345 */
346 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
347 bio_io_error(bio);
348 return;
349 }
350
351 /*
352 * Batch together any bios that trigger commits and then issue a single
353 * commit for them in process_deferred_flush_bios().
354 */
355 spin_lock_irqsave(&clone->lock, flags);
356 bio_list_add(&clone->deferred_flush_bios, bio);
357 spin_unlock_irqrestore(&clone->lock, flags);
358
359 wake_worker(clone);
360}
361
362/*
363 * Remap bio to the destination device and submit it.
364 *
365 * If the bio triggers a commit, delay it, until after the metadata have been
366 * committed.
367 */
368static void remap_and_issue(struct clone *clone, struct bio *bio)
369{
370 remap_to_dest(clone, bio);
371 issue_bio(clone, bio);
372}
373
374/*
375 * Issue bios that have been deferred until after their region has finished
376 * hydrating.
377 *
378 * We delegate the bio submission to the worker thread, so this is safe to call
379 * from interrupt context.
380 */
381static void issue_deferred_bios(struct clone *clone, struct bio_list *bios)
382{
383 struct bio *bio;
384 unsigned long flags;
385 struct bio_list flush_bios = BIO_EMPTY_LIST;
386 struct bio_list normal_bios = BIO_EMPTY_LIST;
387
388 if (bio_list_empty(bios))
389 return;
390
391 while ((bio = bio_list_pop(bios))) {
392 if (bio_triggers_commit(clone, bio))
393 bio_list_add(&flush_bios, bio);
394 else
395 bio_list_add(&normal_bios, bio);
396 }
397
398 spin_lock_irqsave(&clone->lock, flags);
399 bio_list_merge(&clone->deferred_bios, &normal_bios);
400 bio_list_merge(&clone->deferred_flush_bios, &flush_bios);
401 spin_unlock_irqrestore(&clone->lock, flags);
402
403 wake_worker(clone);
404}
405
406static void complete_overwrite_bio(struct clone *clone, struct bio *bio)
407{
408 unsigned long flags;
409
410 /*
411 * If the bio has the REQ_FUA flag set we must commit the metadata
412 * before signaling its completion.
413 *
414 * complete_overwrite_bio() is only called by hydration_complete(),
415 * after having successfully updated the metadata. This means we don't
416 * need to call dm_clone_changed_this_transaction() to check if the
417 * metadata has changed and thus we can avoid taking the metadata spin
418 * lock.
419 */
420 if (!(bio->bi_opf & REQ_FUA)) {
421 bio_endio(bio);
422 return;
423 }
424
425 /*
426 * If the metadata mode is RO or FAIL we won't be able to commit the
427 * metadata, so we complete the bio with an error.
428 */
429 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
430 bio_io_error(bio);
431 return;
432 }
433
434 /*
435 * Batch together any bios that trigger commits and then issue a single
436 * commit for them in process_deferred_flush_bios().
437 */
438 spin_lock_irqsave(&clone->lock, flags);
439 bio_list_add(&clone->deferred_flush_completions, bio);
440 spin_unlock_irqrestore(&clone->lock, flags);
441
442 wake_worker(clone);
443}
444
445static void trim_bio(struct bio *bio, sector_t sector, unsigned int len)
446{
447 bio->bi_iter.bi_sector = sector;
448 bio->bi_iter.bi_size = to_bytes(len);
449}
450
451static void complete_discard_bio(struct clone *clone, struct bio *bio, bool success)
452{
453 unsigned long rs, re;
454
455 /*
456 * If the destination device supports discards, remap and trim the
457 * discard bio and pass it down. Otherwise complete the bio
458 * immediately.
459 */
460 if (test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags) && success) {
461 remap_to_dest(clone, bio);
462 bio_region_range(clone, bio, &rs, &re);
463 trim_bio(bio, rs << clone->region_shift,
464 (re - rs) << clone->region_shift);
465 generic_make_request(bio);
466 } else
467 bio_endio(bio);
468}
469
470static void process_discard_bio(struct clone *clone, struct bio *bio)
471{
472 unsigned long rs, re, flags;
473
474 bio_region_range(clone, bio, &rs, &re);
475 BUG_ON(re > clone->nr_regions);
476
477 if (unlikely(rs == re)) {
478 bio_endio(bio);
479 return;
480 }
481
482 /*
483 * The covered regions are already hydrated so we just need to pass
484 * down the discard.
485 */
486 if (dm_clone_is_range_hydrated(clone->cmd, rs, re - rs)) {
487 complete_discard_bio(clone, bio, true);
488 return;
489 }
490
491 /*
492 * If the metadata mode is RO or FAIL we won't be able to update the
493 * metadata for the regions covered by the discard so we just ignore
494 * it.
495 */
496 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
497 bio_endio(bio);
498 return;
499 }
500
501 /*
502 * Defer discard processing.
503 */
504 spin_lock_irqsave(&clone->lock, flags);
505 bio_list_add(&clone->deferred_discard_bios, bio);
506 spin_unlock_irqrestore(&clone->lock, flags);
507
508 wake_worker(clone);
509}
510
511/*---------------------------------------------------------------------------*/
512
513/*
514 * dm-clone region hydrations.
515 */
516struct dm_clone_region_hydration {
517 struct clone *clone;
518 unsigned long region_nr;
519
520 struct bio *overwrite_bio;
521 bio_end_io_t *overwrite_bio_end_io;
522
523 struct bio_list deferred_bios;
524
525 blk_status_t status;
526
527 /* Used by hydration batching */
528 struct list_head list;
529
530 /* Used by hydration hash table */
531 struct hlist_node h;
532};
533
534/*
535 * Hydration hash table implementation.
536 *
537 * Ideally we would like to use list_bl, which uses bit spin locks and employs
538 * the least significant bit of the list head to lock the corresponding bucket,
539 * reducing the memory overhead for the locks. But, currently, list_bl and bit
540 * spin locks don't support IRQ safe versions. Since we have to take the lock
541 * in both process and interrupt context, we must fall back to using regular
542 * spin locks; one per hash table bucket.
543 */
544struct hash_table_bucket {
545 struct hlist_head head;
546
547 /* Spinlock protecting the bucket */
548 spinlock_t lock;
549};
550
551#define bucket_lock_irqsave(bucket, flags) \
552 spin_lock_irqsave(&(bucket)->lock, flags)
553
554#define bucket_unlock_irqrestore(bucket, flags) \
555 spin_unlock_irqrestore(&(bucket)->lock, flags)
556
557static int hash_table_init(struct clone *clone)
558{
559 unsigned int i, sz;
560 struct hash_table_bucket *bucket;
561
562 sz = 1 << HASH_TABLE_BITS;
563
564 clone->ht = kvmalloc(sz * sizeof(struct hash_table_bucket), GFP_KERNEL);
565 if (!clone->ht)
566 return -ENOMEM;
567
568 for (i = 0; i < sz; i++) {
569 bucket = clone->ht + i;
570
571 INIT_HLIST_HEAD(&bucket->head);
572 spin_lock_init(&bucket->lock);
573 }
574
575 return 0;
576}
577
578static void hash_table_exit(struct clone *clone)
579{
580 kvfree(clone->ht);
581}
582
583static struct hash_table_bucket *get_hash_table_bucket(struct clone *clone,
584 unsigned long region_nr)
585{
586 return &clone->ht[hash_long(region_nr, HASH_TABLE_BITS)];
587}
588
589/*
590 * Search hash table for a hydration with hd->region_nr == region_nr
591 *
592 * NOTE: Must be called with the bucket lock held
593 */
594static struct dm_clone_region_hydration *__hash_find(struct hash_table_bucket *bucket,
595 unsigned long region_nr)
596{
597 struct dm_clone_region_hydration *hd;
598
599 hlist_for_each_entry(hd, &bucket->head, h) {
600 if (hd->region_nr == region_nr)
601 return hd;
602 }
603
604 return NULL;
605}
606
607/*
608 * Insert a hydration into the hash table.
609 *
610 * NOTE: Must be called with the bucket lock held.
611 */
612static inline void __insert_region_hydration(struct hash_table_bucket *bucket,
613 struct dm_clone_region_hydration *hd)
614{
615 hlist_add_head(&hd->h, &bucket->head);
616}
617
618/*
619 * This function inserts a hydration into the hash table, unless someone else
620 * managed to insert a hydration for the same region first. In the latter case
621 * it returns the existing hydration descriptor for this region.
622 *
623 * NOTE: Must be called with the hydration hash table lock held.
624 */
625static struct dm_clone_region_hydration *
626__find_or_insert_region_hydration(struct hash_table_bucket *bucket,
627 struct dm_clone_region_hydration *hd)
628{
629 struct dm_clone_region_hydration *hd2;
630
631 hd2 = __hash_find(bucket, hd->region_nr);
632 if (hd2)
633 return hd2;
634
635 __insert_region_hydration(bucket, hd);
636
637 return hd;
638}
639
640/*---------------------------------------------------------------------------*/
641
642/* Allocate a hydration */
643static struct dm_clone_region_hydration *alloc_hydration(struct clone *clone)
644{
645 struct dm_clone_region_hydration *hd;
646
647 /*
648 * Allocate a hydration from the hydration mempool.
649 * This might block but it can't fail.
650 */
651 hd = mempool_alloc(&clone->hydration_pool, GFP_NOIO);
652 hd->clone = clone;
653
654 return hd;
655}
656
657static inline void free_hydration(struct dm_clone_region_hydration *hd)
658{
659 mempool_free(hd, &hd->clone->hydration_pool);
660}
661
662/* Initialize a hydration */
663static void hydration_init(struct dm_clone_region_hydration *hd, unsigned long region_nr)
664{
665 hd->region_nr = region_nr;
666 hd->overwrite_bio = NULL;
667 bio_list_init(&hd->deferred_bios);
668 hd->status = 0;
669
670 INIT_LIST_HEAD(&hd->list);
671 INIT_HLIST_NODE(&hd->h);
672}
673
674/*---------------------------------------------------------------------------*/
675
676/*
677 * Update dm-clone's metadata after a region has finished hydrating and remove
678 * hydration from the hash table.
679 */
680static int hydration_update_metadata(struct dm_clone_region_hydration *hd)
681{
682 int r = 0;
683 unsigned long flags;
684 struct hash_table_bucket *bucket;
685 struct clone *clone = hd->clone;
686
687 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
688 r = -EPERM;
689
690 /* Update the metadata */
691 if (likely(!r) && hd->status == BLK_STS_OK)
692 r = dm_clone_set_region_hydrated(clone->cmd, hd->region_nr);
693
694 bucket = get_hash_table_bucket(clone, hd->region_nr);
695
696 /* Remove hydration from hash table */
697 bucket_lock_irqsave(bucket, flags);
698 hlist_del(&hd->h);
699 bucket_unlock_irqrestore(bucket, flags);
700
701 return r;
702}
703
704/*
705 * Complete a region's hydration:
706 *
707 * 1. Update dm-clone's metadata.
708 * 2. Remove hydration from hash table.
709 * 3. Complete overwrite bio.
710 * 4. Issue deferred bios.
711 * 5. If this was the last hydration, wake up anyone waiting for
712 * hydrations to finish.
713 */
714static void hydration_complete(struct dm_clone_region_hydration *hd)
715{
716 int r;
717 blk_status_t status;
718 struct clone *clone = hd->clone;
719
720 r = hydration_update_metadata(hd);
721
722 if (hd->status == BLK_STS_OK && likely(!r)) {
723 if (hd->overwrite_bio)
724 complete_overwrite_bio(clone, hd->overwrite_bio);
725
726 issue_deferred_bios(clone, &hd->deferred_bios);
727 } else {
728 status = r ? BLK_STS_IOERR : hd->status;
729
730 if (hd->overwrite_bio)
731 bio_list_add(&hd->deferred_bios, hd->overwrite_bio);
732
733 fail_bios(&hd->deferred_bios, status);
734 }
735
736 free_hydration(hd);
737
738 if (atomic_dec_and_test(&clone->hydrations_in_flight))
739 wakeup_hydration_waiters(clone);
740}
741
742static void hydration_kcopyd_callback(int read_err, unsigned long write_err, void *context)
743{
744 blk_status_t status;
745
746 struct dm_clone_region_hydration *tmp, *hd = context;
747 struct clone *clone = hd->clone;
748
749 LIST_HEAD(batched_hydrations);
750
751 if (read_err || write_err) {
752 DMERR_LIMIT("%s: hydration failed", clone_device_name(clone));
753 status = BLK_STS_IOERR;
754 } else {
755 status = BLK_STS_OK;
756 }
757 list_splice_tail(&hd->list, &batched_hydrations);
758
759 hd->status = status;
760 hydration_complete(hd);
761
762 /* Complete batched hydrations */
763 list_for_each_entry_safe(hd, tmp, &batched_hydrations, list) {
764 hd->status = status;
765 hydration_complete(hd);
766 }
767
768 /* Continue background hydration, if there is no I/O in-flight */
769 if (test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) &&
770 !atomic_read(&clone->ios_in_flight))
771 wake_worker(clone);
772}
773
774static void hydration_copy(struct dm_clone_region_hydration *hd, unsigned int nr_regions)
775{
776 unsigned long region_start, region_end;
777 sector_t tail_size, region_size, total_size;
778 struct dm_io_region from, to;
779 struct clone *clone = hd->clone;
780
781 region_size = clone->region_size;
782 region_start = hd->region_nr;
783 region_end = region_start + nr_regions - 1;
784
785 total_size = (nr_regions - 1) << clone->region_shift;
786
787 if (region_end == clone->nr_regions - 1) {
788 /*
789 * The last region of the target might be smaller than
790 * region_size.
791 */
792 tail_size = clone->ti->len & (region_size - 1);
793 if (!tail_size)
794 tail_size = region_size;
795 } else {
796 tail_size = region_size;
797 }
798
799 total_size += tail_size;
800
801 from.bdev = clone->source_dev->bdev;
802 from.sector = region_to_sector(clone, region_start);
803 from.count = total_size;
804
805 to.bdev = clone->dest_dev->bdev;
806 to.sector = from.sector;
807 to.count = from.count;
808
809 /* Issue copy */
810 atomic_add(nr_regions, &clone->hydrations_in_flight);
811 dm_kcopyd_copy(clone->kcopyd_client, &from, 1, &to, 0,
812 hydration_kcopyd_callback, hd);
813}
814
815static void overwrite_endio(struct bio *bio)
816{
817 struct dm_clone_region_hydration *hd = bio->bi_private;
818
819 bio->bi_end_io = hd->overwrite_bio_end_io;
820 hd->status = bio->bi_status;
821
822 hydration_complete(hd);
823}
824
825static void hydration_overwrite(struct dm_clone_region_hydration *hd, struct bio *bio)
826{
827 /*
828 * We don't need to save and restore bio->bi_private because device
829 * mapper core generates a new bio for us to use, with clean
830 * bi_private.
831 */
832 hd->overwrite_bio = bio;
833 hd->overwrite_bio_end_io = bio->bi_end_io;
834
835 bio->bi_end_io = overwrite_endio;
836 bio->bi_private = hd;
837
838 atomic_inc(&hd->clone->hydrations_in_flight);
839 generic_make_request(bio);
840}
841
842/*
843 * Hydrate bio's region.
844 *
845 * This function starts the hydration of the bio's region and puts the bio in
846 * the list of deferred bios for this region. In case, by the time this
847 * function is called, the region has finished hydrating it's submitted to the
848 * destination device.
849 *
850 * NOTE: The bio remapping must be performed by the caller.
851 */
852static void hydrate_bio_region(struct clone *clone, struct bio *bio)
853{
854 unsigned long flags;
855 unsigned long region_nr;
856 struct hash_table_bucket *bucket;
857 struct dm_clone_region_hydration *hd, *hd2;
858
859 region_nr = bio_to_region(clone, bio);
860 bucket = get_hash_table_bucket(clone, region_nr);
861
862 bucket_lock_irqsave(bucket, flags);
863
864 hd = __hash_find(bucket, region_nr);
865 if (hd) {
866 /* Someone else is hydrating the region */
867 bio_list_add(&hd->deferred_bios, bio);
868 bucket_unlock_irqrestore(bucket, flags);
869 return;
870 }
871
872 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
873 /* The region has been hydrated */
874 bucket_unlock_irqrestore(bucket, flags);
875 issue_bio(clone, bio);
876 return;
877 }
878
879 /*
880 * We must allocate a hydration descriptor and start the hydration of
881 * the corresponding region.
882 */
883 bucket_unlock_irqrestore(bucket, flags);
884
885 hd = alloc_hydration(clone);
886 hydration_init(hd, region_nr);
887
888 bucket_lock_irqsave(bucket, flags);
889
890 /* Check if the region has been hydrated in the meantime. */
891 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
892 bucket_unlock_irqrestore(bucket, flags);
893 free_hydration(hd);
894 issue_bio(clone, bio);
895 return;
896 }
897
898 hd2 = __find_or_insert_region_hydration(bucket, hd);
899 if (hd2 != hd) {
900 /* Someone else started the region's hydration. */
901 bio_list_add(&hd2->deferred_bios, bio);
902 bucket_unlock_irqrestore(bucket, flags);
903 free_hydration(hd);
904 return;
905 }
906
907 /*
908 * If the metadata mode is RO or FAIL then there is no point starting a
909 * hydration, since we will not be able to update the metadata when the
910 * hydration finishes.
911 */
912 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
913 hlist_del(&hd->h);
914 bucket_unlock_irqrestore(bucket, flags);
915 free_hydration(hd);
916 bio_io_error(bio);
917 return;
918 }
919
920 /*
921 * Start region hydration.
922 *
923 * If a bio overwrites a region, i.e., its size is equal to the
924 * region's size, then we don't need to copy the region from the source
925 * to the destination device.
926 */
927 if (is_overwrite_bio(clone, bio)) {
928 bucket_unlock_irqrestore(bucket, flags);
929 hydration_overwrite(hd, bio);
930 } else {
931 bio_list_add(&hd->deferred_bios, bio);
932 bucket_unlock_irqrestore(bucket, flags);
933 hydration_copy(hd, 1);
934 }
935}
936
937/*---------------------------------------------------------------------------*/
938
939/*
940 * Background hydrations.
941 */
942
943/*
944 * Batch region hydrations.
945 *
946 * To better utilize device bandwidth we batch together the hydration of
947 * adjacent regions. This allows us to use small region sizes, e.g., 4KB, which
948 * is good for small, random write performance (because of the overwriting of
949 * un-hydrated regions) and at the same time issue big copy requests to kcopyd
950 * to achieve high hydration bandwidth.
951 */
952struct batch_info {
953 struct dm_clone_region_hydration *head;
954 unsigned int nr_batched_regions;
955};
956
957static void __batch_hydration(struct batch_info *batch,
958 struct dm_clone_region_hydration *hd)
959{
960 struct clone *clone = hd->clone;
961 unsigned int max_batch_size = READ_ONCE(clone->hydration_batch_size);
962
963 if (batch->head) {
964 /* Try to extend the current batch */
965 if (batch->nr_batched_regions < max_batch_size &&
966 (batch->head->region_nr + batch->nr_batched_regions) == hd->region_nr) {
967 list_add_tail(&hd->list, &batch->head->list);
968 batch->nr_batched_regions++;
969 hd = NULL;
970 }
971
972 /* Check if we should issue the current batch */
973 if (batch->nr_batched_regions >= max_batch_size || hd) {
974 hydration_copy(batch->head, batch->nr_batched_regions);
975 batch->head = NULL;
976 batch->nr_batched_regions = 0;
977 }
978 }
979
980 if (!hd)
981 return;
982
983 /* We treat max batch sizes of zero and one equivalently */
984 if (max_batch_size <= 1) {
985 hydration_copy(hd, 1);
986 return;
987 }
988
989 /* Start a new batch */
990 BUG_ON(!list_empty(&hd->list));
991 batch->head = hd;
992 batch->nr_batched_regions = 1;
993}
994
995static unsigned long __start_next_hydration(struct clone *clone,
996 unsigned long offset,
997 struct batch_info *batch)
998{
999 unsigned long flags;
1000 struct hash_table_bucket *bucket;
1001 struct dm_clone_region_hydration *hd;
1002 unsigned long nr_regions = clone->nr_regions;
1003
1004 hd = alloc_hydration(clone);
1005
1006 /* Try to find a region to hydrate. */
1007 do {
1008 offset = dm_clone_find_next_unhydrated_region(clone->cmd, offset);
1009 if (offset == nr_regions)
1010 break;
1011
1012 bucket = get_hash_table_bucket(clone, offset);
1013 bucket_lock_irqsave(bucket, flags);
1014
1015 if (!dm_clone_is_region_hydrated(clone->cmd, offset) &&
1016 !__hash_find(bucket, offset)) {
1017 hydration_init(hd, offset);
1018 __insert_region_hydration(bucket, hd);
1019 bucket_unlock_irqrestore(bucket, flags);
1020
1021 /* Batch hydration */
1022 __batch_hydration(batch, hd);
1023
1024 return (offset + 1);
1025 }
1026
1027 bucket_unlock_irqrestore(bucket, flags);
1028
1029 } while (++offset < nr_regions);
1030
1031 if (hd)
1032 free_hydration(hd);
1033
1034 return offset;
1035}
1036
1037/*
1038 * This function searches for regions that still reside in the source device
1039 * and starts their hydration.
1040 */
1041static void do_hydration(struct clone *clone)
1042{
1043 unsigned int current_volume;
1044 unsigned long offset, nr_regions = clone->nr_regions;
1045
1046 struct batch_info batch = {
1047 .head = NULL,
1048 .nr_batched_regions = 0,
1049 };
1050
1051 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
1052 return;
1053
1054 if (dm_clone_is_hydration_done(clone->cmd))
1055 return;
1056
1057 /*
1058 * Avoid race with device suspension.
1059 */
1060 atomic_inc(&clone->hydrations_in_flight);
1061
1062 /*
1063 * Make sure atomic_inc() is ordered before test_bit(), otherwise we
1064 * might race with clone_postsuspend() and start a region hydration
1065 * after the target has been suspended.
1066 *
1067 * This is paired with the smp_mb__after_atomic() in
1068 * clone_postsuspend().
1069 */
1070 smp_mb__after_atomic();
1071
1072 offset = clone->hydration_offset;
1073 while (likely(!test_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags)) &&
1074 !atomic_read(&clone->ios_in_flight) &&
1075 test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags) &&
1076 offset < nr_regions) {
1077 current_volume = atomic_read(&clone->hydrations_in_flight);
1078 current_volume += batch.nr_batched_regions;
1079
1080 if (current_volume > READ_ONCE(clone->hydration_threshold))
1081 break;
1082
1083 offset = __start_next_hydration(clone, offset, &batch);
1084 }
1085
1086 if (batch.head)
1087 hydration_copy(batch.head, batch.nr_batched_regions);
1088
1089 if (offset >= nr_regions)
1090 offset = 0;
1091
1092 clone->hydration_offset = offset;
1093
1094 if (atomic_dec_and_test(&clone->hydrations_in_flight))
1095 wakeup_hydration_waiters(clone);
1096}
1097
1098/*---------------------------------------------------------------------------*/
1099
1100static bool need_commit_due_to_time(struct clone *clone)
1101{
1102 return !time_in_range(jiffies, clone->last_commit_jiffies,
1103 clone->last_commit_jiffies + COMMIT_PERIOD);
1104}
1105
1106/*
1107 * A non-zero return indicates read-only or fail mode.
1108 */
1109static int commit_metadata(struct clone *clone)
1110{
1111 int r = 0;
1112
1113 mutex_lock(&clone->commit_lock);
1114
1115 if (!dm_clone_changed_this_transaction(clone->cmd))
1116 goto out;
1117
1118 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY)) {
1119 r = -EPERM;
1120 goto out;
1121 }
1122
1123 r = dm_clone_metadata_commit(clone->cmd);
1124
1125 if (unlikely(r)) {
1126 __metadata_operation_failed(clone, "dm_clone_metadata_commit", r);
1127 goto out;
1128 }
1129
1130 if (dm_clone_is_hydration_done(clone->cmd))
1131 dm_table_event(clone->ti->table);
1132out:
1133 mutex_unlock(&clone->commit_lock);
1134
1135 return r;
1136}
1137
1138static void process_deferred_discards(struct clone *clone)
1139{
1140 int r = -EPERM;
1141 struct bio *bio;
1142 struct blk_plug plug;
1143 unsigned long rs, re, flags;
1144 struct bio_list discards = BIO_EMPTY_LIST;
1145
1146 spin_lock_irqsave(&clone->lock, flags);
1147 bio_list_merge(&discards, &clone->deferred_discard_bios);
1148 bio_list_init(&clone->deferred_discard_bios);
1149 spin_unlock_irqrestore(&clone->lock, flags);
1150
1151 if (bio_list_empty(&discards))
1152 return;
1153
1154 if (unlikely(get_clone_mode(clone) >= CM_READ_ONLY))
1155 goto out;
1156
1157 /* Update the metadata */
1158 bio_list_for_each(bio, &discards) {
1159 bio_region_range(clone, bio, &rs, &re);
1160 /*
1161 * A discard request might cover regions that have been already
1162 * hydrated. There is no need to update the metadata for these
1163 * regions.
1164 */
1165 r = dm_clone_cond_set_range(clone->cmd, rs, re - rs);
1166
1167 if (unlikely(r))
1168 break;
1169 }
1170out:
1171 blk_start_plug(&plug);
1172 while ((bio = bio_list_pop(&discards)))
1173 complete_discard_bio(clone, bio, r == 0);
1174 blk_finish_plug(&plug);
1175}
1176
1177static void process_deferred_bios(struct clone *clone)
1178{
1179 unsigned long flags;
1180 struct bio_list bios = BIO_EMPTY_LIST;
1181
1182 spin_lock_irqsave(&clone->lock, flags);
1183 bio_list_merge(&bios, &clone->deferred_bios);
1184 bio_list_init(&clone->deferred_bios);
1185 spin_unlock_irqrestore(&clone->lock, flags);
1186
1187 if (bio_list_empty(&bios))
1188 return;
1189
1190 submit_bios(&bios);
1191}
1192
1193static void process_deferred_flush_bios(struct clone *clone)
1194{
1195 struct bio *bio;
1196 unsigned long flags;
1197 struct bio_list bios = BIO_EMPTY_LIST;
1198 struct bio_list bio_completions = BIO_EMPTY_LIST;
1199
1200 /*
1201 * If there are any deferred flush bios, we must commit the metadata
1202 * before issuing them or signaling their completion.
1203 */
1204 spin_lock_irqsave(&clone->lock, flags);
1205 bio_list_merge(&bios, &clone->deferred_flush_bios);
1206 bio_list_init(&clone->deferred_flush_bios);
1207
1208 bio_list_merge(&bio_completions, &clone->deferred_flush_completions);
1209 bio_list_init(&clone->deferred_flush_completions);
1210 spin_unlock_irqrestore(&clone->lock, flags);
1211
1212 if (bio_list_empty(&bios) && bio_list_empty(&bio_completions) &&
1213 !(dm_clone_changed_this_transaction(clone->cmd) && need_commit_due_to_time(clone)))
1214 return;
1215
1216 if (commit_metadata(clone)) {
1217 bio_list_merge(&bios, &bio_completions);
1218
1219 while ((bio = bio_list_pop(&bios)))
1220 bio_io_error(bio);
1221
1222 return;
1223 }
1224
1225 clone->last_commit_jiffies = jiffies;
1226
1227 while ((bio = bio_list_pop(&bio_completions)))
1228 bio_endio(bio);
1229
1230 while ((bio = bio_list_pop(&bios)))
1231 generic_make_request(bio);
1232}
1233
1234static void do_worker(struct work_struct *work)
1235{
1236 struct clone *clone = container_of(work, typeof(*clone), worker);
1237
1238 process_deferred_bios(clone);
1239 process_deferred_discards(clone);
1240
1241 /*
1242 * process_deferred_flush_bios():
1243 *
1244 * - Commit metadata
1245 *
1246 * - Process deferred REQ_FUA completions
1247 *
1248 * - Process deferred REQ_PREFLUSH bios
1249 */
1250 process_deferred_flush_bios(clone);
1251
1252 /* Background hydration */
1253 do_hydration(clone);
1254}
1255
1256/*
1257 * Commit periodically so that not too much unwritten data builds up.
1258 *
1259 * Also, restart background hydration, if it has been stopped by in-flight I/O.
1260 */
1261static void do_waker(struct work_struct *work)
1262{
1263 struct clone *clone = container_of(to_delayed_work(work), struct clone, waker);
1264
1265 wake_worker(clone);
1266 queue_delayed_work(clone->wq, &clone->waker, COMMIT_PERIOD);
1267}
1268
1269/*---------------------------------------------------------------------------*/
1270
1271/*
1272 * Target methods
1273 */
1274static int clone_map(struct dm_target *ti, struct bio *bio)
1275{
1276 struct clone *clone = ti->private;
1277 unsigned long region_nr;
1278
1279 atomic_inc(&clone->ios_in_flight);
1280
1281 if (unlikely(get_clone_mode(clone) == CM_FAIL))
1282 return DM_MAPIO_KILL;
1283
1284 /*
1285 * REQ_PREFLUSH bios carry no data:
1286 *
1287 * - Commit metadata, if changed
1288 *
1289 * - Pass down to destination device
1290 */
1291 if (bio->bi_opf & REQ_PREFLUSH) {
1292 remap_and_issue(clone, bio);
1293 return DM_MAPIO_SUBMITTED;
1294 }
1295
1296 bio->bi_iter.bi_sector = dm_target_offset(ti, bio->bi_iter.bi_sector);
1297
1298 /*
1299 * dm-clone interprets discards and performs a fast hydration of the
1300 * discarded regions, i.e., we skip the copy from the source device and
1301 * just mark the regions as hydrated.
1302 */
1303 if (bio_op(bio) == REQ_OP_DISCARD) {
1304 process_discard_bio(clone, bio);
1305 return DM_MAPIO_SUBMITTED;
1306 }
1307
1308 /*
1309 * If the bio's region is hydrated, redirect it to the destination
1310 * device.
1311 *
1312 * If the region is not hydrated and the bio is a READ, redirect it to
1313 * the source device.
1314 *
1315 * Else, defer WRITE bio until after its region has been hydrated and
1316 * start the region's hydration immediately.
1317 */
1318 region_nr = bio_to_region(clone, bio);
1319 if (dm_clone_is_region_hydrated(clone->cmd, region_nr)) {
1320 remap_and_issue(clone, bio);
1321 return DM_MAPIO_SUBMITTED;
1322 } else if (bio_data_dir(bio) == READ) {
1323 remap_to_source(clone, bio);
1324 return DM_MAPIO_REMAPPED;
1325 }
1326
1327 remap_to_dest(clone, bio);
1328 hydrate_bio_region(clone, bio);
1329
1330 return DM_MAPIO_SUBMITTED;
1331}
1332
1333static int clone_endio(struct dm_target *ti, struct bio *bio, blk_status_t *error)
1334{
1335 struct clone *clone = ti->private;
1336
1337 atomic_dec(&clone->ios_in_flight);
1338
1339 return DM_ENDIO_DONE;
1340}
1341
1342static void emit_flags(struct clone *clone, char *result, unsigned int maxlen,
1343 ssize_t *sz_ptr)
1344{
1345 ssize_t sz = *sz_ptr;
1346 unsigned int count;
1347
1348 count = !test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
1349 count += !test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
1350
1351 DMEMIT("%u ", count);
1352
1353 if (!test_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags))
1354 DMEMIT("no_hydration ");
1355
1356 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags))
1357 DMEMIT("no_discard_passdown ");
1358
1359 *sz_ptr = sz;
1360}
1361
1362static void emit_core_args(struct clone *clone, char *result,
1363 unsigned int maxlen, ssize_t *sz_ptr)
1364{
1365 ssize_t sz = *sz_ptr;
1366 unsigned int count = 4;
1367
1368 DMEMIT("%u hydration_threshold %u hydration_batch_size %u ", count,
1369 READ_ONCE(clone->hydration_threshold),
1370 READ_ONCE(clone->hydration_batch_size));
1371
1372 *sz_ptr = sz;
1373}
1374
1375/*
1376 * Status format:
1377 *
1378 * <metadata block size> <#used metadata blocks>/<#total metadata blocks>
1379 * <clone region size> <#hydrated regions>/<#total regions> <#hydrating regions>
1380 * <#features> <features>* <#core args> <core args>* <clone metadata mode>
1381 */
1382static void clone_status(struct dm_target *ti, status_type_t type,
1383 unsigned int status_flags, char *result,
1384 unsigned int maxlen)
1385{
1386 int r;
1387 unsigned int i;
1388 ssize_t sz = 0;
1389 dm_block_t nr_free_metadata_blocks = 0;
1390 dm_block_t nr_metadata_blocks = 0;
1391 char buf[BDEVNAME_SIZE];
1392 struct clone *clone = ti->private;
1393
1394 switch (type) {
1395 case STATUSTYPE_INFO:
1396 if (get_clone_mode(clone) == CM_FAIL) {
1397 DMEMIT("Fail");
1398 break;
1399 }
1400
1401 /* Commit to ensure statistics aren't out-of-date */
1402 if (!(status_flags & DM_STATUS_NOFLUSH_FLAG) && !dm_suspended(ti))
1403 (void) commit_metadata(clone);
1404
1405 r = dm_clone_get_free_metadata_block_count(clone->cmd, &nr_free_metadata_blocks);
1406
1407 if (r) {
1408 DMERR("%s: dm_clone_get_free_metadata_block_count returned %d",
1409 clone_device_name(clone), r);
1410 goto error;
1411 }
1412
1413 r = dm_clone_get_metadata_dev_size(clone->cmd, &nr_metadata_blocks);
1414
1415 if (r) {
1416 DMERR("%s: dm_clone_get_metadata_dev_size returned %d",
1417 clone_device_name(clone), r);
1418 goto error;
1419 }
1420
1421 DMEMIT("%u %llu/%llu %llu %lu/%lu %u ",
1422 DM_CLONE_METADATA_BLOCK_SIZE,
1423 (unsigned long long)(nr_metadata_blocks - nr_free_metadata_blocks),
1424 (unsigned long long)nr_metadata_blocks,
1425 (unsigned long long)clone->region_size,
1426 dm_clone_nr_of_hydrated_regions(clone->cmd),
1427 clone->nr_regions,
1428 atomic_read(&clone->hydrations_in_flight));
1429
1430 emit_flags(clone, result, maxlen, &sz);
1431 emit_core_args(clone, result, maxlen, &sz);
1432
1433 switch (get_clone_mode(clone)) {
1434 case CM_WRITE:
1435 DMEMIT("rw");
1436 break;
1437 case CM_READ_ONLY:
1438 DMEMIT("ro");
1439 break;
1440 case CM_FAIL:
1441 DMEMIT("Fail");
1442 }
1443
1444 break;
1445
1446 case STATUSTYPE_TABLE:
1447 format_dev_t(buf, clone->metadata_dev->bdev->bd_dev);
1448 DMEMIT("%s ", buf);
1449
1450 format_dev_t(buf, clone->dest_dev->bdev->bd_dev);
1451 DMEMIT("%s ", buf);
1452
1453 format_dev_t(buf, clone->source_dev->bdev->bd_dev);
1454 DMEMIT("%s", buf);
1455
1456 for (i = 0; i < clone->nr_ctr_args; i++)
1457 DMEMIT(" %s", clone->ctr_args[i]);
1458 }
1459
1460 return;
1461
1462error:
1463 DMEMIT("Error");
1464}
1465
1466static int clone_is_congested(struct dm_target_callbacks *cb, int bdi_bits)
1467{
1468 struct request_queue *dest_q, *source_q;
1469 struct clone *clone = container_of(cb, struct clone, callbacks);
1470
1471 source_q = bdev_get_queue(clone->source_dev->bdev);
1472 dest_q = bdev_get_queue(clone->dest_dev->bdev);
1473
1474 return (bdi_congested(dest_q->backing_dev_info, bdi_bits) |
1475 bdi_congested(source_q->backing_dev_info, bdi_bits));
1476}
1477
1478static sector_t get_dev_size(struct dm_dev *dev)
1479{
1480 return i_size_read(dev->bdev->bd_inode) >> SECTOR_SHIFT;
1481}
1482
1483/*---------------------------------------------------------------------------*/
1484
1485/*
1486 * Construct a clone device mapping:
1487 *
1488 * clone <metadata dev> <destination dev> <source dev> <region size>
1489 * [<#feature args> [<feature arg>]* [<#core args> [key value]*]]
1490 *
1491 * metadata dev: Fast device holding the persistent metadata
1492 * destination dev: The destination device, which will become a clone of the
1493 * source device
1494 * source dev: The read-only source device that gets cloned
1495 * region size: dm-clone unit size in sectors
1496 *
1497 * #feature args: Number of feature arguments passed
1498 * feature args: E.g. no_hydration, no_discard_passdown
1499 *
1500 * #core arguments: An even number of core arguments
1501 * core arguments: Key/value pairs for tuning the core
1502 * E.g. 'hydration_threshold 256'
1503 */
1504static int parse_feature_args(struct dm_arg_set *as, struct clone *clone)
1505{
1506 int r;
1507 unsigned int argc;
1508 const char *arg_name;
1509 struct dm_target *ti = clone->ti;
1510
1511 const struct dm_arg args = {
1512 .min = 0,
1513 .max = 2,
1514 .error = "Invalid number of feature arguments"
1515 };
1516
1517 /* No feature arguments supplied */
1518 if (!as->argc)
1519 return 0;
1520
1521 r = dm_read_arg_group(&args, as, &argc, &ti->error);
1522 if (r)
1523 return r;
1524
1525 while (argc) {
1526 arg_name = dm_shift_arg(as);
1527 argc--;
1528
1529 if (!strcasecmp(arg_name, "no_hydration")) {
1530 __clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
1531 } else if (!strcasecmp(arg_name, "no_discard_passdown")) {
1532 __clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
1533 } else {
1534 ti->error = "Invalid feature argument";
1535 return -EINVAL;
1536 }
1537 }
1538
1539 return 0;
1540}
1541
1542static int parse_core_args(struct dm_arg_set *as, struct clone *clone)
1543{
1544 int r;
1545 unsigned int argc;
1546 unsigned int value;
1547 const char *arg_name;
1548 struct dm_target *ti = clone->ti;
1549
1550 const struct dm_arg args = {
1551 .min = 0,
1552 .max = 4,
1553 .error = "Invalid number of core arguments"
1554 };
1555
1556 /* Initialize core arguments */
1557 clone->hydration_batch_size = DEFAULT_HYDRATION_BATCH_SIZE;
1558 clone->hydration_threshold = DEFAULT_HYDRATION_THRESHOLD;
1559
1560 /* No core arguments supplied */
1561 if (!as->argc)
1562 return 0;
1563
1564 r = dm_read_arg_group(&args, as, &argc, &ti->error);
1565 if (r)
1566 return r;
1567
1568 if (argc & 1) {
1569 ti->error = "Number of core arguments must be even";
1570 return -EINVAL;
1571 }
1572
1573 while (argc) {
1574 arg_name = dm_shift_arg(as);
1575 argc -= 2;
1576
1577 if (!strcasecmp(arg_name, "hydration_threshold")) {
1578 if (kstrtouint(dm_shift_arg(as), 10, &value)) {
1579 ti->error = "Invalid value for argument `hydration_threshold'";
1580 return -EINVAL;
1581 }
1582 clone->hydration_threshold = value;
1583 } else if (!strcasecmp(arg_name, "hydration_batch_size")) {
1584 if (kstrtouint(dm_shift_arg(as), 10, &value)) {
1585 ti->error = "Invalid value for argument `hydration_batch_size'";
1586 return -EINVAL;
1587 }
1588 clone->hydration_batch_size = value;
1589 } else {
1590 ti->error = "Invalid core argument";
1591 return -EINVAL;
1592 }
1593 }
1594
1595 return 0;
1596}
1597
1598static int parse_region_size(struct clone *clone, struct dm_arg_set *as, char **error)
1599{
1600 int r;
1601 unsigned int region_size;
1602 struct dm_arg arg;
1603
1604 arg.min = MIN_REGION_SIZE;
1605 arg.max = MAX_REGION_SIZE;
1606 arg.error = "Invalid region size";
1607
1608 r = dm_read_arg(&arg, as, ®ion_size, error);
1609 if (r)
1610 return r;
1611
1612 /* Check region size is a power of 2 */
1613 if (!is_power_of_2(region_size)) {
1614 *error = "Region size is not a power of 2";
1615 return -EINVAL;
1616 }
1617
1618 /* Validate the region size against the device logical block size */
1619 if (region_size % (bdev_logical_block_size(clone->source_dev->bdev) >> 9) ||
1620 region_size % (bdev_logical_block_size(clone->dest_dev->bdev) >> 9)) {
1621 *error = "Region size is not a multiple of device logical block size";
1622 return -EINVAL;
1623 }
1624
1625 clone->region_size = region_size;
1626
1627 return 0;
1628}
1629
1630static int validate_nr_regions(unsigned long n, char **error)
1631{
1632 /*
1633 * dm_bitset restricts us to 2^32 regions. test_bit & co. restrict us
1634 * further to 2^31 regions.
1635 */
1636 if (n > (1UL << 31)) {
1637 *error = "Too many regions. Consider increasing the region size";
1638 return -EINVAL;
1639 }
1640
1641 return 0;
1642}
1643
1644static int parse_metadata_dev(struct clone *clone, struct dm_arg_set *as, char **error)
1645{
1646 int r;
1647 sector_t metadata_dev_size;
1648 char b[BDEVNAME_SIZE];
1649
1650 r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
1651 &clone->metadata_dev);
1652 if (r) {
1653 *error = "Error opening metadata device";
1654 return r;
1655 }
1656
1657 metadata_dev_size = get_dev_size(clone->metadata_dev);
1658 if (metadata_dev_size > DM_CLONE_METADATA_MAX_SECTORS_WARNING)
1659 DMWARN("Metadata device %s is larger than %u sectors: excess space will not be used.",
1660 bdevname(clone->metadata_dev->bdev, b), DM_CLONE_METADATA_MAX_SECTORS);
1661
1662 return 0;
1663}
1664
1665static int parse_dest_dev(struct clone *clone, struct dm_arg_set *as, char **error)
1666{
1667 int r;
1668 sector_t dest_dev_size;
1669
1670 r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ | FMODE_WRITE,
1671 &clone->dest_dev);
1672 if (r) {
1673 *error = "Error opening destination device";
1674 return r;
1675 }
1676
1677 dest_dev_size = get_dev_size(clone->dest_dev);
1678 if (dest_dev_size < clone->ti->len) {
1679 dm_put_device(clone->ti, clone->dest_dev);
1680 *error = "Device size larger than destination device";
1681 return -EINVAL;
1682 }
1683
1684 return 0;
1685}
1686
1687static int parse_source_dev(struct clone *clone, struct dm_arg_set *as, char **error)
1688{
1689 int r;
1690 sector_t source_dev_size;
1691
1692 r = dm_get_device(clone->ti, dm_shift_arg(as), FMODE_READ,
1693 &clone->source_dev);
1694 if (r) {
1695 *error = "Error opening source device";
1696 return r;
1697 }
1698
1699 source_dev_size = get_dev_size(clone->source_dev);
1700 if (source_dev_size < clone->ti->len) {
1701 dm_put_device(clone->ti, clone->source_dev);
1702 *error = "Device size larger than source device";
1703 return -EINVAL;
1704 }
1705
1706 return 0;
1707}
1708
1709static int copy_ctr_args(struct clone *clone, int argc, const char **argv, char **error)
1710{
1711 unsigned int i;
1712 const char **copy;
1713
1714 copy = kcalloc(argc, sizeof(*copy), GFP_KERNEL);
1715 if (!copy)
1716 goto error;
1717
1718 for (i = 0; i < argc; i++) {
1719 copy[i] = kstrdup(argv[i], GFP_KERNEL);
1720
1721 if (!copy[i]) {
1722 while (i--)
1723 kfree(copy[i]);
1724 kfree(copy);
1725 goto error;
1726 }
1727 }
1728
1729 clone->nr_ctr_args = argc;
1730 clone->ctr_args = copy;
1731 return 0;
1732
1733error:
1734 *error = "Failed to allocate memory for table line";
1735 return -ENOMEM;
1736}
1737
1738static int clone_ctr(struct dm_target *ti, unsigned int argc, char **argv)
1739{
1740 int r;
1741 struct clone *clone;
1742 struct dm_arg_set as;
1743
1744 if (argc < 4) {
1745 ti->error = "Invalid number of arguments";
1746 return -EINVAL;
1747 }
1748
1749 as.argc = argc;
1750 as.argv = argv;
1751
1752 clone = kzalloc(sizeof(*clone), GFP_KERNEL);
1753 if (!clone) {
1754 ti->error = "Failed to allocate clone structure";
1755 return -ENOMEM;
1756 }
1757
1758 clone->ti = ti;
1759
1760 /* Initialize dm-clone flags */
1761 __set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
1762 __set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
1763 __set_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
1764
1765 r = parse_metadata_dev(clone, &as, &ti->error);
1766 if (r)
1767 goto out_with_clone;
1768
1769 r = parse_dest_dev(clone, &as, &ti->error);
1770 if (r)
1771 goto out_with_meta_dev;
1772
1773 r = parse_source_dev(clone, &as, &ti->error);
1774 if (r)
1775 goto out_with_dest_dev;
1776
1777 r = parse_region_size(clone, &as, &ti->error);
1778 if (r)
1779 goto out_with_source_dev;
1780
1781 clone->region_shift = __ffs(clone->region_size);
1782 clone->nr_regions = dm_sector_div_up(ti->len, clone->region_size);
1783
1784 r = validate_nr_regions(clone->nr_regions, &ti->error);
1785 if (r)
1786 goto out_with_source_dev;
1787
1788 r = dm_set_target_max_io_len(ti, clone->region_size);
1789 if (r) {
1790 ti->error = "Failed to set max io len";
1791 goto out_with_source_dev;
1792 }
1793
1794 r = parse_feature_args(&as, clone);
1795 if (r)
1796 goto out_with_source_dev;
1797
1798 r = parse_core_args(&as, clone);
1799 if (r)
1800 goto out_with_source_dev;
1801
1802 /* Load metadata */
1803 clone->cmd = dm_clone_metadata_open(clone->metadata_dev->bdev, ti->len,
1804 clone->region_size);
1805 if (IS_ERR(clone->cmd)) {
1806 ti->error = "Failed to load metadata";
1807 r = PTR_ERR(clone->cmd);
1808 goto out_with_source_dev;
1809 }
1810
1811 __set_clone_mode(clone, CM_WRITE);
1812
1813 if (get_clone_mode(clone) != CM_WRITE) {
1814 ti->error = "Unable to get write access to metadata, please check/repair metadata";
1815 r = -EPERM;
1816 goto out_with_metadata;
1817 }
1818
1819 clone->last_commit_jiffies = jiffies;
1820
1821 /* Allocate hydration hash table */
1822 r = hash_table_init(clone);
1823 if (r) {
1824 ti->error = "Failed to allocate hydration hash table";
1825 goto out_with_metadata;
1826 }
1827
1828 atomic_set(&clone->ios_in_flight, 0);
1829 init_waitqueue_head(&clone->hydration_stopped);
1830 spin_lock_init(&clone->lock);
1831 bio_list_init(&clone->deferred_bios);
1832 bio_list_init(&clone->deferred_discard_bios);
1833 bio_list_init(&clone->deferred_flush_bios);
1834 bio_list_init(&clone->deferred_flush_completions);
1835 clone->hydration_offset = 0;
1836 atomic_set(&clone->hydrations_in_flight, 0);
1837
1838 clone->wq = alloc_workqueue("dm-" DM_MSG_PREFIX, WQ_MEM_RECLAIM, 0);
1839 if (!clone->wq) {
1840 ti->error = "Failed to allocate workqueue";
1841 r = -ENOMEM;
1842 goto out_with_ht;
1843 }
1844
1845 INIT_WORK(&clone->worker, do_worker);
1846 INIT_DELAYED_WORK(&clone->waker, do_waker);
1847
1848 clone->kcopyd_client = dm_kcopyd_client_create(&dm_kcopyd_throttle);
1849 if (IS_ERR(clone->kcopyd_client)) {
1850 r = PTR_ERR(clone->kcopyd_client);
1851 goto out_with_wq;
1852 }
1853
1854 r = mempool_init_slab_pool(&clone->hydration_pool, MIN_HYDRATIONS,
1855 _hydration_cache);
1856 if (r) {
1857 ti->error = "Failed to create dm_clone_region_hydration memory pool";
1858 goto out_with_kcopyd;
1859 }
1860
1861 /* Save a copy of the table line */
1862 r = copy_ctr_args(clone, argc - 3, (const char **)argv + 3, &ti->error);
1863 if (r)
1864 goto out_with_mempool;
1865
1866 mutex_init(&clone->commit_lock);
1867 clone->callbacks.congested_fn = clone_is_congested;
1868 dm_table_add_target_callbacks(ti->table, &clone->callbacks);
1869
1870 /* Enable flushes */
1871 ti->num_flush_bios = 1;
1872 ti->flush_supported = true;
1873
1874 /* Enable discards */
1875 ti->discards_supported = true;
1876 ti->num_discard_bios = 1;
1877
1878 ti->private = clone;
1879
1880 return 0;
1881
1882out_with_mempool:
1883 mempool_exit(&clone->hydration_pool);
1884out_with_kcopyd:
1885 dm_kcopyd_client_destroy(clone->kcopyd_client);
1886out_with_wq:
1887 destroy_workqueue(clone->wq);
1888out_with_ht:
1889 hash_table_exit(clone);
1890out_with_metadata:
1891 dm_clone_metadata_close(clone->cmd);
1892out_with_source_dev:
1893 dm_put_device(ti, clone->source_dev);
1894out_with_dest_dev:
1895 dm_put_device(ti, clone->dest_dev);
1896out_with_meta_dev:
1897 dm_put_device(ti, clone->metadata_dev);
1898out_with_clone:
1899 kfree(clone);
1900
1901 return r;
1902}
1903
1904static void clone_dtr(struct dm_target *ti)
1905{
1906 unsigned int i;
1907 struct clone *clone = ti->private;
1908
1909 mutex_destroy(&clone->commit_lock);
1910
1911 for (i = 0; i < clone->nr_ctr_args; i++)
1912 kfree(clone->ctr_args[i]);
1913 kfree(clone->ctr_args);
1914
1915 mempool_exit(&clone->hydration_pool);
1916 dm_kcopyd_client_destroy(clone->kcopyd_client);
1917 destroy_workqueue(clone->wq);
1918 hash_table_exit(clone);
1919 dm_clone_metadata_close(clone->cmd);
1920 dm_put_device(ti, clone->source_dev);
1921 dm_put_device(ti, clone->dest_dev);
1922 dm_put_device(ti, clone->metadata_dev);
1923
1924 kfree(clone);
1925}
1926
1927/*---------------------------------------------------------------------------*/
1928
1929static void clone_postsuspend(struct dm_target *ti)
1930{
1931 struct clone *clone = ti->private;
1932
1933 /*
1934 * To successfully suspend the device:
1935 *
1936 * - We cancel the delayed work for periodic commits and wait for
1937 * it to finish.
1938 *
1939 * - We stop the background hydration, i.e. we prevent new region
1940 * hydrations from starting.
1941 *
1942 * - We wait for any in-flight hydrations to finish.
1943 *
1944 * - We flush the workqueue.
1945 *
1946 * - We commit the metadata.
1947 */
1948 cancel_delayed_work_sync(&clone->waker);
1949
1950 set_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
1951
1952 /*
1953 * Make sure set_bit() is ordered before atomic_read(), otherwise we
1954 * might race with do_hydration() and miss some started region
1955 * hydrations.
1956 *
1957 * This is paired with smp_mb__after_atomic() in do_hydration().
1958 */
1959 smp_mb__after_atomic();
1960
1961 wait_event(clone->hydration_stopped, !atomic_read(&clone->hydrations_in_flight));
1962 flush_workqueue(clone->wq);
1963
1964 (void) commit_metadata(clone);
1965}
1966
1967static void clone_resume(struct dm_target *ti)
1968{
1969 struct clone *clone = ti->private;
1970
1971 clear_bit(DM_CLONE_HYDRATION_SUSPENDED, &clone->flags);
1972 do_waker(&clone->waker.work);
1973}
1974
1975static bool bdev_supports_discards(struct block_device *bdev)
1976{
1977 struct request_queue *q = bdev_get_queue(bdev);
1978
1979 return (q && blk_queue_discard(q));
1980}
1981
1982/*
1983 * If discard_passdown was enabled verify that the destination device supports
1984 * discards. Disable discard_passdown if not.
1985 */
1986static void disable_passdown_if_not_supported(struct clone *clone)
1987{
1988 struct block_device *dest_dev = clone->dest_dev->bdev;
1989 struct queue_limits *dest_limits = &bdev_get_queue(dest_dev)->limits;
1990 const char *reason = NULL;
1991 char buf[BDEVNAME_SIZE];
1992
1993 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags))
1994 return;
1995
1996 if (!bdev_supports_discards(dest_dev))
1997 reason = "discard unsupported";
1998 else if (dest_limits->max_discard_sectors < clone->region_size)
1999 reason = "max discard sectors smaller than a region";
2000
2001 if (reason) {
2002 DMWARN("Destination device (%s) %s: Disabling discard passdown.",
2003 bdevname(dest_dev, buf), reason);
2004 clear_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags);
2005 }
2006}
2007
2008static void set_discard_limits(struct clone *clone, struct queue_limits *limits)
2009{
2010 struct block_device *dest_bdev = clone->dest_dev->bdev;
2011 struct queue_limits *dest_limits = &bdev_get_queue(dest_bdev)->limits;
2012
2013 if (!test_bit(DM_CLONE_DISCARD_PASSDOWN, &clone->flags)) {
2014 /* No passdown is done so we set our own virtual limits */
2015 limits->discard_granularity = clone->region_size << SECTOR_SHIFT;
2016 limits->max_discard_sectors = round_down(UINT_MAX >> SECTOR_SHIFT, clone->region_size);
2017 return;
2018 }
2019
2020 /*
2021 * clone_iterate_devices() is stacking both the source and destination
2022 * device limits but discards aren't passed to the source device, so
2023 * inherit destination's limits.
2024 */
2025 limits->max_discard_sectors = dest_limits->max_discard_sectors;
2026 limits->max_hw_discard_sectors = dest_limits->max_hw_discard_sectors;
2027 limits->discard_granularity = dest_limits->discard_granularity;
2028 limits->discard_alignment = dest_limits->discard_alignment;
2029 limits->discard_misaligned = dest_limits->discard_misaligned;
2030 limits->max_discard_segments = dest_limits->max_discard_segments;
2031}
2032
2033static void clone_io_hints(struct dm_target *ti, struct queue_limits *limits)
2034{
2035 struct clone *clone = ti->private;
2036 u64 io_opt_sectors = limits->io_opt >> SECTOR_SHIFT;
2037
2038 /*
2039 * If the system-determined stacked limits are compatible with
2040 * dm-clone's region size (io_opt is a factor) do not override them.
2041 */
2042 if (io_opt_sectors < clone->region_size ||
2043 do_div(io_opt_sectors, clone->region_size)) {
2044 blk_limits_io_min(limits, clone->region_size << SECTOR_SHIFT);
2045 blk_limits_io_opt(limits, clone->region_size << SECTOR_SHIFT);
2046 }
2047
2048 disable_passdown_if_not_supported(clone);
2049 set_discard_limits(clone, limits);
2050}
2051
2052static int clone_iterate_devices(struct dm_target *ti,
2053 iterate_devices_callout_fn fn, void *data)
2054{
2055 int ret;
2056 struct clone *clone = ti->private;
2057 struct dm_dev *dest_dev = clone->dest_dev;
2058 struct dm_dev *source_dev = clone->source_dev;
2059
2060 ret = fn(ti, source_dev, 0, ti->len, data);
2061 if (!ret)
2062 ret = fn(ti, dest_dev, 0, ti->len, data);
2063 return ret;
2064}
2065
2066/*
2067 * dm-clone message functions.
2068 */
2069static void set_hydration_threshold(struct clone *clone, unsigned int nr_regions)
2070{
2071 WRITE_ONCE(clone->hydration_threshold, nr_regions);
2072
2073 /*
2074 * If user space sets hydration_threshold to zero then the hydration
2075 * will stop. If at a later time the hydration_threshold is increased
2076 * we must restart the hydration process by waking up the worker.
2077 */
2078 wake_worker(clone);
2079}
2080
2081static void set_hydration_batch_size(struct clone *clone, unsigned int nr_regions)
2082{
2083 WRITE_ONCE(clone->hydration_batch_size, nr_regions);
2084}
2085
2086static void enable_hydration(struct clone *clone)
2087{
2088 if (!test_and_set_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags))
2089 wake_worker(clone);
2090}
2091
2092static void disable_hydration(struct clone *clone)
2093{
2094 clear_bit(DM_CLONE_HYDRATION_ENABLED, &clone->flags);
2095}
2096
2097static int clone_message(struct dm_target *ti, unsigned int argc, char **argv,
2098 char *result, unsigned int maxlen)
2099{
2100 struct clone *clone = ti->private;
2101 unsigned int value;
2102
2103 if (!argc)
2104 return -EINVAL;
2105
2106 if (!strcasecmp(argv[0], "enable_hydration")) {
2107 enable_hydration(clone);
2108 return 0;
2109 }
2110
2111 if (!strcasecmp(argv[0], "disable_hydration")) {
2112 disable_hydration(clone);
2113 return 0;
2114 }
2115
2116 if (argc != 2)
2117 return -EINVAL;
2118
2119 if (!strcasecmp(argv[0], "hydration_threshold")) {
2120 if (kstrtouint(argv[1], 10, &value))
2121 return -EINVAL;
2122
2123 set_hydration_threshold(clone, value);
2124
2125 return 0;
2126 }
2127
2128 if (!strcasecmp(argv[0], "hydration_batch_size")) {
2129 if (kstrtouint(argv[1], 10, &value))
2130 return -EINVAL;
2131
2132 set_hydration_batch_size(clone, value);
2133
2134 return 0;
2135 }
2136
2137 DMERR("%s: Unsupported message `%s'", clone_device_name(clone), argv[0]);
2138 return -EINVAL;
2139}
2140
2141static struct target_type clone_target = {
2142 .name = "clone",
2143 .version = {1, 0, 0},
2144 .module = THIS_MODULE,
2145 .ctr = clone_ctr,
2146 .dtr = clone_dtr,
2147 .map = clone_map,
2148 .end_io = clone_endio,
2149 .postsuspend = clone_postsuspend,
2150 .resume = clone_resume,
2151 .status = clone_status,
2152 .message = clone_message,
2153 .io_hints = clone_io_hints,
2154 .iterate_devices = clone_iterate_devices,
2155};
2156
2157/*---------------------------------------------------------------------------*/
2158
2159/* Module functions */
2160static int __init dm_clone_init(void)
2161{
2162 int r;
2163
2164 _hydration_cache = KMEM_CACHE(dm_clone_region_hydration, 0);
2165 if (!_hydration_cache)
2166 return -ENOMEM;
2167
2168 r = dm_register_target(&clone_target);
2169 if (r < 0) {
2170 DMERR("Failed to register clone target");
2171 return r;
2172 }
2173
2174 return 0;
2175}
2176
2177static void __exit dm_clone_exit(void)
2178{
2179 dm_unregister_target(&clone_target);
2180
2181 kmem_cache_destroy(_hydration_cache);
2182 _hydration_cache = NULL;
2183}
2184
2185/* Module hooks */
2186module_init(dm_clone_init);
2187module_exit(dm_clone_exit);
2188
2189MODULE_DESCRIPTION(DM_NAME " clone target");
2190MODULE_AUTHOR("Nikos Tsironis <ntsironis@arrikto.com>");
2191MODULE_LICENSE("GPL");