Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2017 Western Digital Corporation or its affiliates.
4 *
5 * This file is released under the GPL.
6 */
7
8#include "dm-zoned.h"
9
10#include <linux/module.h>
11#include <linux/crc32.h>
12#include <linux/sched/mm.h>
13
14#define DM_MSG_PREFIX "zoned metadata"
15
16/*
17 * Metadata version.
18 */
19#define DMZ_META_VER 2
20
21/*
22 * On-disk super block magic.
23 */
24#define DMZ_MAGIC ((((unsigned int)('D')) << 24) | \
25 (((unsigned int)('Z')) << 16) | \
26 (((unsigned int)('B')) << 8) | \
27 ((unsigned int)('D')))
28
29/*
30 * On disk super block.
31 * This uses only 512 B but uses on disk a full 4KB block. This block is
32 * followed on disk by the mapping table of chunks to zones and the bitmap
33 * blocks indicating zone block validity.
34 * The overall resulting metadata format is:
35 * (1) Super block (1 block)
36 * (2) Chunk mapping table (nr_map_blocks)
37 * (3) Bitmap blocks (nr_bitmap_blocks)
38 * All metadata blocks are stored in conventional zones, starting from
39 * the first conventional zone found on disk.
40 */
41struct dmz_super {
42 /* Magic number */
43 __le32 magic; /* 4 */
44
45 /* Metadata version number */
46 __le32 version; /* 8 */
47
48 /* Generation number */
49 __le64 gen; /* 16 */
50
51 /* This block number */
52 __le64 sb_block; /* 24 */
53
54 /* The number of metadata blocks, including this super block */
55 __le32 nr_meta_blocks; /* 28 */
56
57 /* The number of sequential zones reserved for reclaim */
58 __le32 nr_reserved_seq; /* 32 */
59
60 /* The number of entries in the mapping table */
61 __le32 nr_chunks; /* 36 */
62
63 /* The number of blocks used for the chunk mapping table */
64 __le32 nr_map_blocks; /* 40 */
65
66 /* The number of blocks used for the block bitmaps */
67 __le32 nr_bitmap_blocks; /* 44 */
68
69 /* Checksum */
70 __le32 crc; /* 48 */
71
72 /* DM-Zoned label */
73 u8 dmz_label[32]; /* 80 */
74
75 /* DM-Zoned UUID */
76 u8 dmz_uuid[16]; /* 96 */
77
78 /* Device UUID */
79 u8 dev_uuid[16]; /* 112 */
80
81 /* Padding to full 512B sector */
82 u8 reserved[400]; /* 512 */
83};
84
85/*
86 * Chunk mapping entry: entries are indexed by chunk number
87 * and give the zone ID (dzone_id) mapping the chunk on disk.
88 * This zone may be sequential or random. If it is a sequential
89 * zone, a second zone (bzone_id) used as a write buffer may
90 * also be specified. This second zone will always be a randomly
91 * writeable zone.
92 */
93struct dmz_map {
94 __le32 dzone_id;
95 __le32 bzone_id;
96};
97
98/*
99 * Chunk mapping table metadata: 512 8-bytes entries per 4KB block.
100 */
101#define DMZ_MAP_ENTRIES (DMZ_BLOCK_SIZE / sizeof(struct dmz_map))
102#define DMZ_MAP_ENTRIES_SHIFT (ilog2(DMZ_MAP_ENTRIES))
103#define DMZ_MAP_ENTRIES_MASK (DMZ_MAP_ENTRIES - 1)
104#define DMZ_MAP_UNMAPPED UINT_MAX
105
106/*
107 * Meta data block descriptor (for cached metadata blocks).
108 */
109struct dmz_mblock {
110 struct rb_node node;
111 struct list_head link;
112 sector_t no;
113 unsigned int ref;
114 unsigned long state;
115 struct page *page;
116 void *data;
117};
118
119/*
120 * Metadata block state flags.
121 */
122enum {
123 DMZ_META_DIRTY,
124 DMZ_META_READING,
125 DMZ_META_WRITING,
126 DMZ_META_ERROR,
127};
128
129/*
130 * Super block information (one per metadata set).
131 */
132struct dmz_sb {
133 sector_t block;
134 struct dmz_dev *dev;
135 struct dmz_mblock *mblk;
136 struct dmz_super *sb;
137 struct dm_zone *zone;
138};
139
140/*
141 * In-memory metadata.
142 */
143struct dmz_metadata {
144 struct dmz_dev *dev;
145 unsigned int nr_devs;
146
147 char devname[BDEVNAME_SIZE];
148 char label[BDEVNAME_SIZE];
149 uuid_t uuid;
150
151 sector_t zone_bitmap_size;
152 unsigned int zone_nr_bitmap_blocks;
153 unsigned int zone_bits_per_mblk;
154
155 sector_t zone_nr_blocks;
156 sector_t zone_nr_blocks_shift;
157
158 sector_t zone_nr_sectors;
159 sector_t zone_nr_sectors_shift;
160
161 unsigned int nr_bitmap_blocks;
162 unsigned int nr_map_blocks;
163
164 unsigned int nr_zones;
165 unsigned int nr_useable_zones;
166 unsigned int nr_meta_blocks;
167 unsigned int nr_meta_zones;
168 unsigned int nr_data_zones;
169 unsigned int nr_cache_zones;
170 unsigned int nr_rnd_zones;
171 unsigned int nr_reserved_seq;
172 unsigned int nr_chunks;
173
174 /* Zone information array */
175 struct xarray zones;
176
177 struct dmz_sb sb[2];
178 unsigned int mblk_primary;
179 unsigned int sb_version;
180 u64 sb_gen;
181 unsigned int min_nr_mblks;
182 unsigned int max_nr_mblks;
183 atomic_t nr_mblks;
184 struct rw_semaphore mblk_sem;
185 struct mutex mblk_flush_lock;
186 spinlock_t mblk_lock;
187 struct rb_root mblk_rbtree;
188 struct list_head mblk_lru_list;
189 struct list_head mblk_dirty_list;
190 struct shrinker *mblk_shrinker;
191
192 /* Zone allocation management */
193 struct mutex map_lock;
194 struct dmz_mblock **map_mblk;
195
196 unsigned int nr_cache;
197 atomic_t unmap_nr_cache;
198 struct list_head unmap_cache_list;
199 struct list_head map_cache_list;
200
201 atomic_t nr_reserved_seq_zones;
202 struct list_head reserved_seq_zones_list;
203
204 wait_queue_head_t free_wq;
205};
206
207#define dmz_zmd_info(zmd, format, args...) \
208 DMINFO("(%s): " format, (zmd)->label, ## args)
209
210#define dmz_zmd_err(zmd, format, args...) \
211 DMERR("(%s): " format, (zmd)->label, ## args)
212
213#define dmz_zmd_warn(zmd, format, args...) \
214 DMWARN("(%s): " format, (zmd)->label, ## args)
215
216#define dmz_zmd_debug(zmd, format, args...) \
217 DMDEBUG("(%s): " format, (zmd)->label, ## args)
218/*
219 * Various accessors
220 */
221static unsigned int dmz_dev_zone_id(struct dmz_metadata *zmd, struct dm_zone *zone)
222{
223 if (WARN_ON(!zone))
224 return 0;
225
226 return zone->id - zone->dev->zone_offset;
227}
228
229sector_t dmz_start_sect(struct dmz_metadata *zmd, struct dm_zone *zone)
230{
231 unsigned int zone_id = dmz_dev_zone_id(zmd, zone);
232
233 return (sector_t)zone_id << zmd->zone_nr_sectors_shift;
234}
235
236sector_t dmz_start_block(struct dmz_metadata *zmd, struct dm_zone *zone)
237{
238 unsigned int zone_id = dmz_dev_zone_id(zmd, zone);
239
240 return (sector_t)zone_id << zmd->zone_nr_blocks_shift;
241}
242
243unsigned int dmz_zone_nr_blocks(struct dmz_metadata *zmd)
244{
245 return zmd->zone_nr_blocks;
246}
247
248unsigned int dmz_zone_nr_sectors(struct dmz_metadata *zmd)
249{
250 return zmd->zone_nr_sectors;
251}
252
253unsigned int dmz_zone_nr_sectors_shift(struct dmz_metadata *zmd)
254{
255 return zmd->zone_nr_sectors_shift;
256}
257
258unsigned int dmz_nr_zones(struct dmz_metadata *zmd)
259{
260 return zmd->nr_zones;
261}
262
263unsigned int dmz_nr_chunks(struct dmz_metadata *zmd)
264{
265 return zmd->nr_chunks;
266}
267
268unsigned int dmz_nr_rnd_zones(struct dmz_metadata *zmd, int idx)
269{
270 return zmd->dev[idx].nr_rnd;
271}
272
273unsigned int dmz_nr_unmap_rnd_zones(struct dmz_metadata *zmd, int idx)
274{
275 return atomic_read(&zmd->dev[idx].unmap_nr_rnd);
276}
277
278unsigned int dmz_nr_cache_zones(struct dmz_metadata *zmd)
279{
280 return zmd->nr_cache;
281}
282
283unsigned int dmz_nr_unmap_cache_zones(struct dmz_metadata *zmd)
284{
285 return atomic_read(&zmd->unmap_nr_cache);
286}
287
288unsigned int dmz_nr_seq_zones(struct dmz_metadata *zmd, int idx)
289{
290 return zmd->dev[idx].nr_seq;
291}
292
293unsigned int dmz_nr_unmap_seq_zones(struct dmz_metadata *zmd, int idx)
294{
295 return atomic_read(&zmd->dev[idx].unmap_nr_seq);
296}
297
298static struct dm_zone *dmz_get(struct dmz_metadata *zmd, unsigned int zone_id)
299{
300 return xa_load(&zmd->zones, zone_id);
301}
302
303static struct dm_zone *dmz_insert(struct dmz_metadata *zmd,
304 unsigned int zone_id, struct dmz_dev *dev)
305{
306 struct dm_zone *zone = kzalloc(sizeof(struct dm_zone), GFP_KERNEL);
307
308 if (!zone)
309 return ERR_PTR(-ENOMEM);
310
311 if (xa_insert(&zmd->zones, zone_id, zone, GFP_KERNEL)) {
312 kfree(zone);
313 return ERR_PTR(-EBUSY);
314 }
315
316 INIT_LIST_HEAD(&zone->link);
317 atomic_set(&zone->refcount, 0);
318 zone->id = zone_id;
319 zone->chunk = DMZ_MAP_UNMAPPED;
320 zone->dev = dev;
321
322 return zone;
323}
324
325const char *dmz_metadata_label(struct dmz_metadata *zmd)
326{
327 return (const char *)zmd->label;
328}
329
330bool dmz_check_dev(struct dmz_metadata *zmd)
331{
332 unsigned int i;
333
334 for (i = 0; i < zmd->nr_devs; i++) {
335 if (!dmz_check_bdev(&zmd->dev[i]))
336 return false;
337 }
338 return true;
339}
340
341bool dmz_dev_is_dying(struct dmz_metadata *zmd)
342{
343 unsigned int i;
344
345 for (i = 0; i < zmd->nr_devs; i++) {
346 if (dmz_bdev_is_dying(&zmd->dev[i]))
347 return true;
348 }
349 return false;
350}
351
352/*
353 * Lock/unlock mapping table.
354 * The map lock also protects all the zone lists.
355 */
356void dmz_lock_map(struct dmz_metadata *zmd)
357{
358 mutex_lock(&zmd->map_lock);
359}
360
361void dmz_unlock_map(struct dmz_metadata *zmd)
362{
363 mutex_unlock(&zmd->map_lock);
364}
365
366/*
367 * Lock/unlock metadata access. This is a "read" lock on a semaphore
368 * that prevents metadata flush from running while metadata are being
369 * modified. The actual metadata write mutual exclusion is achieved with
370 * the map lock and zone state management (active and reclaim state are
371 * mutually exclusive).
372 */
373void dmz_lock_metadata(struct dmz_metadata *zmd)
374{
375 down_read(&zmd->mblk_sem);
376}
377
378void dmz_unlock_metadata(struct dmz_metadata *zmd)
379{
380 up_read(&zmd->mblk_sem);
381}
382
383/*
384 * Lock/unlock flush: prevent concurrent executions
385 * of dmz_flush_metadata as well as metadata modification in reclaim
386 * while flush is being executed.
387 */
388void dmz_lock_flush(struct dmz_metadata *zmd)
389{
390 mutex_lock(&zmd->mblk_flush_lock);
391}
392
393void dmz_unlock_flush(struct dmz_metadata *zmd)
394{
395 mutex_unlock(&zmd->mblk_flush_lock);
396}
397
398/*
399 * Allocate a metadata block.
400 */
401static struct dmz_mblock *dmz_alloc_mblock(struct dmz_metadata *zmd,
402 sector_t mblk_no)
403{
404 struct dmz_mblock *mblk = NULL;
405
406 /* See if we can reuse cached blocks */
407 if (zmd->max_nr_mblks && atomic_read(&zmd->nr_mblks) > zmd->max_nr_mblks) {
408 spin_lock(&zmd->mblk_lock);
409 mblk = list_first_entry_or_null(&zmd->mblk_lru_list,
410 struct dmz_mblock, link);
411 if (mblk) {
412 list_del_init(&mblk->link);
413 rb_erase(&mblk->node, &zmd->mblk_rbtree);
414 mblk->no = mblk_no;
415 }
416 spin_unlock(&zmd->mblk_lock);
417 if (mblk)
418 return mblk;
419 }
420
421 /* Allocate a new block */
422 mblk = kmalloc(sizeof(struct dmz_mblock), GFP_NOIO);
423 if (!mblk)
424 return NULL;
425
426 mblk->page = alloc_page(GFP_NOIO);
427 if (!mblk->page) {
428 kfree(mblk);
429 return NULL;
430 }
431
432 RB_CLEAR_NODE(&mblk->node);
433 INIT_LIST_HEAD(&mblk->link);
434 mblk->ref = 0;
435 mblk->state = 0;
436 mblk->no = mblk_no;
437 mblk->data = page_address(mblk->page);
438
439 atomic_inc(&zmd->nr_mblks);
440
441 return mblk;
442}
443
444/*
445 * Free a metadata block.
446 */
447static void dmz_free_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
448{
449 __free_pages(mblk->page, 0);
450 kfree(mblk);
451
452 atomic_dec(&zmd->nr_mblks);
453}
454
455/*
456 * Insert a metadata block in the rbtree.
457 */
458static void dmz_insert_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
459{
460 struct rb_root *root = &zmd->mblk_rbtree;
461 struct rb_node **new = &(root->rb_node), *parent = NULL;
462 struct dmz_mblock *b;
463
464 /* Figure out where to put the new node */
465 while (*new) {
466 b = container_of(*new, struct dmz_mblock, node);
467 parent = *new;
468 new = (b->no < mblk->no) ? &((*new)->rb_left) : &((*new)->rb_right);
469 }
470
471 /* Add new node and rebalance tree */
472 rb_link_node(&mblk->node, parent, new);
473 rb_insert_color(&mblk->node, root);
474}
475
476/*
477 * Lookup a metadata block in the rbtree. If the block is found, increment
478 * its reference count.
479 */
480static struct dmz_mblock *dmz_get_mblock_fast(struct dmz_metadata *zmd,
481 sector_t mblk_no)
482{
483 struct rb_root *root = &zmd->mblk_rbtree;
484 struct rb_node *node = root->rb_node;
485 struct dmz_mblock *mblk;
486
487 while (node) {
488 mblk = container_of(node, struct dmz_mblock, node);
489 if (mblk->no == mblk_no) {
490 /*
491 * If this is the first reference to the block,
492 * remove it from the LRU list.
493 */
494 mblk->ref++;
495 if (mblk->ref == 1 &&
496 !test_bit(DMZ_META_DIRTY, &mblk->state))
497 list_del_init(&mblk->link);
498 return mblk;
499 }
500 node = (mblk->no < mblk_no) ? node->rb_left : node->rb_right;
501 }
502
503 return NULL;
504}
505
506/*
507 * Metadata block BIO end callback.
508 */
509static void dmz_mblock_bio_end_io(struct bio *bio)
510{
511 struct dmz_mblock *mblk = bio->bi_private;
512 int flag;
513
514 if (bio->bi_status)
515 set_bit(DMZ_META_ERROR, &mblk->state);
516
517 if (bio_op(bio) == REQ_OP_WRITE)
518 flag = DMZ_META_WRITING;
519 else
520 flag = DMZ_META_READING;
521
522 clear_bit_unlock(flag, &mblk->state);
523 smp_mb__after_atomic();
524 wake_up_bit(&mblk->state, flag);
525
526 bio_put(bio);
527}
528
529/*
530 * Read an uncached metadata block from disk and add it to the cache.
531 */
532static struct dmz_mblock *dmz_get_mblock_slow(struct dmz_metadata *zmd,
533 sector_t mblk_no)
534{
535 struct dmz_mblock *mblk, *m;
536 sector_t block = zmd->sb[zmd->mblk_primary].block + mblk_no;
537 struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;
538 struct bio *bio;
539
540 if (dmz_bdev_is_dying(dev))
541 return ERR_PTR(-EIO);
542
543 /* Get a new block and a BIO to read it */
544 mblk = dmz_alloc_mblock(zmd, mblk_no);
545 if (!mblk)
546 return ERR_PTR(-ENOMEM);
547
548 bio = bio_alloc(dev->bdev, 1, REQ_OP_READ | REQ_META | REQ_PRIO,
549 GFP_NOIO);
550
551 spin_lock(&zmd->mblk_lock);
552
553 /*
554 * Make sure that another context did not start reading
555 * the block already.
556 */
557 m = dmz_get_mblock_fast(zmd, mblk_no);
558 if (m) {
559 spin_unlock(&zmd->mblk_lock);
560 dmz_free_mblock(zmd, mblk);
561 bio_put(bio);
562 return m;
563 }
564
565 mblk->ref++;
566 set_bit(DMZ_META_READING, &mblk->state);
567 dmz_insert_mblock(zmd, mblk);
568
569 spin_unlock(&zmd->mblk_lock);
570
571 /* Submit read BIO */
572 bio->bi_iter.bi_sector = dmz_blk2sect(block);
573 bio->bi_private = mblk;
574 bio->bi_end_io = dmz_mblock_bio_end_io;
575 __bio_add_page(bio, mblk->page, DMZ_BLOCK_SIZE, 0);
576 submit_bio(bio);
577
578 return mblk;
579}
580
581/*
582 * Free metadata blocks.
583 */
584static unsigned long dmz_shrink_mblock_cache(struct dmz_metadata *zmd,
585 unsigned long limit)
586{
587 struct dmz_mblock *mblk;
588 unsigned long count = 0;
589
590 if (!zmd->max_nr_mblks)
591 return 0;
592
593 while (!list_empty(&zmd->mblk_lru_list) &&
594 atomic_read(&zmd->nr_mblks) > zmd->min_nr_mblks &&
595 count < limit) {
596 mblk = list_first_entry(&zmd->mblk_lru_list,
597 struct dmz_mblock, link);
598 list_del_init(&mblk->link);
599 rb_erase(&mblk->node, &zmd->mblk_rbtree);
600 dmz_free_mblock(zmd, mblk);
601 count++;
602 }
603
604 return count;
605}
606
607/*
608 * For mblock shrinker: get the number of unused metadata blocks in the cache.
609 */
610static unsigned long dmz_mblock_shrinker_count(struct shrinker *shrink,
611 struct shrink_control *sc)
612{
613 struct dmz_metadata *zmd = shrink->private_data;
614
615 return atomic_read(&zmd->nr_mblks);
616}
617
618/*
619 * For mblock shrinker: scan unused metadata blocks and shrink the cache.
620 */
621static unsigned long dmz_mblock_shrinker_scan(struct shrinker *shrink,
622 struct shrink_control *sc)
623{
624 struct dmz_metadata *zmd = shrink->private_data;
625 unsigned long count;
626
627 spin_lock(&zmd->mblk_lock);
628 count = dmz_shrink_mblock_cache(zmd, sc->nr_to_scan);
629 spin_unlock(&zmd->mblk_lock);
630
631 return count ? count : SHRINK_STOP;
632}
633
634/*
635 * Release a metadata block.
636 */
637static void dmz_release_mblock(struct dmz_metadata *zmd,
638 struct dmz_mblock *mblk)
639{
640
641 if (!mblk)
642 return;
643
644 spin_lock(&zmd->mblk_lock);
645
646 mblk->ref--;
647 if (mblk->ref == 0) {
648 if (test_bit(DMZ_META_ERROR, &mblk->state)) {
649 rb_erase(&mblk->node, &zmd->mblk_rbtree);
650 dmz_free_mblock(zmd, mblk);
651 } else if (!test_bit(DMZ_META_DIRTY, &mblk->state)) {
652 list_add_tail(&mblk->link, &zmd->mblk_lru_list);
653 dmz_shrink_mblock_cache(zmd, 1);
654 }
655 }
656
657 spin_unlock(&zmd->mblk_lock);
658}
659
660/*
661 * Get a metadata block from the rbtree. If the block
662 * is not present, read it from disk.
663 */
664static struct dmz_mblock *dmz_get_mblock(struct dmz_metadata *zmd,
665 sector_t mblk_no)
666{
667 struct dmz_mblock *mblk;
668 struct dmz_dev *dev = zmd->sb[zmd->mblk_primary].dev;
669
670 /* Check rbtree */
671 spin_lock(&zmd->mblk_lock);
672 mblk = dmz_get_mblock_fast(zmd, mblk_no);
673 spin_unlock(&zmd->mblk_lock);
674
675 if (!mblk) {
676 /* Cache miss: read the block from disk */
677 mblk = dmz_get_mblock_slow(zmd, mblk_no);
678 if (IS_ERR(mblk))
679 return mblk;
680 }
681
682 /* Wait for on-going read I/O and check for error */
683 wait_on_bit_io(&mblk->state, DMZ_META_READING,
684 TASK_UNINTERRUPTIBLE);
685 if (test_bit(DMZ_META_ERROR, &mblk->state)) {
686 dmz_release_mblock(zmd, mblk);
687 dmz_check_bdev(dev);
688 return ERR_PTR(-EIO);
689 }
690
691 return mblk;
692}
693
694/*
695 * Mark a metadata block dirty.
696 */
697static void dmz_dirty_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk)
698{
699 spin_lock(&zmd->mblk_lock);
700 if (!test_and_set_bit(DMZ_META_DIRTY, &mblk->state))
701 list_add_tail(&mblk->link, &zmd->mblk_dirty_list);
702 spin_unlock(&zmd->mblk_lock);
703}
704
705/*
706 * Issue a metadata block write BIO.
707 */
708static int dmz_write_mblock(struct dmz_metadata *zmd, struct dmz_mblock *mblk,
709 unsigned int set)
710{
711 struct dmz_dev *dev = zmd->sb[set].dev;
712 sector_t block = zmd->sb[set].block + mblk->no;
713 struct bio *bio;
714
715 if (dmz_bdev_is_dying(dev))
716 return -EIO;
717
718 bio = bio_alloc(dev->bdev, 1, REQ_OP_WRITE | REQ_META | REQ_PRIO,
719 GFP_NOIO);
720
721 set_bit(DMZ_META_WRITING, &mblk->state);
722
723 bio->bi_iter.bi_sector = dmz_blk2sect(block);
724 bio->bi_private = mblk;
725 bio->bi_end_io = dmz_mblock_bio_end_io;
726 __bio_add_page(bio, mblk->page, DMZ_BLOCK_SIZE, 0);
727 submit_bio(bio);
728
729 return 0;
730}
731
732/*
733 * Read/write a metadata block.
734 */
735static int dmz_rdwr_block(struct dmz_dev *dev, enum req_op op,
736 sector_t block, struct page *page)
737{
738 struct bio *bio;
739 int ret;
740
741 if (WARN_ON(!dev))
742 return -EIO;
743
744 if (dmz_bdev_is_dying(dev))
745 return -EIO;
746
747 bio = bio_alloc(dev->bdev, 1, op | REQ_SYNC | REQ_META | REQ_PRIO,
748 GFP_NOIO);
749 bio->bi_iter.bi_sector = dmz_blk2sect(block);
750 __bio_add_page(bio, page, DMZ_BLOCK_SIZE, 0);
751 ret = submit_bio_wait(bio);
752 bio_put(bio);
753
754 if (ret)
755 dmz_check_bdev(dev);
756 return ret;
757}
758
759/*
760 * Write super block of the specified metadata set.
761 */
762static int dmz_write_sb(struct dmz_metadata *zmd, unsigned int set)
763{
764 struct dmz_mblock *mblk = zmd->sb[set].mblk;
765 struct dmz_super *sb = zmd->sb[set].sb;
766 struct dmz_dev *dev = zmd->sb[set].dev;
767 sector_t sb_block;
768 u64 sb_gen = zmd->sb_gen + 1;
769 int ret;
770
771 sb->magic = cpu_to_le32(DMZ_MAGIC);
772
773 sb->version = cpu_to_le32(zmd->sb_version);
774 if (zmd->sb_version > 1) {
775 BUILD_BUG_ON(UUID_SIZE != 16);
776 export_uuid(sb->dmz_uuid, &zmd->uuid);
777 memcpy(sb->dmz_label, zmd->label, BDEVNAME_SIZE);
778 export_uuid(sb->dev_uuid, &dev->uuid);
779 }
780
781 sb->gen = cpu_to_le64(sb_gen);
782
783 /*
784 * The metadata always references the absolute block address,
785 * ie relative to the entire block range, not the per-device
786 * block address.
787 */
788 sb_block = zmd->sb[set].zone->id << zmd->zone_nr_blocks_shift;
789 sb->sb_block = cpu_to_le64(sb_block);
790 sb->nr_meta_blocks = cpu_to_le32(zmd->nr_meta_blocks);
791 sb->nr_reserved_seq = cpu_to_le32(zmd->nr_reserved_seq);
792 sb->nr_chunks = cpu_to_le32(zmd->nr_chunks);
793
794 sb->nr_map_blocks = cpu_to_le32(zmd->nr_map_blocks);
795 sb->nr_bitmap_blocks = cpu_to_le32(zmd->nr_bitmap_blocks);
796
797 sb->crc = 0;
798 sb->crc = cpu_to_le32(crc32_le(sb_gen, (unsigned char *)sb, DMZ_BLOCK_SIZE));
799
800 ret = dmz_rdwr_block(dev, REQ_OP_WRITE, zmd->sb[set].block,
801 mblk->page);
802 if (ret == 0)
803 ret = blkdev_issue_flush(dev->bdev);
804
805 return ret;
806}
807
808/*
809 * Write dirty metadata blocks to the specified set.
810 */
811static int dmz_write_dirty_mblocks(struct dmz_metadata *zmd,
812 struct list_head *write_list,
813 unsigned int set)
814{
815 struct dmz_mblock *mblk;
816 struct dmz_dev *dev = zmd->sb[set].dev;
817 struct blk_plug plug;
818 int ret = 0, nr_mblks_submitted = 0;
819
820 /* Issue writes */
821 blk_start_plug(&plug);
822 list_for_each_entry(mblk, write_list, link) {
823 ret = dmz_write_mblock(zmd, mblk, set);
824 if (ret)
825 break;
826 nr_mblks_submitted++;
827 }
828 blk_finish_plug(&plug);
829
830 /* Wait for completion */
831 list_for_each_entry(mblk, write_list, link) {
832 if (!nr_mblks_submitted)
833 break;
834 wait_on_bit_io(&mblk->state, DMZ_META_WRITING,
835 TASK_UNINTERRUPTIBLE);
836 if (test_bit(DMZ_META_ERROR, &mblk->state)) {
837 clear_bit(DMZ_META_ERROR, &mblk->state);
838 dmz_check_bdev(dev);
839 ret = -EIO;
840 }
841 nr_mblks_submitted--;
842 }
843
844 /* Flush drive cache (this will also sync data) */
845 if (ret == 0)
846 ret = blkdev_issue_flush(dev->bdev);
847
848 return ret;
849}
850
851/*
852 * Log dirty metadata blocks.
853 */
854static int dmz_log_dirty_mblocks(struct dmz_metadata *zmd,
855 struct list_head *write_list)
856{
857 unsigned int log_set = zmd->mblk_primary ^ 0x1;
858 int ret;
859
860 /* Write dirty blocks to the log */
861 ret = dmz_write_dirty_mblocks(zmd, write_list, log_set);
862 if (ret)
863 return ret;
864
865 /*
866 * No error so far: now validate the log by updating the
867 * log index super block generation.
868 */
869 ret = dmz_write_sb(zmd, log_set);
870 if (ret)
871 return ret;
872
873 return 0;
874}
875
876/*
877 * Flush dirty metadata blocks.
878 */
879int dmz_flush_metadata(struct dmz_metadata *zmd)
880{
881 struct dmz_mblock *mblk;
882 struct list_head write_list;
883 struct dmz_dev *dev;
884 int ret;
885
886 if (WARN_ON(!zmd))
887 return 0;
888
889 INIT_LIST_HEAD(&write_list);
890
891 /*
892 * Make sure that metadata blocks are stable before logging: take
893 * the write lock on the metadata semaphore to prevent target BIOs
894 * from modifying metadata.
895 */
896 down_write(&zmd->mblk_sem);
897 dev = zmd->sb[zmd->mblk_primary].dev;
898
899 /*
900 * This is called from the target flush work and reclaim work.
901 * Concurrent execution is not allowed.
902 */
903 dmz_lock_flush(zmd);
904
905 if (dmz_bdev_is_dying(dev)) {
906 ret = -EIO;
907 goto out;
908 }
909
910 /* Get dirty blocks */
911 spin_lock(&zmd->mblk_lock);
912 list_splice_init(&zmd->mblk_dirty_list, &write_list);
913 spin_unlock(&zmd->mblk_lock);
914
915 /* If there are no dirty metadata blocks, just flush the device cache */
916 if (list_empty(&write_list)) {
917 ret = blkdev_issue_flush(dev->bdev);
918 goto err;
919 }
920
921 /*
922 * The primary metadata set is still clean. Keep it this way until
923 * all updates are successful in the secondary set. That is, use
924 * the secondary set as a log.
925 */
926 ret = dmz_log_dirty_mblocks(zmd, &write_list);
927 if (ret)
928 goto err;
929
930 /*
931 * The log is on disk. It is now safe to update in place
932 * in the primary metadata set.
933 */
934 ret = dmz_write_dirty_mblocks(zmd, &write_list, zmd->mblk_primary);
935 if (ret)
936 goto err;
937
938 ret = dmz_write_sb(zmd, zmd->mblk_primary);
939 if (ret)
940 goto err;
941
942 while (!list_empty(&write_list)) {
943 mblk = list_first_entry(&write_list, struct dmz_mblock, link);
944 list_del_init(&mblk->link);
945
946 spin_lock(&zmd->mblk_lock);
947 clear_bit(DMZ_META_DIRTY, &mblk->state);
948 if (mblk->ref == 0)
949 list_add_tail(&mblk->link, &zmd->mblk_lru_list);
950 spin_unlock(&zmd->mblk_lock);
951 }
952
953 zmd->sb_gen++;
954out:
955 dmz_unlock_flush(zmd);
956 up_write(&zmd->mblk_sem);
957
958 return ret;
959
960err:
961 if (!list_empty(&write_list)) {
962 spin_lock(&zmd->mblk_lock);
963 list_splice(&write_list, &zmd->mblk_dirty_list);
964 spin_unlock(&zmd->mblk_lock);
965 }
966 if (!dmz_check_bdev(dev))
967 ret = -EIO;
968 goto out;
969}
970
971/*
972 * Check super block.
973 */
974static int dmz_check_sb(struct dmz_metadata *zmd, struct dmz_sb *dsb,
975 bool tertiary)
976{
977 struct dmz_super *sb = dsb->sb;
978 struct dmz_dev *dev = dsb->dev;
979 unsigned int nr_meta_zones, nr_data_zones;
980 u32 crc, stored_crc;
981 u64 gen, sb_block;
982
983 if (le32_to_cpu(sb->magic) != DMZ_MAGIC) {
984 dmz_dev_err(dev, "Invalid meta magic (needed 0x%08x, got 0x%08x)",
985 DMZ_MAGIC, le32_to_cpu(sb->magic));
986 return -ENXIO;
987 }
988
989 zmd->sb_version = le32_to_cpu(sb->version);
990 if (zmd->sb_version > DMZ_META_VER) {
991 dmz_dev_err(dev, "Invalid meta version (needed %d, got %d)",
992 DMZ_META_VER, zmd->sb_version);
993 return -EINVAL;
994 }
995 if (zmd->sb_version < 2 && tertiary) {
996 dmz_dev_err(dev, "Tertiary superblocks are not supported");
997 return -EINVAL;
998 }
999
1000 gen = le64_to_cpu(sb->gen);
1001 stored_crc = le32_to_cpu(sb->crc);
1002 sb->crc = 0;
1003 crc = crc32_le(gen, (unsigned char *)sb, DMZ_BLOCK_SIZE);
1004 if (crc != stored_crc) {
1005 dmz_dev_err(dev, "Invalid checksum (needed 0x%08x, got 0x%08x)",
1006 crc, stored_crc);
1007 return -ENXIO;
1008 }
1009
1010 sb_block = le64_to_cpu(sb->sb_block);
1011 if (sb_block != (u64)dsb->zone->id << zmd->zone_nr_blocks_shift) {
1012 dmz_dev_err(dev, "Invalid superblock position (is %llu expected %llu)",
1013 sb_block, (u64)dsb->zone->id << zmd->zone_nr_blocks_shift);
1014 return -EINVAL;
1015 }
1016 if (zmd->sb_version > 1) {
1017 uuid_t sb_uuid;
1018
1019 import_uuid(&sb_uuid, sb->dmz_uuid);
1020 if (uuid_is_null(&sb_uuid)) {
1021 dmz_dev_err(dev, "NULL DM-Zoned uuid");
1022 return -ENXIO;
1023 } else if (uuid_is_null(&zmd->uuid)) {
1024 uuid_copy(&zmd->uuid, &sb_uuid);
1025 } else if (!uuid_equal(&zmd->uuid, &sb_uuid)) {
1026 dmz_dev_err(dev, "mismatching DM-Zoned uuid, is %pUl expected %pUl",
1027 &sb_uuid, &zmd->uuid);
1028 return -ENXIO;
1029 }
1030 if (!strlen(zmd->label))
1031 memcpy(zmd->label, sb->dmz_label, BDEVNAME_SIZE);
1032 else if (memcmp(zmd->label, sb->dmz_label, BDEVNAME_SIZE)) {
1033 dmz_dev_err(dev, "mismatching DM-Zoned label, is %s expected %s",
1034 sb->dmz_label, zmd->label);
1035 return -ENXIO;
1036 }
1037 import_uuid(&dev->uuid, sb->dev_uuid);
1038 if (uuid_is_null(&dev->uuid)) {
1039 dmz_dev_err(dev, "NULL device uuid");
1040 return -ENXIO;
1041 }
1042
1043 if (tertiary) {
1044 /*
1045 * Generation number should be 0, but it doesn't
1046 * really matter if it isn't.
1047 */
1048 if (gen != 0)
1049 dmz_dev_warn(dev, "Invalid generation %llu",
1050 gen);
1051 return 0;
1052 }
1053 }
1054
1055 nr_meta_zones = (le32_to_cpu(sb->nr_meta_blocks) + zmd->zone_nr_blocks - 1)
1056 >> zmd->zone_nr_blocks_shift;
1057 if (!nr_meta_zones ||
1058 (zmd->nr_devs <= 1 && nr_meta_zones >= zmd->nr_rnd_zones) ||
1059 (zmd->nr_devs > 1 && nr_meta_zones >= zmd->nr_cache_zones)) {
1060 dmz_dev_err(dev, "Invalid number of metadata blocks");
1061 return -ENXIO;
1062 }
1063
1064 if (!le32_to_cpu(sb->nr_reserved_seq) ||
1065 le32_to_cpu(sb->nr_reserved_seq) >= (zmd->nr_useable_zones - nr_meta_zones)) {
1066 dmz_dev_err(dev, "Invalid number of reserved sequential zones");
1067 return -ENXIO;
1068 }
1069
1070 nr_data_zones = zmd->nr_useable_zones -
1071 (nr_meta_zones * 2 + le32_to_cpu(sb->nr_reserved_seq));
1072 if (le32_to_cpu(sb->nr_chunks) > nr_data_zones) {
1073 dmz_dev_err(dev, "Invalid number of chunks %u / %u",
1074 le32_to_cpu(sb->nr_chunks), nr_data_zones);
1075 return -ENXIO;
1076 }
1077
1078 /* OK */
1079 zmd->nr_meta_blocks = le32_to_cpu(sb->nr_meta_blocks);
1080 zmd->nr_reserved_seq = le32_to_cpu(sb->nr_reserved_seq);
1081 zmd->nr_chunks = le32_to_cpu(sb->nr_chunks);
1082 zmd->nr_map_blocks = le32_to_cpu(sb->nr_map_blocks);
1083 zmd->nr_bitmap_blocks = le32_to_cpu(sb->nr_bitmap_blocks);
1084 zmd->nr_meta_zones = nr_meta_zones;
1085 zmd->nr_data_zones = nr_data_zones;
1086
1087 return 0;
1088}
1089
1090/*
1091 * Read the first or second super block from disk.
1092 */
1093static int dmz_read_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)
1094{
1095 dmz_zmd_debug(zmd, "read superblock set %d dev %pg block %llu",
1096 set, sb->dev->bdev, sb->block);
1097
1098 return dmz_rdwr_block(sb->dev, REQ_OP_READ,
1099 sb->block, sb->mblk->page);
1100}
1101
1102/*
1103 * Determine the position of the secondary super blocks on disk.
1104 * This is used only if a corruption of the primary super block
1105 * is detected.
1106 */
1107static int dmz_lookup_secondary_sb(struct dmz_metadata *zmd)
1108{
1109 unsigned int zone_nr_blocks = zmd->zone_nr_blocks;
1110 struct dmz_mblock *mblk;
1111 unsigned int zone_id = zmd->sb[0].zone->id;
1112 int i;
1113
1114 /* Allocate a block */
1115 mblk = dmz_alloc_mblock(zmd, 0);
1116 if (!mblk)
1117 return -ENOMEM;
1118
1119 zmd->sb[1].mblk = mblk;
1120 zmd->sb[1].sb = mblk->data;
1121
1122 /* Bad first super block: search for the second one */
1123 zmd->sb[1].block = zmd->sb[0].block + zone_nr_blocks;
1124 zmd->sb[1].zone = dmz_get(zmd, zone_id + 1);
1125 zmd->sb[1].dev = zmd->sb[0].dev;
1126 for (i = 1; i < zmd->nr_rnd_zones; i++) {
1127 if (dmz_read_sb(zmd, &zmd->sb[1], 1) != 0)
1128 break;
1129 if (le32_to_cpu(zmd->sb[1].sb->magic) == DMZ_MAGIC)
1130 return 0;
1131 zmd->sb[1].block += zone_nr_blocks;
1132 zmd->sb[1].zone = dmz_get(zmd, zone_id + i);
1133 }
1134
1135 dmz_free_mblock(zmd, mblk);
1136 zmd->sb[1].mblk = NULL;
1137 zmd->sb[1].zone = NULL;
1138 zmd->sb[1].dev = NULL;
1139
1140 return -EIO;
1141}
1142
1143/*
1144 * Read a super block from disk.
1145 */
1146static int dmz_get_sb(struct dmz_metadata *zmd, struct dmz_sb *sb, int set)
1147{
1148 struct dmz_mblock *mblk;
1149 int ret;
1150
1151 /* Allocate a block */
1152 mblk = dmz_alloc_mblock(zmd, 0);
1153 if (!mblk)
1154 return -ENOMEM;
1155
1156 sb->mblk = mblk;
1157 sb->sb = mblk->data;
1158
1159 /* Read super block */
1160 ret = dmz_read_sb(zmd, sb, set);
1161 if (ret) {
1162 dmz_free_mblock(zmd, mblk);
1163 sb->mblk = NULL;
1164 return ret;
1165 }
1166
1167 return 0;
1168}
1169
1170/*
1171 * Recover a metadata set.
1172 */
1173static int dmz_recover_mblocks(struct dmz_metadata *zmd, unsigned int dst_set)
1174{
1175 unsigned int src_set = dst_set ^ 0x1;
1176 struct page *page;
1177 int i, ret;
1178
1179 dmz_dev_warn(zmd->sb[dst_set].dev,
1180 "Metadata set %u invalid: recovering", dst_set);
1181
1182 if (dst_set == 0)
1183 zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);
1184 else
1185 zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);
1186
1187 page = alloc_page(GFP_NOIO);
1188 if (!page)
1189 return -ENOMEM;
1190
1191 /* Copy metadata blocks */
1192 for (i = 1; i < zmd->nr_meta_blocks; i++) {
1193 ret = dmz_rdwr_block(zmd->sb[src_set].dev, REQ_OP_READ,
1194 zmd->sb[src_set].block + i, page);
1195 if (ret)
1196 goto out;
1197 ret = dmz_rdwr_block(zmd->sb[dst_set].dev, REQ_OP_WRITE,
1198 zmd->sb[dst_set].block + i, page);
1199 if (ret)
1200 goto out;
1201 }
1202
1203 /* Finalize with the super block */
1204 if (!zmd->sb[dst_set].mblk) {
1205 zmd->sb[dst_set].mblk = dmz_alloc_mblock(zmd, 0);
1206 if (!zmd->sb[dst_set].mblk) {
1207 ret = -ENOMEM;
1208 goto out;
1209 }
1210 zmd->sb[dst_set].sb = zmd->sb[dst_set].mblk->data;
1211 }
1212
1213 ret = dmz_write_sb(zmd, dst_set);
1214out:
1215 __free_pages(page, 0);
1216
1217 return ret;
1218}
1219
1220/*
1221 * Get super block from disk.
1222 */
1223static int dmz_load_sb(struct dmz_metadata *zmd)
1224{
1225 bool sb_good[2] = {false, false};
1226 u64 sb_gen[2] = {0, 0};
1227 int ret;
1228
1229 if (!zmd->sb[0].zone) {
1230 dmz_zmd_err(zmd, "Primary super block zone not set");
1231 return -ENXIO;
1232 }
1233
1234 /* Read and check the primary super block */
1235 zmd->sb[0].block = dmz_start_block(zmd, zmd->sb[0].zone);
1236 zmd->sb[0].dev = zmd->sb[0].zone->dev;
1237 ret = dmz_get_sb(zmd, &zmd->sb[0], 0);
1238 if (ret) {
1239 dmz_dev_err(zmd->sb[0].dev, "Read primary super block failed");
1240 return ret;
1241 }
1242
1243 ret = dmz_check_sb(zmd, &zmd->sb[0], false);
1244
1245 /* Read and check secondary super block */
1246 if (ret == 0) {
1247 sb_good[0] = true;
1248 if (!zmd->sb[1].zone) {
1249 unsigned int zone_id =
1250 zmd->sb[0].zone->id + zmd->nr_meta_zones;
1251
1252 zmd->sb[1].zone = dmz_get(zmd, zone_id);
1253 }
1254 zmd->sb[1].block = dmz_start_block(zmd, zmd->sb[1].zone);
1255 zmd->sb[1].dev = zmd->sb[0].dev;
1256 ret = dmz_get_sb(zmd, &zmd->sb[1], 1);
1257 } else
1258 ret = dmz_lookup_secondary_sb(zmd);
1259
1260 if (ret) {
1261 dmz_dev_err(zmd->sb[1].dev, "Read secondary super block failed");
1262 return ret;
1263 }
1264
1265 ret = dmz_check_sb(zmd, &zmd->sb[1], false);
1266 if (ret == 0)
1267 sb_good[1] = true;
1268
1269 /* Use highest generation sb first */
1270 if (!sb_good[0] && !sb_good[1]) {
1271 dmz_zmd_err(zmd, "No valid super block found");
1272 return -EIO;
1273 }
1274
1275 if (sb_good[0])
1276 sb_gen[0] = le64_to_cpu(zmd->sb[0].sb->gen);
1277 else {
1278 ret = dmz_recover_mblocks(zmd, 0);
1279 if (ret) {
1280 dmz_dev_err(zmd->sb[0].dev,
1281 "Recovery of superblock 0 failed");
1282 return -EIO;
1283 }
1284 }
1285
1286 if (sb_good[1])
1287 sb_gen[1] = le64_to_cpu(zmd->sb[1].sb->gen);
1288 else {
1289 ret = dmz_recover_mblocks(zmd, 1);
1290
1291 if (ret) {
1292 dmz_dev_err(zmd->sb[1].dev,
1293 "Recovery of superblock 1 failed");
1294 return -EIO;
1295 }
1296 }
1297
1298 if (sb_gen[0] >= sb_gen[1]) {
1299 zmd->sb_gen = sb_gen[0];
1300 zmd->mblk_primary = 0;
1301 } else {
1302 zmd->sb_gen = sb_gen[1];
1303 zmd->mblk_primary = 1;
1304 }
1305
1306 dmz_dev_debug(zmd->sb[zmd->mblk_primary].dev,
1307 "Using super block %u (gen %llu)",
1308 zmd->mblk_primary, zmd->sb_gen);
1309
1310 if (zmd->sb_version > 1) {
1311 int i;
1312 struct dmz_sb *sb;
1313
1314 sb = kzalloc(sizeof(struct dmz_sb), GFP_KERNEL);
1315 if (!sb)
1316 return -ENOMEM;
1317 for (i = 1; i < zmd->nr_devs; i++) {
1318 sb->block = 0;
1319 sb->zone = dmz_get(zmd, zmd->dev[i].zone_offset);
1320 sb->dev = &zmd->dev[i];
1321 if (!dmz_is_meta(sb->zone)) {
1322 dmz_dev_err(sb->dev,
1323 "Tertiary super block zone %u not marked as metadata zone",
1324 sb->zone->id);
1325 ret = -EINVAL;
1326 goto out_kfree;
1327 }
1328 ret = dmz_get_sb(zmd, sb, i + 1);
1329 if (ret) {
1330 dmz_dev_err(sb->dev,
1331 "Read tertiary super block failed");
1332 dmz_free_mblock(zmd, sb->mblk);
1333 goto out_kfree;
1334 }
1335 ret = dmz_check_sb(zmd, sb, true);
1336 dmz_free_mblock(zmd, sb->mblk);
1337 if (ret == -EINVAL)
1338 goto out_kfree;
1339 }
1340out_kfree:
1341 kfree(sb);
1342 }
1343 return ret;
1344}
1345
1346/*
1347 * Initialize a zone descriptor.
1348 */
1349static int dmz_init_zone(struct blk_zone *blkz, unsigned int num, void *data)
1350{
1351 struct dmz_dev *dev = data;
1352 struct dmz_metadata *zmd = dev->metadata;
1353 int idx = num + dev->zone_offset;
1354 struct dm_zone *zone;
1355
1356 zone = dmz_insert(zmd, idx, dev);
1357 if (IS_ERR(zone))
1358 return PTR_ERR(zone);
1359
1360 if (blkz->len != zmd->zone_nr_sectors) {
1361 if (zmd->sb_version > 1) {
1362 /* Ignore the eventual runt (smaller) zone */
1363 set_bit(DMZ_OFFLINE, &zone->flags);
1364 return 0;
1365 } else if (blkz->start + blkz->len == dev->capacity)
1366 return 0;
1367 return -ENXIO;
1368 }
1369
1370 /*
1371 * Devices that have zones with a capacity smaller than the zone size
1372 * (e.g. NVMe zoned namespaces) are not supported.
1373 */
1374 if (blkz->capacity != blkz->len)
1375 return -ENXIO;
1376
1377 switch (blkz->type) {
1378 case BLK_ZONE_TYPE_CONVENTIONAL:
1379 set_bit(DMZ_RND, &zone->flags);
1380 break;
1381 case BLK_ZONE_TYPE_SEQWRITE_REQ:
1382 case BLK_ZONE_TYPE_SEQWRITE_PREF:
1383 set_bit(DMZ_SEQ, &zone->flags);
1384 break;
1385 default:
1386 return -ENXIO;
1387 }
1388
1389 if (dmz_is_rnd(zone))
1390 zone->wp_block = 0;
1391 else
1392 zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start);
1393
1394 if (blkz->cond == BLK_ZONE_COND_OFFLINE)
1395 set_bit(DMZ_OFFLINE, &zone->flags);
1396 else if (blkz->cond == BLK_ZONE_COND_READONLY)
1397 set_bit(DMZ_READ_ONLY, &zone->flags);
1398 else {
1399 zmd->nr_useable_zones++;
1400 if (dmz_is_rnd(zone)) {
1401 zmd->nr_rnd_zones++;
1402 if (zmd->nr_devs == 1 && !zmd->sb[0].zone) {
1403 /* Primary super block zone */
1404 zmd->sb[0].zone = zone;
1405 }
1406 }
1407 if (zmd->nr_devs > 1 && num == 0) {
1408 /*
1409 * Tertiary superblock zones are always at the
1410 * start of the zoned devices, so mark them
1411 * as metadata zone.
1412 */
1413 set_bit(DMZ_META, &zone->flags);
1414 }
1415 }
1416 return 0;
1417}
1418
1419static int dmz_emulate_zones(struct dmz_metadata *zmd, struct dmz_dev *dev)
1420{
1421 int idx;
1422 sector_t zone_offset = 0;
1423
1424 for (idx = 0; idx < dev->nr_zones; idx++) {
1425 struct dm_zone *zone;
1426
1427 zone = dmz_insert(zmd, idx, dev);
1428 if (IS_ERR(zone))
1429 return PTR_ERR(zone);
1430 set_bit(DMZ_CACHE, &zone->flags);
1431 zone->wp_block = 0;
1432 zmd->nr_cache_zones++;
1433 zmd->nr_useable_zones++;
1434 if (dev->capacity - zone_offset < zmd->zone_nr_sectors) {
1435 /* Disable runt zone */
1436 set_bit(DMZ_OFFLINE, &zone->flags);
1437 break;
1438 }
1439 zone_offset += zmd->zone_nr_sectors;
1440 }
1441 return 0;
1442}
1443
1444/*
1445 * Free zones descriptors.
1446 */
1447static void dmz_drop_zones(struct dmz_metadata *zmd)
1448{
1449 int idx;
1450
1451 for (idx = 0; idx < zmd->nr_zones; idx++) {
1452 struct dm_zone *zone = xa_load(&zmd->zones, idx);
1453
1454 kfree(zone);
1455 xa_erase(&zmd->zones, idx);
1456 }
1457 xa_destroy(&zmd->zones);
1458}
1459
1460/*
1461 * Allocate and initialize zone descriptors using the zone
1462 * information from disk.
1463 */
1464static int dmz_init_zones(struct dmz_metadata *zmd)
1465{
1466 int i, ret;
1467 struct dmz_dev *zoned_dev = &zmd->dev[0];
1468
1469 /* Init */
1470 zmd->zone_nr_sectors = zmd->dev[0].zone_nr_sectors;
1471 zmd->zone_nr_sectors_shift = ilog2(zmd->zone_nr_sectors);
1472 zmd->zone_nr_blocks = dmz_sect2blk(zmd->zone_nr_sectors);
1473 zmd->zone_nr_blocks_shift = ilog2(zmd->zone_nr_blocks);
1474 zmd->zone_bitmap_size = zmd->zone_nr_blocks >> 3;
1475 zmd->zone_nr_bitmap_blocks =
1476 max_t(sector_t, 1, zmd->zone_bitmap_size >> DMZ_BLOCK_SHIFT);
1477 zmd->zone_bits_per_mblk = min_t(sector_t, zmd->zone_nr_blocks,
1478 DMZ_BLOCK_SIZE_BITS);
1479
1480 /* Allocate zone array */
1481 zmd->nr_zones = 0;
1482 for (i = 0; i < zmd->nr_devs; i++) {
1483 struct dmz_dev *dev = &zmd->dev[i];
1484
1485 dev->metadata = zmd;
1486 zmd->nr_zones += dev->nr_zones;
1487
1488 atomic_set(&dev->unmap_nr_rnd, 0);
1489 INIT_LIST_HEAD(&dev->unmap_rnd_list);
1490 INIT_LIST_HEAD(&dev->map_rnd_list);
1491
1492 atomic_set(&dev->unmap_nr_seq, 0);
1493 INIT_LIST_HEAD(&dev->unmap_seq_list);
1494 INIT_LIST_HEAD(&dev->map_seq_list);
1495 }
1496
1497 if (!zmd->nr_zones) {
1498 DMERR("(%s): No zones found", zmd->devname);
1499 return -ENXIO;
1500 }
1501 xa_init(&zmd->zones);
1502
1503 DMDEBUG("(%s): Using %zu B for zone information",
1504 zmd->devname, sizeof(struct dm_zone) * zmd->nr_zones);
1505
1506 if (zmd->nr_devs > 1) {
1507 ret = dmz_emulate_zones(zmd, &zmd->dev[0]);
1508 if (ret < 0) {
1509 DMDEBUG("(%s): Failed to emulate zones, error %d",
1510 zmd->devname, ret);
1511 dmz_drop_zones(zmd);
1512 return ret;
1513 }
1514
1515 /*
1516 * Primary superblock zone is always at zone 0 when multiple
1517 * drives are present.
1518 */
1519 zmd->sb[0].zone = dmz_get(zmd, 0);
1520
1521 for (i = 1; i < zmd->nr_devs; i++) {
1522 zoned_dev = &zmd->dev[i];
1523
1524 ret = blkdev_report_zones(zoned_dev->bdev, 0,
1525 BLK_ALL_ZONES,
1526 dmz_init_zone, zoned_dev);
1527 if (ret < 0) {
1528 DMDEBUG("(%s): Failed to report zones, error %d",
1529 zmd->devname, ret);
1530 dmz_drop_zones(zmd);
1531 return ret;
1532 }
1533 }
1534 return 0;
1535 }
1536
1537 /*
1538 * Get zone information and initialize zone descriptors. At the same
1539 * time, determine where the super block should be: first block of the
1540 * first randomly writable zone.
1541 */
1542 ret = blkdev_report_zones(zoned_dev->bdev, 0, BLK_ALL_ZONES,
1543 dmz_init_zone, zoned_dev);
1544 if (ret < 0) {
1545 DMDEBUG("(%s): Failed to report zones, error %d",
1546 zmd->devname, ret);
1547 dmz_drop_zones(zmd);
1548 return ret;
1549 }
1550
1551 return 0;
1552}
1553
1554static int dmz_update_zone_cb(struct blk_zone *blkz, unsigned int idx,
1555 void *data)
1556{
1557 struct dm_zone *zone = data;
1558
1559 clear_bit(DMZ_OFFLINE, &zone->flags);
1560 clear_bit(DMZ_READ_ONLY, &zone->flags);
1561 if (blkz->cond == BLK_ZONE_COND_OFFLINE)
1562 set_bit(DMZ_OFFLINE, &zone->flags);
1563 else if (blkz->cond == BLK_ZONE_COND_READONLY)
1564 set_bit(DMZ_READ_ONLY, &zone->flags);
1565
1566 if (dmz_is_seq(zone))
1567 zone->wp_block = dmz_sect2blk(blkz->wp - blkz->start);
1568 else
1569 zone->wp_block = 0;
1570 return 0;
1571}
1572
1573/*
1574 * Update a zone information.
1575 */
1576static int dmz_update_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1577{
1578 struct dmz_dev *dev = zone->dev;
1579 unsigned int noio_flag;
1580 int ret;
1581
1582 if (dev->flags & DMZ_BDEV_REGULAR)
1583 return 0;
1584
1585 /*
1586 * Get zone information from disk. Since blkdev_report_zones() uses
1587 * GFP_KERNEL by default for memory allocations, set the per-task
1588 * PF_MEMALLOC_NOIO flag so that all allocations are done as if
1589 * GFP_NOIO was specified.
1590 */
1591 noio_flag = memalloc_noio_save();
1592 ret = blkdev_report_zones(dev->bdev, dmz_start_sect(zmd, zone), 1,
1593 dmz_update_zone_cb, zone);
1594 memalloc_noio_restore(noio_flag);
1595
1596 if (ret == 0)
1597 ret = -EIO;
1598 if (ret < 0) {
1599 dmz_dev_err(dev, "Get zone %u report failed",
1600 zone->id);
1601 dmz_check_bdev(dev);
1602 return ret;
1603 }
1604
1605 return 0;
1606}
1607
1608/*
1609 * Check a zone write pointer position when the zone is marked
1610 * with the sequential write error flag.
1611 */
1612static int dmz_handle_seq_write_err(struct dmz_metadata *zmd,
1613 struct dm_zone *zone)
1614{
1615 struct dmz_dev *dev = zone->dev;
1616 unsigned int wp = 0;
1617 int ret;
1618
1619 wp = zone->wp_block;
1620 ret = dmz_update_zone(zmd, zone);
1621 if (ret)
1622 return ret;
1623
1624 dmz_dev_warn(dev, "Processing zone %u write error (zone wp %u/%u)",
1625 zone->id, zone->wp_block, wp);
1626
1627 if (zone->wp_block < wp) {
1628 dmz_invalidate_blocks(zmd, zone, zone->wp_block,
1629 wp - zone->wp_block);
1630 }
1631
1632 return 0;
1633}
1634
1635/*
1636 * Reset a zone write pointer.
1637 */
1638static int dmz_reset_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1639{
1640 int ret;
1641
1642 /*
1643 * Ignore offline zones, read only zones,
1644 * and conventional zones.
1645 */
1646 if (dmz_is_offline(zone) ||
1647 dmz_is_readonly(zone) ||
1648 dmz_is_rnd(zone))
1649 return 0;
1650
1651 if (!dmz_is_empty(zone) || dmz_seq_write_err(zone)) {
1652 struct dmz_dev *dev = zone->dev;
1653 unsigned int noio_flag;
1654
1655 noio_flag = memalloc_noio_save();
1656 ret = blkdev_zone_mgmt(dev->bdev, REQ_OP_ZONE_RESET,
1657 dmz_start_sect(zmd, zone),
1658 zmd->zone_nr_sectors);
1659 memalloc_noio_restore(noio_flag);
1660 if (ret) {
1661 dmz_dev_err(dev, "Reset zone %u failed %d",
1662 zone->id, ret);
1663 return ret;
1664 }
1665 }
1666
1667 /* Clear write error bit and rewind write pointer position */
1668 clear_bit(DMZ_SEQ_WRITE_ERR, &zone->flags);
1669 zone->wp_block = 0;
1670
1671 return 0;
1672}
1673
1674static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone);
1675
1676/*
1677 * Initialize chunk mapping.
1678 */
1679static int dmz_load_mapping(struct dmz_metadata *zmd)
1680{
1681 struct dm_zone *dzone, *bzone;
1682 struct dmz_mblock *dmap_mblk = NULL;
1683 struct dmz_map *dmap;
1684 unsigned int i = 0, e = 0, chunk = 0;
1685 unsigned int dzone_id;
1686 unsigned int bzone_id;
1687
1688 /* Metadata block array for the chunk mapping table */
1689 zmd->map_mblk = kcalloc(zmd->nr_map_blocks,
1690 sizeof(struct dmz_mblk *), GFP_KERNEL);
1691 if (!zmd->map_mblk)
1692 return -ENOMEM;
1693
1694 /* Get chunk mapping table blocks and initialize zone mapping */
1695 while (chunk < zmd->nr_chunks) {
1696 if (!dmap_mblk) {
1697 /* Get mapping block */
1698 dmap_mblk = dmz_get_mblock(zmd, i + 1);
1699 if (IS_ERR(dmap_mblk))
1700 return PTR_ERR(dmap_mblk);
1701 zmd->map_mblk[i] = dmap_mblk;
1702 dmap = dmap_mblk->data;
1703 i++;
1704 e = 0;
1705 }
1706
1707 /* Check data zone */
1708 dzone_id = le32_to_cpu(dmap[e].dzone_id);
1709 if (dzone_id == DMZ_MAP_UNMAPPED)
1710 goto next;
1711
1712 if (dzone_id >= zmd->nr_zones) {
1713 dmz_zmd_err(zmd, "Chunk %u mapping: invalid data zone ID %u",
1714 chunk, dzone_id);
1715 return -EIO;
1716 }
1717
1718 dzone = dmz_get(zmd, dzone_id);
1719 if (!dzone) {
1720 dmz_zmd_err(zmd, "Chunk %u mapping: data zone %u not present",
1721 chunk, dzone_id);
1722 return -EIO;
1723 }
1724 set_bit(DMZ_DATA, &dzone->flags);
1725 dzone->chunk = chunk;
1726 dmz_get_zone_weight(zmd, dzone);
1727
1728 if (dmz_is_cache(dzone))
1729 list_add_tail(&dzone->link, &zmd->map_cache_list);
1730 else if (dmz_is_rnd(dzone))
1731 list_add_tail(&dzone->link, &dzone->dev->map_rnd_list);
1732 else
1733 list_add_tail(&dzone->link, &dzone->dev->map_seq_list);
1734
1735 /* Check buffer zone */
1736 bzone_id = le32_to_cpu(dmap[e].bzone_id);
1737 if (bzone_id == DMZ_MAP_UNMAPPED)
1738 goto next;
1739
1740 if (bzone_id >= zmd->nr_zones) {
1741 dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone ID %u",
1742 chunk, bzone_id);
1743 return -EIO;
1744 }
1745
1746 bzone = dmz_get(zmd, bzone_id);
1747 if (!bzone) {
1748 dmz_zmd_err(zmd, "Chunk %u mapping: buffer zone %u not present",
1749 chunk, bzone_id);
1750 return -EIO;
1751 }
1752 if (!dmz_is_rnd(bzone) && !dmz_is_cache(bzone)) {
1753 dmz_zmd_err(zmd, "Chunk %u mapping: invalid buffer zone %u",
1754 chunk, bzone_id);
1755 return -EIO;
1756 }
1757
1758 set_bit(DMZ_DATA, &bzone->flags);
1759 set_bit(DMZ_BUF, &bzone->flags);
1760 bzone->chunk = chunk;
1761 bzone->bzone = dzone;
1762 dzone->bzone = bzone;
1763 dmz_get_zone_weight(zmd, bzone);
1764 if (dmz_is_cache(bzone))
1765 list_add_tail(&bzone->link, &zmd->map_cache_list);
1766 else
1767 list_add_tail(&bzone->link, &bzone->dev->map_rnd_list);
1768next:
1769 chunk++;
1770 e++;
1771 if (e >= DMZ_MAP_ENTRIES)
1772 dmap_mblk = NULL;
1773 }
1774
1775 /*
1776 * At this point, only meta zones and mapped data zones were
1777 * fully initialized. All remaining zones are unmapped data
1778 * zones. Finish initializing those here.
1779 */
1780 for (i = 0; i < zmd->nr_zones; i++) {
1781 dzone = dmz_get(zmd, i);
1782 if (!dzone)
1783 continue;
1784 if (dmz_is_meta(dzone))
1785 continue;
1786 if (dmz_is_offline(dzone))
1787 continue;
1788
1789 if (dmz_is_cache(dzone))
1790 zmd->nr_cache++;
1791 else if (dmz_is_rnd(dzone))
1792 dzone->dev->nr_rnd++;
1793 else
1794 dzone->dev->nr_seq++;
1795
1796 if (dmz_is_data(dzone)) {
1797 /* Already initialized */
1798 continue;
1799 }
1800
1801 /* Unmapped data zone */
1802 set_bit(DMZ_DATA, &dzone->flags);
1803 dzone->chunk = DMZ_MAP_UNMAPPED;
1804 if (dmz_is_cache(dzone)) {
1805 list_add_tail(&dzone->link, &zmd->unmap_cache_list);
1806 atomic_inc(&zmd->unmap_nr_cache);
1807 } else if (dmz_is_rnd(dzone)) {
1808 list_add_tail(&dzone->link,
1809 &dzone->dev->unmap_rnd_list);
1810 atomic_inc(&dzone->dev->unmap_nr_rnd);
1811 } else if (atomic_read(&zmd->nr_reserved_seq_zones) < zmd->nr_reserved_seq) {
1812 list_add_tail(&dzone->link, &zmd->reserved_seq_zones_list);
1813 set_bit(DMZ_RESERVED, &dzone->flags);
1814 atomic_inc(&zmd->nr_reserved_seq_zones);
1815 dzone->dev->nr_seq--;
1816 } else {
1817 list_add_tail(&dzone->link,
1818 &dzone->dev->unmap_seq_list);
1819 atomic_inc(&dzone->dev->unmap_nr_seq);
1820 }
1821 }
1822
1823 return 0;
1824}
1825
1826/*
1827 * Set a data chunk mapping.
1828 */
1829static void dmz_set_chunk_mapping(struct dmz_metadata *zmd, unsigned int chunk,
1830 unsigned int dzone_id, unsigned int bzone_id)
1831{
1832 struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
1833 struct dmz_map *dmap = dmap_mblk->data;
1834 int map_idx = chunk & DMZ_MAP_ENTRIES_MASK;
1835
1836 dmap[map_idx].dzone_id = cpu_to_le32(dzone_id);
1837 dmap[map_idx].bzone_id = cpu_to_le32(bzone_id);
1838 dmz_dirty_mblock(zmd, dmap_mblk);
1839}
1840
1841/*
1842 * The list of mapped zones is maintained in LRU order.
1843 * This rotates a zone at the end of its map list.
1844 */
1845static void __dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1846{
1847 if (list_empty(&zone->link))
1848 return;
1849
1850 list_del_init(&zone->link);
1851 if (dmz_is_seq(zone)) {
1852 /* LRU rotate sequential zone */
1853 list_add_tail(&zone->link, &zone->dev->map_seq_list);
1854 } else if (dmz_is_cache(zone)) {
1855 /* LRU rotate cache zone */
1856 list_add_tail(&zone->link, &zmd->map_cache_list);
1857 } else {
1858 /* LRU rotate random zone */
1859 list_add_tail(&zone->link, &zone->dev->map_rnd_list);
1860 }
1861}
1862
1863/*
1864 * The list of mapped random zones is maintained
1865 * in LRU order. This rotates a zone at the end of the list.
1866 */
1867static void dmz_lru_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
1868{
1869 __dmz_lru_zone(zmd, zone);
1870 if (zone->bzone)
1871 __dmz_lru_zone(zmd, zone->bzone);
1872}
1873
1874/*
1875 * Wait for any zone to be freed.
1876 */
1877static void dmz_wait_for_free_zones(struct dmz_metadata *zmd)
1878{
1879 DEFINE_WAIT(wait);
1880
1881 prepare_to_wait(&zmd->free_wq, &wait, TASK_UNINTERRUPTIBLE);
1882 dmz_unlock_map(zmd);
1883 dmz_unlock_metadata(zmd);
1884
1885 io_schedule_timeout(HZ);
1886
1887 dmz_lock_metadata(zmd);
1888 dmz_lock_map(zmd);
1889 finish_wait(&zmd->free_wq, &wait);
1890}
1891
1892/*
1893 * Lock a zone for reclaim (set the zone RECLAIM bit).
1894 * Returns false if the zone cannot be locked or if it is already locked
1895 * and 1 otherwise.
1896 */
1897int dmz_lock_zone_reclaim(struct dm_zone *zone)
1898{
1899 /* Active zones cannot be reclaimed */
1900 if (dmz_is_active(zone))
1901 return 0;
1902
1903 return !test_and_set_bit(DMZ_RECLAIM, &zone->flags);
1904}
1905
1906/*
1907 * Clear a zone reclaim flag.
1908 */
1909void dmz_unlock_zone_reclaim(struct dm_zone *zone)
1910{
1911 WARN_ON(dmz_is_active(zone));
1912 WARN_ON(!dmz_in_reclaim(zone));
1913
1914 clear_bit_unlock(DMZ_RECLAIM, &zone->flags);
1915 smp_mb__after_atomic();
1916 wake_up_bit(&zone->flags, DMZ_RECLAIM);
1917}
1918
1919/*
1920 * Wait for a zone reclaim to complete.
1921 */
1922static void dmz_wait_for_reclaim(struct dmz_metadata *zmd, struct dm_zone *zone)
1923{
1924 dmz_unlock_map(zmd);
1925 dmz_unlock_metadata(zmd);
1926 set_bit(DMZ_RECLAIM_TERMINATE, &zone->flags);
1927 wait_on_bit_timeout(&zone->flags, DMZ_RECLAIM, TASK_UNINTERRUPTIBLE, HZ);
1928 clear_bit(DMZ_RECLAIM_TERMINATE, &zone->flags);
1929 dmz_lock_metadata(zmd);
1930 dmz_lock_map(zmd);
1931}
1932
1933/*
1934 * Select a cache or random write zone for reclaim.
1935 */
1936static struct dm_zone *dmz_get_rnd_zone_for_reclaim(struct dmz_metadata *zmd,
1937 unsigned int idx, bool idle)
1938{
1939 struct dm_zone *dzone = NULL;
1940 struct dm_zone *zone, *maxw_z = NULL;
1941 struct list_head *zone_list;
1942
1943 /* If we have cache zones select from the cache zone list */
1944 if (zmd->nr_cache) {
1945 zone_list = &zmd->map_cache_list;
1946 /* Try to relaim random zones, too, when idle */
1947 if (idle && list_empty(zone_list))
1948 zone_list = &zmd->dev[idx].map_rnd_list;
1949 } else
1950 zone_list = &zmd->dev[idx].map_rnd_list;
1951
1952 /*
1953 * Find the buffer zone with the heaviest weight or the first (oldest)
1954 * data zone that can be reclaimed.
1955 */
1956 list_for_each_entry(zone, zone_list, link) {
1957 if (dmz_is_buf(zone)) {
1958 dzone = zone->bzone;
1959 if (dmz_is_rnd(dzone) && dzone->dev->dev_idx != idx)
1960 continue;
1961 if (!maxw_z || maxw_z->weight < dzone->weight)
1962 maxw_z = dzone;
1963 } else {
1964 dzone = zone;
1965 if (dmz_lock_zone_reclaim(dzone))
1966 return dzone;
1967 }
1968 }
1969
1970 if (maxw_z && dmz_lock_zone_reclaim(maxw_z))
1971 return maxw_z;
1972
1973 /*
1974 * If we come here, none of the zones inspected could be locked for
1975 * reclaim. Try again, being more aggressive, that is, find the
1976 * first zone that can be reclaimed regardless of its weitght.
1977 */
1978 list_for_each_entry(zone, zone_list, link) {
1979 if (dmz_is_buf(zone)) {
1980 dzone = zone->bzone;
1981 if (dmz_is_rnd(dzone) && dzone->dev->dev_idx != idx)
1982 continue;
1983 } else
1984 dzone = zone;
1985 if (dmz_lock_zone_reclaim(dzone))
1986 return dzone;
1987 }
1988
1989 return NULL;
1990}
1991
1992/*
1993 * Select a buffered sequential zone for reclaim.
1994 */
1995static struct dm_zone *dmz_get_seq_zone_for_reclaim(struct dmz_metadata *zmd,
1996 unsigned int idx)
1997{
1998 struct dm_zone *zone;
1999
2000 list_for_each_entry(zone, &zmd->dev[idx].map_seq_list, link) {
2001 if (!zone->bzone)
2002 continue;
2003 if (dmz_lock_zone_reclaim(zone))
2004 return zone;
2005 }
2006
2007 return NULL;
2008}
2009
2010/*
2011 * Select a zone for reclaim.
2012 */
2013struct dm_zone *dmz_get_zone_for_reclaim(struct dmz_metadata *zmd,
2014 unsigned int dev_idx, bool idle)
2015{
2016 struct dm_zone *zone = NULL;
2017
2018 /*
2019 * Search for a zone candidate to reclaim: 2 cases are possible.
2020 * (1) There is no free sequential zones. Then a random data zone
2021 * cannot be reclaimed. So choose a sequential zone to reclaim so
2022 * that afterward a random zone can be reclaimed.
2023 * (2) At least one free sequential zone is available, then choose
2024 * the oldest random zone (data or buffer) that can be locked.
2025 */
2026 dmz_lock_map(zmd);
2027 if (list_empty(&zmd->reserved_seq_zones_list))
2028 zone = dmz_get_seq_zone_for_reclaim(zmd, dev_idx);
2029 if (!zone)
2030 zone = dmz_get_rnd_zone_for_reclaim(zmd, dev_idx, idle);
2031 dmz_unlock_map(zmd);
2032
2033 return zone;
2034}
2035
2036/*
2037 * Get the zone mapping a chunk, if the chunk is mapped already.
2038 * If no mapping exist and the operation is WRITE, a zone is
2039 * allocated and used to map the chunk.
2040 * The zone returned will be set to the active state.
2041 */
2042struct dm_zone *dmz_get_chunk_mapping(struct dmz_metadata *zmd,
2043 unsigned int chunk, enum req_op op)
2044{
2045 struct dmz_mblock *dmap_mblk = zmd->map_mblk[chunk >> DMZ_MAP_ENTRIES_SHIFT];
2046 struct dmz_map *dmap = dmap_mblk->data;
2047 int dmap_idx = chunk & DMZ_MAP_ENTRIES_MASK;
2048 unsigned int dzone_id;
2049 struct dm_zone *dzone = NULL;
2050 int ret = 0;
2051 int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;
2052
2053 dmz_lock_map(zmd);
2054again:
2055 /* Get the chunk mapping */
2056 dzone_id = le32_to_cpu(dmap[dmap_idx].dzone_id);
2057 if (dzone_id == DMZ_MAP_UNMAPPED) {
2058 /*
2059 * Read or discard in unmapped chunks are fine. But for
2060 * writes, we need a mapping, so get one.
2061 */
2062 if (op != REQ_OP_WRITE)
2063 goto out;
2064
2065 /* Allocate a random zone */
2066 dzone = dmz_alloc_zone(zmd, 0, alloc_flags);
2067 if (!dzone) {
2068 if (dmz_dev_is_dying(zmd)) {
2069 dzone = ERR_PTR(-EIO);
2070 goto out;
2071 }
2072 dmz_wait_for_free_zones(zmd);
2073 goto again;
2074 }
2075
2076 dmz_map_zone(zmd, dzone, chunk);
2077
2078 } else {
2079 /* The chunk is already mapped: get the mapping zone */
2080 dzone = dmz_get(zmd, dzone_id);
2081 if (!dzone) {
2082 dzone = ERR_PTR(-EIO);
2083 goto out;
2084 }
2085 if (dzone->chunk != chunk) {
2086 dzone = ERR_PTR(-EIO);
2087 goto out;
2088 }
2089
2090 /* Repair write pointer if the sequential dzone has error */
2091 if (dmz_seq_write_err(dzone)) {
2092 ret = dmz_handle_seq_write_err(zmd, dzone);
2093 if (ret) {
2094 dzone = ERR_PTR(-EIO);
2095 goto out;
2096 }
2097 clear_bit(DMZ_SEQ_WRITE_ERR, &dzone->flags);
2098 }
2099 }
2100
2101 /*
2102 * If the zone is being reclaimed, the chunk mapping may change
2103 * to a different zone. So wait for reclaim and retry. Otherwise,
2104 * activate the zone (this will prevent reclaim from touching it).
2105 */
2106 if (dmz_in_reclaim(dzone)) {
2107 dmz_wait_for_reclaim(zmd, dzone);
2108 goto again;
2109 }
2110 dmz_activate_zone(dzone);
2111 dmz_lru_zone(zmd, dzone);
2112out:
2113 dmz_unlock_map(zmd);
2114
2115 return dzone;
2116}
2117
2118/*
2119 * Write and discard change the block validity of data zones and their buffer
2120 * zones. Check here that valid blocks are still present. If all blocks are
2121 * invalid, the zones can be unmapped on the fly without waiting for reclaim
2122 * to do it.
2123 */
2124void dmz_put_chunk_mapping(struct dmz_metadata *zmd, struct dm_zone *dzone)
2125{
2126 struct dm_zone *bzone;
2127
2128 dmz_lock_map(zmd);
2129
2130 bzone = dzone->bzone;
2131 if (bzone) {
2132 if (dmz_weight(bzone))
2133 dmz_lru_zone(zmd, bzone);
2134 else {
2135 /* Empty buffer zone: reclaim it */
2136 dmz_unmap_zone(zmd, bzone);
2137 dmz_free_zone(zmd, bzone);
2138 bzone = NULL;
2139 }
2140 }
2141
2142 /* Deactivate the data zone */
2143 dmz_deactivate_zone(dzone);
2144 if (dmz_is_active(dzone) || bzone || dmz_weight(dzone))
2145 dmz_lru_zone(zmd, dzone);
2146 else {
2147 /* Unbuffered inactive empty data zone: reclaim it */
2148 dmz_unmap_zone(zmd, dzone);
2149 dmz_free_zone(zmd, dzone);
2150 }
2151
2152 dmz_unlock_map(zmd);
2153}
2154
2155/*
2156 * Allocate and map a random zone to buffer a chunk
2157 * already mapped to a sequential zone.
2158 */
2159struct dm_zone *dmz_get_chunk_buffer(struct dmz_metadata *zmd,
2160 struct dm_zone *dzone)
2161{
2162 struct dm_zone *bzone;
2163 int alloc_flags = zmd->nr_cache ? DMZ_ALLOC_CACHE : DMZ_ALLOC_RND;
2164
2165 dmz_lock_map(zmd);
2166again:
2167 bzone = dzone->bzone;
2168 if (bzone)
2169 goto out;
2170
2171 /* Allocate a random zone */
2172 bzone = dmz_alloc_zone(zmd, 0, alloc_flags);
2173 if (!bzone) {
2174 if (dmz_dev_is_dying(zmd)) {
2175 bzone = ERR_PTR(-EIO);
2176 goto out;
2177 }
2178 dmz_wait_for_free_zones(zmd);
2179 goto again;
2180 }
2181
2182 /* Update the chunk mapping */
2183 dmz_set_chunk_mapping(zmd, dzone->chunk, dzone->id, bzone->id);
2184
2185 set_bit(DMZ_BUF, &bzone->flags);
2186 bzone->chunk = dzone->chunk;
2187 bzone->bzone = dzone;
2188 dzone->bzone = bzone;
2189 if (dmz_is_cache(bzone))
2190 list_add_tail(&bzone->link, &zmd->map_cache_list);
2191 else
2192 list_add_tail(&bzone->link, &bzone->dev->map_rnd_list);
2193out:
2194 dmz_unlock_map(zmd);
2195
2196 return bzone;
2197}
2198
2199/*
2200 * Get an unmapped (free) zone.
2201 * This must be called with the mapping lock held.
2202 */
2203struct dm_zone *dmz_alloc_zone(struct dmz_metadata *zmd, unsigned int dev_idx,
2204 unsigned long flags)
2205{
2206 struct list_head *list;
2207 struct dm_zone *zone;
2208 int i;
2209
2210 /* Schedule reclaim to ensure free zones are available */
2211 if (!(flags & DMZ_ALLOC_RECLAIM)) {
2212 for (i = 0; i < zmd->nr_devs; i++)
2213 dmz_schedule_reclaim(zmd->dev[i].reclaim);
2214 }
2215
2216 i = 0;
2217again:
2218 if (flags & DMZ_ALLOC_CACHE)
2219 list = &zmd->unmap_cache_list;
2220 else if (flags & DMZ_ALLOC_RND)
2221 list = &zmd->dev[dev_idx].unmap_rnd_list;
2222 else
2223 list = &zmd->dev[dev_idx].unmap_seq_list;
2224
2225 if (list_empty(list)) {
2226 /*
2227 * No free zone: return NULL if this is for not reclaim.
2228 */
2229 if (!(flags & DMZ_ALLOC_RECLAIM))
2230 return NULL;
2231 /*
2232 * Try to allocate from other devices
2233 */
2234 if (i < zmd->nr_devs) {
2235 dev_idx = (dev_idx + 1) % zmd->nr_devs;
2236 i++;
2237 goto again;
2238 }
2239
2240 /*
2241 * Fallback to the reserved sequential zones
2242 */
2243 zone = list_first_entry_or_null(&zmd->reserved_seq_zones_list,
2244 struct dm_zone, link);
2245 if (zone) {
2246 list_del_init(&zone->link);
2247 atomic_dec(&zmd->nr_reserved_seq_zones);
2248 }
2249 return zone;
2250 }
2251
2252 zone = list_first_entry(list, struct dm_zone, link);
2253 list_del_init(&zone->link);
2254
2255 if (dmz_is_cache(zone))
2256 atomic_dec(&zmd->unmap_nr_cache);
2257 else if (dmz_is_rnd(zone))
2258 atomic_dec(&zone->dev->unmap_nr_rnd);
2259 else
2260 atomic_dec(&zone->dev->unmap_nr_seq);
2261
2262 if (dmz_is_offline(zone)) {
2263 dmz_zmd_warn(zmd, "Zone %u is offline", zone->id);
2264 zone = NULL;
2265 goto again;
2266 }
2267 if (dmz_is_meta(zone)) {
2268 dmz_zmd_warn(zmd, "Zone %u has metadata", zone->id);
2269 zone = NULL;
2270 goto again;
2271 }
2272 return zone;
2273}
2274
2275/*
2276 * Free a zone.
2277 * This must be called with the mapping lock held.
2278 */
2279void dmz_free_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
2280{
2281 /* If this is a sequential zone, reset it */
2282 if (dmz_is_seq(zone))
2283 dmz_reset_zone(zmd, zone);
2284
2285 /* Return the zone to its type unmap list */
2286 if (dmz_is_cache(zone)) {
2287 list_add_tail(&zone->link, &zmd->unmap_cache_list);
2288 atomic_inc(&zmd->unmap_nr_cache);
2289 } else if (dmz_is_rnd(zone)) {
2290 list_add_tail(&zone->link, &zone->dev->unmap_rnd_list);
2291 atomic_inc(&zone->dev->unmap_nr_rnd);
2292 } else if (dmz_is_reserved(zone)) {
2293 list_add_tail(&zone->link, &zmd->reserved_seq_zones_list);
2294 atomic_inc(&zmd->nr_reserved_seq_zones);
2295 } else {
2296 list_add_tail(&zone->link, &zone->dev->unmap_seq_list);
2297 atomic_inc(&zone->dev->unmap_nr_seq);
2298 }
2299
2300 wake_up_all(&zmd->free_wq);
2301}
2302
2303/*
2304 * Map a chunk to a zone.
2305 * This must be called with the mapping lock held.
2306 */
2307void dmz_map_zone(struct dmz_metadata *zmd, struct dm_zone *dzone,
2308 unsigned int chunk)
2309{
2310 /* Set the chunk mapping */
2311 dmz_set_chunk_mapping(zmd, chunk, dzone->id,
2312 DMZ_MAP_UNMAPPED);
2313 dzone->chunk = chunk;
2314 if (dmz_is_cache(dzone))
2315 list_add_tail(&dzone->link, &zmd->map_cache_list);
2316 else if (dmz_is_rnd(dzone))
2317 list_add_tail(&dzone->link, &dzone->dev->map_rnd_list);
2318 else
2319 list_add_tail(&dzone->link, &dzone->dev->map_seq_list);
2320}
2321
2322/*
2323 * Unmap a zone.
2324 * This must be called with the mapping lock held.
2325 */
2326void dmz_unmap_zone(struct dmz_metadata *zmd, struct dm_zone *zone)
2327{
2328 unsigned int chunk = zone->chunk;
2329 unsigned int dzone_id;
2330
2331 if (chunk == DMZ_MAP_UNMAPPED) {
2332 /* Already unmapped */
2333 return;
2334 }
2335
2336 if (test_and_clear_bit(DMZ_BUF, &zone->flags)) {
2337 /*
2338 * Unmapping the chunk buffer zone: clear only
2339 * the chunk buffer mapping
2340 */
2341 dzone_id = zone->bzone->id;
2342 zone->bzone->bzone = NULL;
2343 zone->bzone = NULL;
2344
2345 } else {
2346 /*
2347 * Unmapping the chunk data zone: the zone must
2348 * not be buffered.
2349 */
2350 if (WARN_ON(zone->bzone)) {
2351 zone->bzone->bzone = NULL;
2352 zone->bzone = NULL;
2353 }
2354 dzone_id = DMZ_MAP_UNMAPPED;
2355 }
2356
2357 dmz_set_chunk_mapping(zmd, chunk, dzone_id, DMZ_MAP_UNMAPPED);
2358
2359 zone->chunk = DMZ_MAP_UNMAPPED;
2360 list_del_init(&zone->link);
2361}
2362
2363/*
2364 * Set @nr_bits bits in @bitmap starting from @bit.
2365 * Return the number of bits changed from 0 to 1.
2366 */
2367static unsigned int dmz_set_bits(unsigned long *bitmap,
2368 unsigned int bit, unsigned int nr_bits)
2369{
2370 unsigned long *addr;
2371 unsigned int end = bit + nr_bits;
2372 unsigned int n = 0;
2373
2374 while (bit < end) {
2375 if (((bit & (BITS_PER_LONG - 1)) == 0) &&
2376 ((end - bit) >= BITS_PER_LONG)) {
2377 /* Try to set the whole word at once */
2378 addr = bitmap + BIT_WORD(bit);
2379 if (*addr == 0) {
2380 *addr = ULONG_MAX;
2381 n += BITS_PER_LONG;
2382 bit += BITS_PER_LONG;
2383 continue;
2384 }
2385 }
2386
2387 if (!test_and_set_bit(bit, bitmap))
2388 n++;
2389 bit++;
2390 }
2391
2392 return n;
2393}
2394
2395/*
2396 * Get the bitmap block storing the bit for chunk_block in zone.
2397 */
2398static struct dmz_mblock *dmz_get_bitmap(struct dmz_metadata *zmd,
2399 struct dm_zone *zone,
2400 sector_t chunk_block)
2401{
2402 sector_t bitmap_block = 1 + zmd->nr_map_blocks +
2403 (sector_t)(zone->id * zmd->zone_nr_bitmap_blocks) +
2404 (chunk_block >> DMZ_BLOCK_SHIFT_BITS);
2405
2406 return dmz_get_mblock(zmd, bitmap_block);
2407}
2408
2409/*
2410 * Copy the valid blocks bitmap of from_zone to the bitmap of to_zone.
2411 */
2412int dmz_copy_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
2413 struct dm_zone *to_zone)
2414{
2415 struct dmz_mblock *from_mblk, *to_mblk;
2416 sector_t chunk_block = 0;
2417
2418 /* Get the zones bitmap blocks */
2419 while (chunk_block < zmd->zone_nr_blocks) {
2420 from_mblk = dmz_get_bitmap(zmd, from_zone, chunk_block);
2421 if (IS_ERR(from_mblk))
2422 return PTR_ERR(from_mblk);
2423 to_mblk = dmz_get_bitmap(zmd, to_zone, chunk_block);
2424 if (IS_ERR(to_mblk)) {
2425 dmz_release_mblock(zmd, from_mblk);
2426 return PTR_ERR(to_mblk);
2427 }
2428
2429 memcpy(to_mblk->data, from_mblk->data, DMZ_BLOCK_SIZE);
2430 dmz_dirty_mblock(zmd, to_mblk);
2431
2432 dmz_release_mblock(zmd, to_mblk);
2433 dmz_release_mblock(zmd, from_mblk);
2434
2435 chunk_block += zmd->zone_bits_per_mblk;
2436 }
2437
2438 to_zone->weight = from_zone->weight;
2439
2440 return 0;
2441}
2442
2443/*
2444 * Merge the valid blocks bitmap of from_zone into the bitmap of to_zone,
2445 * starting from chunk_block.
2446 */
2447int dmz_merge_valid_blocks(struct dmz_metadata *zmd, struct dm_zone *from_zone,
2448 struct dm_zone *to_zone, sector_t chunk_block)
2449{
2450 unsigned int nr_blocks;
2451 int ret;
2452
2453 /* Get the zones bitmap blocks */
2454 while (chunk_block < zmd->zone_nr_blocks) {
2455 /* Get a valid region from the source zone */
2456 ret = dmz_first_valid_block(zmd, from_zone, &chunk_block);
2457 if (ret <= 0)
2458 return ret;
2459
2460 nr_blocks = ret;
2461 ret = dmz_validate_blocks(zmd, to_zone, chunk_block, nr_blocks);
2462 if (ret)
2463 return ret;
2464
2465 chunk_block += nr_blocks;
2466 }
2467
2468 return 0;
2469}
2470
2471/*
2472 * Validate all the blocks in the range [block..block+nr_blocks-1].
2473 */
2474int dmz_validate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
2475 sector_t chunk_block, unsigned int nr_blocks)
2476{
2477 unsigned int count, bit, nr_bits;
2478 unsigned int zone_nr_blocks = zmd->zone_nr_blocks;
2479 struct dmz_mblock *mblk;
2480 unsigned int n = 0;
2481
2482 dmz_zmd_debug(zmd, "=> VALIDATE zone %u, block %llu, %u blocks",
2483 zone->id, (unsigned long long)chunk_block,
2484 nr_blocks);
2485
2486 WARN_ON(chunk_block + nr_blocks > zone_nr_blocks);
2487
2488 while (nr_blocks) {
2489 /* Get bitmap block */
2490 mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2491 if (IS_ERR(mblk))
2492 return PTR_ERR(mblk);
2493
2494 /* Set bits */
2495 bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2496 nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
2497
2498 count = dmz_set_bits((unsigned long *)mblk->data, bit, nr_bits);
2499 if (count) {
2500 dmz_dirty_mblock(zmd, mblk);
2501 n += count;
2502 }
2503 dmz_release_mblock(zmd, mblk);
2504
2505 nr_blocks -= nr_bits;
2506 chunk_block += nr_bits;
2507 }
2508
2509 if (likely(zone->weight + n <= zone_nr_blocks))
2510 zone->weight += n;
2511 else {
2512 dmz_zmd_warn(zmd, "Zone %u: weight %u should be <= %u",
2513 zone->id, zone->weight,
2514 zone_nr_blocks - n);
2515 zone->weight = zone_nr_blocks;
2516 }
2517
2518 return 0;
2519}
2520
2521/*
2522 * Clear nr_bits bits in bitmap starting from bit.
2523 * Return the number of bits cleared.
2524 */
2525static int dmz_clear_bits(unsigned long *bitmap, int bit, int nr_bits)
2526{
2527 unsigned long *addr;
2528 int end = bit + nr_bits;
2529 int n = 0;
2530
2531 while (bit < end) {
2532 if (((bit & (BITS_PER_LONG - 1)) == 0) &&
2533 ((end - bit) >= BITS_PER_LONG)) {
2534 /* Try to clear whole word at once */
2535 addr = bitmap + BIT_WORD(bit);
2536 if (*addr == ULONG_MAX) {
2537 *addr = 0;
2538 n += BITS_PER_LONG;
2539 bit += BITS_PER_LONG;
2540 continue;
2541 }
2542 }
2543
2544 if (test_and_clear_bit(bit, bitmap))
2545 n++;
2546 bit++;
2547 }
2548
2549 return n;
2550}
2551
2552/*
2553 * Invalidate all the blocks in the range [block..block+nr_blocks-1].
2554 */
2555int dmz_invalidate_blocks(struct dmz_metadata *zmd, struct dm_zone *zone,
2556 sector_t chunk_block, unsigned int nr_blocks)
2557{
2558 unsigned int count, bit, nr_bits;
2559 struct dmz_mblock *mblk;
2560 unsigned int n = 0;
2561
2562 dmz_zmd_debug(zmd, "=> INVALIDATE zone %u, block %llu, %u blocks",
2563 zone->id, (u64)chunk_block, nr_blocks);
2564
2565 WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);
2566
2567 while (nr_blocks) {
2568 /* Get bitmap block */
2569 mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2570 if (IS_ERR(mblk))
2571 return PTR_ERR(mblk);
2572
2573 /* Clear bits */
2574 bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2575 nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
2576
2577 count = dmz_clear_bits((unsigned long *)mblk->data,
2578 bit, nr_bits);
2579 if (count) {
2580 dmz_dirty_mblock(zmd, mblk);
2581 n += count;
2582 }
2583 dmz_release_mblock(zmd, mblk);
2584
2585 nr_blocks -= nr_bits;
2586 chunk_block += nr_bits;
2587 }
2588
2589 if (zone->weight >= n)
2590 zone->weight -= n;
2591 else {
2592 dmz_zmd_warn(zmd, "Zone %u: weight %u should be >= %u",
2593 zone->id, zone->weight, n);
2594 zone->weight = 0;
2595 }
2596
2597 return 0;
2598}
2599
2600/*
2601 * Get a block bit value.
2602 */
2603static int dmz_test_block(struct dmz_metadata *zmd, struct dm_zone *zone,
2604 sector_t chunk_block)
2605{
2606 struct dmz_mblock *mblk;
2607 int ret;
2608
2609 WARN_ON(chunk_block >= zmd->zone_nr_blocks);
2610
2611 /* Get bitmap block */
2612 mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2613 if (IS_ERR(mblk))
2614 return PTR_ERR(mblk);
2615
2616 /* Get offset */
2617 ret = test_bit(chunk_block & DMZ_BLOCK_MASK_BITS,
2618 (unsigned long *) mblk->data) != 0;
2619
2620 dmz_release_mblock(zmd, mblk);
2621
2622 return ret;
2623}
2624
2625/*
2626 * Return the number of blocks from chunk_block to the first block with a bit
2627 * value specified by set. Search at most nr_blocks blocks from chunk_block.
2628 */
2629static int dmz_to_next_set_block(struct dmz_metadata *zmd, struct dm_zone *zone,
2630 sector_t chunk_block, unsigned int nr_blocks,
2631 int set)
2632{
2633 struct dmz_mblock *mblk;
2634 unsigned int bit, set_bit, nr_bits;
2635 unsigned int zone_bits = zmd->zone_bits_per_mblk;
2636 unsigned long *bitmap;
2637 int n = 0;
2638
2639 WARN_ON(chunk_block + nr_blocks > zmd->zone_nr_blocks);
2640
2641 while (nr_blocks) {
2642 /* Get bitmap block */
2643 mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2644 if (IS_ERR(mblk))
2645 return PTR_ERR(mblk);
2646
2647 /* Get offset */
2648 bitmap = (unsigned long *) mblk->data;
2649 bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2650 nr_bits = min(nr_blocks, zone_bits - bit);
2651 if (set)
2652 set_bit = find_next_bit(bitmap, zone_bits, bit);
2653 else
2654 set_bit = find_next_zero_bit(bitmap, zone_bits, bit);
2655 dmz_release_mblock(zmd, mblk);
2656
2657 n += set_bit - bit;
2658 if (set_bit < zone_bits)
2659 break;
2660
2661 nr_blocks -= nr_bits;
2662 chunk_block += nr_bits;
2663 }
2664
2665 return n;
2666}
2667
2668/*
2669 * Test if chunk_block is valid. If it is, the number of consecutive
2670 * valid blocks from chunk_block will be returned.
2671 */
2672int dmz_block_valid(struct dmz_metadata *zmd, struct dm_zone *zone,
2673 sector_t chunk_block)
2674{
2675 int valid;
2676
2677 valid = dmz_test_block(zmd, zone, chunk_block);
2678 if (valid <= 0)
2679 return valid;
2680
2681 /* The block is valid: get the number of valid blocks from block */
2682 return dmz_to_next_set_block(zmd, zone, chunk_block,
2683 zmd->zone_nr_blocks - chunk_block, 0);
2684}
2685
2686/*
2687 * Find the first valid block from @chunk_block in @zone.
2688 * If such a block is found, its number is returned using
2689 * @chunk_block and the total number of valid blocks from @chunk_block
2690 * is returned.
2691 */
2692int dmz_first_valid_block(struct dmz_metadata *zmd, struct dm_zone *zone,
2693 sector_t *chunk_block)
2694{
2695 sector_t start_block = *chunk_block;
2696 int ret;
2697
2698 ret = dmz_to_next_set_block(zmd, zone, start_block,
2699 zmd->zone_nr_blocks - start_block, 1);
2700 if (ret < 0)
2701 return ret;
2702
2703 start_block += ret;
2704 *chunk_block = start_block;
2705
2706 return dmz_to_next_set_block(zmd, zone, start_block,
2707 zmd->zone_nr_blocks - start_block, 0);
2708}
2709
2710/*
2711 * Count the number of bits set starting from bit up to bit + nr_bits - 1.
2712 */
2713static int dmz_count_bits(void *bitmap, int bit, int nr_bits)
2714{
2715 unsigned long *addr;
2716 int end = bit + nr_bits;
2717 int n = 0;
2718
2719 while (bit < end) {
2720 if (((bit & (BITS_PER_LONG - 1)) == 0) &&
2721 ((end - bit) >= BITS_PER_LONG)) {
2722 addr = (unsigned long *)bitmap + BIT_WORD(bit);
2723 if (*addr == ULONG_MAX) {
2724 n += BITS_PER_LONG;
2725 bit += BITS_PER_LONG;
2726 continue;
2727 }
2728 }
2729
2730 if (test_bit(bit, bitmap))
2731 n++;
2732 bit++;
2733 }
2734
2735 return n;
2736}
2737
2738/*
2739 * Get a zone weight.
2740 */
2741static void dmz_get_zone_weight(struct dmz_metadata *zmd, struct dm_zone *zone)
2742{
2743 struct dmz_mblock *mblk;
2744 sector_t chunk_block = 0;
2745 unsigned int bit, nr_bits;
2746 unsigned int nr_blocks = zmd->zone_nr_blocks;
2747 void *bitmap;
2748 int n = 0;
2749
2750 while (nr_blocks) {
2751 /* Get bitmap block */
2752 mblk = dmz_get_bitmap(zmd, zone, chunk_block);
2753 if (IS_ERR(mblk)) {
2754 n = 0;
2755 break;
2756 }
2757
2758 /* Count bits in this block */
2759 bitmap = mblk->data;
2760 bit = chunk_block & DMZ_BLOCK_MASK_BITS;
2761 nr_bits = min(nr_blocks, zmd->zone_bits_per_mblk - bit);
2762 n += dmz_count_bits(bitmap, bit, nr_bits);
2763
2764 dmz_release_mblock(zmd, mblk);
2765
2766 nr_blocks -= nr_bits;
2767 chunk_block += nr_bits;
2768 }
2769
2770 zone->weight = n;
2771}
2772
2773/*
2774 * Cleanup the zoned metadata resources.
2775 */
2776static void dmz_cleanup_metadata(struct dmz_metadata *zmd)
2777{
2778 struct rb_root *root;
2779 struct dmz_mblock *mblk, *next;
2780 int i;
2781
2782 /* Release zone mapping resources */
2783 if (zmd->map_mblk) {
2784 for (i = 0; i < zmd->nr_map_blocks; i++)
2785 dmz_release_mblock(zmd, zmd->map_mblk[i]);
2786 kfree(zmd->map_mblk);
2787 zmd->map_mblk = NULL;
2788 }
2789
2790 /* Release super blocks */
2791 for (i = 0; i < 2; i++) {
2792 if (zmd->sb[i].mblk) {
2793 dmz_free_mblock(zmd, zmd->sb[i].mblk);
2794 zmd->sb[i].mblk = NULL;
2795 }
2796 }
2797
2798 /* Free cached blocks */
2799 while (!list_empty(&zmd->mblk_dirty_list)) {
2800 mblk = list_first_entry(&zmd->mblk_dirty_list,
2801 struct dmz_mblock, link);
2802 dmz_zmd_warn(zmd, "mblock %llu still in dirty list (ref %u)",
2803 (u64)mblk->no, mblk->ref);
2804 list_del_init(&mblk->link);
2805 rb_erase(&mblk->node, &zmd->mblk_rbtree);
2806 dmz_free_mblock(zmd, mblk);
2807 }
2808
2809 while (!list_empty(&zmd->mblk_lru_list)) {
2810 mblk = list_first_entry(&zmd->mblk_lru_list,
2811 struct dmz_mblock, link);
2812 list_del_init(&mblk->link);
2813 rb_erase(&mblk->node, &zmd->mblk_rbtree);
2814 dmz_free_mblock(zmd, mblk);
2815 }
2816
2817 /* Sanity checks: the mblock rbtree should now be empty */
2818 root = &zmd->mblk_rbtree;
2819 rbtree_postorder_for_each_entry_safe(mblk, next, root, node) {
2820 dmz_zmd_warn(zmd, "mblock %llu ref %u still in rbtree",
2821 (u64)mblk->no, mblk->ref);
2822 mblk->ref = 0;
2823 dmz_free_mblock(zmd, mblk);
2824 }
2825
2826 /* Free the zone descriptors */
2827 dmz_drop_zones(zmd);
2828
2829 mutex_destroy(&zmd->mblk_flush_lock);
2830 mutex_destroy(&zmd->map_lock);
2831}
2832
2833static void dmz_print_dev(struct dmz_metadata *zmd, int num)
2834{
2835 struct dmz_dev *dev = &zmd->dev[num];
2836
2837 if (!bdev_is_zoned(dev->bdev))
2838 dmz_dev_info(dev, "Regular block device");
2839 else
2840 dmz_dev_info(dev, "Host-managed zoned block device");
2841
2842 if (zmd->sb_version > 1) {
2843 sector_t sector_offset =
2844 dev->zone_offset << zmd->zone_nr_sectors_shift;
2845
2846 dmz_dev_info(dev, " %llu 512-byte logical sectors (offset %llu)",
2847 (u64)dev->capacity, (u64)sector_offset);
2848 dmz_dev_info(dev, " %u zones of %llu 512-byte logical sectors (offset %llu)",
2849 dev->nr_zones, (u64)zmd->zone_nr_sectors,
2850 (u64)dev->zone_offset);
2851 } else {
2852 dmz_dev_info(dev, " %llu 512-byte logical sectors",
2853 (u64)dev->capacity);
2854 dmz_dev_info(dev, " %u zones of %llu 512-byte logical sectors",
2855 dev->nr_zones, (u64)zmd->zone_nr_sectors);
2856 }
2857}
2858
2859/*
2860 * Initialize the zoned metadata.
2861 */
2862int dmz_ctr_metadata(struct dmz_dev *dev, int num_dev,
2863 struct dmz_metadata **metadata,
2864 const char *devname)
2865{
2866 struct dmz_metadata *zmd;
2867 unsigned int i;
2868 struct dm_zone *zone;
2869 int ret;
2870
2871 zmd = kzalloc(sizeof(struct dmz_metadata), GFP_KERNEL);
2872 if (!zmd)
2873 return -ENOMEM;
2874
2875 strcpy(zmd->devname, devname);
2876 zmd->dev = dev;
2877 zmd->nr_devs = num_dev;
2878 zmd->mblk_rbtree = RB_ROOT;
2879 init_rwsem(&zmd->mblk_sem);
2880 mutex_init(&zmd->mblk_flush_lock);
2881 spin_lock_init(&zmd->mblk_lock);
2882 INIT_LIST_HEAD(&zmd->mblk_lru_list);
2883 INIT_LIST_HEAD(&zmd->mblk_dirty_list);
2884
2885 mutex_init(&zmd->map_lock);
2886
2887 atomic_set(&zmd->unmap_nr_cache, 0);
2888 INIT_LIST_HEAD(&zmd->unmap_cache_list);
2889 INIT_LIST_HEAD(&zmd->map_cache_list);
2890
2891 atomic_set(&zmd->nr_reserved_seq_zones, 0);
2892 INIT_LIST_HEAD(&zmd->reserved_seq_zones_list);
2893
2894 init_waitqueue_head(&zmd->free_wq);
2895
2896 /* Initialize zone descriptors */
2897 ret = dmz_init_zones(zmd);
2898 if (ret)
2899 goto err;
2900
2901 /* Get super block */
2902 ret = dmz_load_sb(zmd);
2903 if (ret)
2904 goto err;
2905
2906 /* Set metadata zones starting from sb_zone */
2907 for (i = 0; i < zmd->nr_meta_zones << 1; i++) {
2908 zone = dmz_get(zmd, zmd->sb[0].zone->id + i);
2909 if (!zone) {
2910 dmz_zmd_err(zmd,
2911 "metadata zone %u not present", i);
2912 ret = -ENXIO;
2913 goto err;
2914 }
2915 if (!dmz_is_rnd(zone) && !dmz_is_cache(zone)) {
2916 dmz_zmd_err(zmd,
2917 "metadata zone %d is not random", i);
2918 ret = -ENXIO;
2919 goto err;
2920 }
2921 set_bit(DMZ_META, &zone->flags);
2922 }
2923 /* Load mapping table */
2924 ret = dmz_load_mapping(zmd);
2925 if (ret)
2926 goto err;
2927
2928 /*
2929 * Cache size boundaries: allow at least 2 super blocks, the chunk map
2930 * blocks and enough blocks to be able to cache the bitmap blocks of
2931 * up to 16 zones when idle (min_nr_mblks). Otherwise, if busy, allow
2932 * the cache to add 512 more metadata blocks.
2933 */
2934 zmd->min_nr_mblks = 2 + zmd->nr_map_blocks + zmd->zone_nr_bitmap_blocks * 16;
2935 zmd->max_nr_mblks = zmd->min_nr_mblks + 512;
2936
2937 /* Metadata cache shrinker */
2938 zmd->mblk_shrinker = shrinker_alloc(0, "dm-zoned-meta:(%u:%u)",
2939 MAJOR(dev->bdev->bd_dev),
2940 MINOR(dev->bdev->bd_dev));
2941 if (!zmd->mblk_shrinker) {
2942 ret = -ENOMEM;
2943 dmz_zmd_err(zmd, "Allocate metadata cache shrinker failed");
2944 goto err;
2945 }
2946
2947 zmd->mblk_shrinker->count_objects = dmz_mblock_shrinker_count;
2948 zmd->mblk_shrinker->scan_objects = dmz_mblock_shrinker_scan;
2949 zmd->mblk_shrinker->private_data = zmd;
2950
2951 shrinker_register(zmd->mblk_shrinker);
2952
2953 dmz_zmd_info(zmd, "DM-Zoned metadata version %d", zmd->sb_version);
2954 for (i = 0; i < zmd->nr_devs; i++)
2955 dmz_print_dev(zmd, i);
2956
2957 dmz_zmd_info(zmd, " %u zones of %llu 512-byte logical sectors",
2958 zmd->nr_zones, (u64)zmd->zone_nr_sectors);
2959 dmz_zmd_debug(zmd, " %u metadata zones",
2960 zmd->nr_meta_zones * 2);
2961 dmz_zmd_debug(zmd, " %u data zones for %u chunks",
2962 zmd->nr_data_zones, zmd->nr_chunks);
2963 dmz_zmd_debug(zmd, " %u cache zones (%u unmapped)",
2964 zmd->nr_cache, atomic_read(&zmd->unmap_nr_cache));
2965 for (i = 0; i < zmd->nr_devs; i++) {
2966 dmz_zmd_debug(zmd, " %u random zones (%u unmapped)",
2967 dmz_nr_rnd_zones(zmd, i),
2968 dmz_nr_unmap_rnd_zones(zmd, i));
2969 dmz_zmd_debug(zmd, " %u sequential zones (%u unmapped)",
2970 dmz_nr_seq_zones(zmd, i),
2971 dmz_nr_unmap_seq_zones(zmd, i));
2972 }
2973 dmz_zmd_debug(zmd, " %u reserved sequential data zones",
2974 zmd->nr_reserved_seq);
2975 dmz_zmd_debug(zmd, "Format:");
2976 dmz_zmd_debug(zmd, "%u metadata blocks per set (%u max cache)",
2977 zmd->nr_meta_blocks, zmd->max_nr_mblks);
2978 dmz_zmd_debug(zmd, " %u data zone mapping blocks",
2979 zmd->nr_map_blocks);
2980 dmz_zmd_debug(zmd, " %u bitmap blocks",
2981 zmd->nr_bitmap_blocks);
2982
2983 *metadata = zmd;
2984
2985 return 0;
2986err:
2987 dmz_cleanup_metadata(zmd);
2988 kfree(zmd);
2989 *metadata = NULL;
2990
2991 return ret;
2992}
2993
2994/*
2995 * Cleanup the zoned metadata resources.
2996 */
2997void dmz_dtr_metadata(struct dmz_metadata *zmd)
2998{
2999 shrinker_free(zmd->mblk_shrinker);
3000 dmz_cleanup_metadata(zmd);
3001 kfree(zmd);
3002}