Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 */
5
6#include <linux/blkdev.h>
7#include <linux/mm.h>
8#include <linux/sched/mm.h>
9#include <linux/slab.h>
10#include <linux/bitmap.h>
11
12#include "dm-core.h"
13
14#define DM_MSG_PREFIX "zone"
15
16#define DM_ZONE_INVALID_WP_OFST UINT_MAX
17
18/*
19 * For internal zone reports bypassing the top BIO submission path.
20 */
21static int dm_blk_do_report_zones(struct mapped_device *md, struct dm_table *t,
22 sector_t sector, unsigned int nr_zones,
23 report_zones_cb cb, void *data)
24{
25 struct gendisk *disk = md->disk;
26 int ret;
27 struct dm_report_zones_args args = {
28 .next_sector = sector,
29 .orig_data = data,
30 .orig_cb = cb,
31 };
32
33 do {
34 struct dm_target *tgt;
35
36 tgt = dm_table_find_target(t, args.next_sector);
37 if (WARN_ON_ONCE(!tgt->type->report_zones))
38 return -EIO;
39
40 args.tgt = tgt;
41 ret = tgt->type->report_zones(tgt, &args,
42 nr_zones - args.zone_idx);
43 if (ret < 0)
44 return ret;
45 } while (args.zone_idx < nr_zones &&
46 args.next_sector < get_capacity(disk));
47
48 return args.zone_idx;
49}
50
51/*
52 * User facing dm device block device report zone operation. This calls the
53 * report_zones operation for each target of a device table. This operation is
54 * generally implemented by targets using dm_report_zones().
55 */
56int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
57 unsigned int nr_zones, report_zones_cb cb, void *data)
58{
59 struct mapped_device *md = disk->private_data;
60 struct dm_table *map;
61 int srcu_idx, ret;
62
63 if (dm_suspended_md(md))
64 return -EAGAIN;
65
66 map = dm_get_live_table(md, &srcu_idx);
67 if (!map)
68 return -EIO;
69
70 ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data);
71
72 dm_put_live_table(md, srcu_idx);
73
74 return ret;
75}
76
77static int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx,
78 void *data)
79{
80 struct dm_report_zones_args *args = data;
81 sector_t sector_diff = args->tgt->begin - args->start;
82
83 /*
84 * Ignore zones beyond the target range.
85 */
86 if (zone->start >= args->start + args->tgt->len)
87 return 0;
88
89 /*
90 * Remap the start sector and write pointer position of the zone
91 * to match its position in the target range.
92 */
93 zone->start += sector_diff;
94 if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
95 if (zone->cond == BLK_ZONE_COND_FULL)
96 zone->wp = zone->start + zone->len;
97 else if (zone->cond == BLK_ZONE_COND_EMPTY)
98 zone->wp = zone->start;
99 else
100 zone->wp += sector_diff;
101 }
102
103 args->next_sector = zone->start + zone->len;
104 return args->orig_cb(zone, args->zone_idx++, args->orig_data);
105}
106
107/*
108 * Helper for drivers of zoned targets to implement struct target_type
109 * report_zones operation.
110 */
111int dm_report_zones(struct block_device *bdev, sector_t start, sector_t sector,
112 struct dm_report_zones_args *args, unsigned int nr_zones)
113{
114 /*
115 * Set the target mapping start sector first so that
116 * dm_report_zones_cb() can correctly remap zone information.
117 */
118 args->start = start;
119
120 return blkdev_report_zones(bdev, sector, nr_zones,
121 dm_report_zones_cb, args);
122}
123EXPORT_SYMBOL_GPL(dm_report_zones);
124
125bool dm_is_zone_write(struct mapped_device *md, struct bio *bio)
126{
127 struct request_queue *q = md->queue;
128
129 if (!blk_queue_is_zoned(q))
130 return false;
131
132 switch (bio_op(bio)) {
133 case REQ_OP_WRITE_ZEROES:
134 case REQ_OP_WRITE:
135 return !op_is_flush(bio->bi_opf) && bio_sectors(bio);
136 default:
137 return false;
138 }
139}
140
141void dm_cleanup_zoned_dev(struct mapped_device *md)
142{
143 if (md->disk) {
144 bitmap_free(md->disk->conv_zones_bitmap);
145 md->disk->conv_zones_bitmap = NULL;
146 bitmap_free(md->disk->seq_zones_wlock);
147 md->disk->seq_zones_wlock = NULL;
148 }
149
150 kvfree(md->zwp_offset);
151 md->zwp_offset = NULL;
152 md->nr_zones = 0;
153}
154
155static unsigned int dm_get_zone_wp_offset(struct blk_zone *zone)
156{
157 switch (zone->cond) {
158 case BLK_ZONE_COND_IMP_OPEN:
159 case BLK_ZONE_COND_EXP_OPEN:
160 case BLK_ZONE_COND_CLOSED:
161 return zone->wp - zone->start;
162 case BLK_ZONE_COND_FULL:
163 return zone->len;
164 case BLK_ZONE_COND_EMPTY:
165 case BLK_ZONE_COND_NOT_WP:
166 case BLK_ZONE_COND_OFFLINE:
167 case BLK_ZONE_COND_READONLY:
168 default:
169 /*
170 * Conventional, offline and read-only zones do not have a valid
171 * write pointer. Use 0 as for an empty zone.
172 */
173 return 0;
174 }
175}
176
177static int dm_zone_revalidate_cb(struct blk_zone *zone, unsigned int idx,
178 void *data)
179{
180 struct mapped_device *md = data;
181 struct gendisk *disk = md->disk;
182
183 switch (zone->type) {
184 case BLK_ZONE_TYPE_CONVENTIONAL:
185 if (!disk->conv_zones_bitmap) {
186 disk->conv_zones_bitmap = bitmap_zalloc(disk->nr_zones,
187 GFP_NOIO);
188 if (!disk->conv_zones_bitmap)
189 return -ENOMEM;
190 }
191 set_bit(idx, disk->conv_zones_bitmap);
192 break;
193 case BLK_ZONE_TYPE_SEQWRITE_REQ:
194 case BLK_ZONE_TYPE_SEQWRITE_PREF:
195 if (!disk->seq_zones_wlock) {
196 disk->seq_zones_wlock = bitmap_zalloc(disk->nr_zones,
197 GFP_NOIO);
198 if (!disk->seq_zones_wlock)
199 return -ENOMEM;
200 }
201 if (!md->zwp_offset) {
202 md->zwp_offset =
203 kvcalloc(disk->nr_zones, sizeof(unsigned int),
204 GFP_KERNEL);
205 if (!md->zwp_offset)
206 return -ENOMEM;
207 }
208 md->zwp_offset[idx] = dm_get_zone_wp_offset(zone);
209
210 break;
211 default:
212 DMERR("Invalid zone type 0x%x at sectors %llu",
213 (int)zone->type, zone->start);
214 return -ENODEV;
215 }
216
217 return 0;
218}
219
220/*
221 * Revalidate the zones of a mapped device to initialize resource necessary
222 * for zone append emulation. Note that we cannot simply use the block layer
223 * blk_revalidate_disk_zones() function here as the mapped device is suspended
224 * (this is called from __bind() context).
225 */
226static int dm_revalidate_zones(struct mapped_device *md, struct dm_table *t)
227{
228 struct gendisk *disk = md->disk;
229 unsigned int noio_flag;
230 int ret;
231
232 /*
233 * Check if something changed. If yes, cleanup the current resources
234 * and reallocate everything.
235 */
236 if (!disk->nr_zones || disk->nr_zones != md->nr_zones)
237 dm_cleanup_zoned_dev(md);
238 if (md->nr_zones)
239 return 0;
240
241 /*
242 * Scan all zones to initialize everything. Ensure that all vmalloc
243 * operations in this context are done as if GFP_NOIO was specified.
244 */
245 noio_flag = memalloc_noio_save();
246 ret = dm_blk_do_report_zones(md, t, 0, disk->nr_zones,
247 dm_zone_revalidate_cb, md);
248 memalloc_noio_restore(noio_flag);
249 if (ret < 0)
250 goto err;
251 if (ret != disk->nr_zones) {
252 ret = -EIO;
253 goto err;
254 }
255
256 md->nr_zones = disk->nr_zones;
257
258 return 0;
259
260err:
261 DMERR("Revalidate zones failed %d", ret);
262 dm_cleanup_zoned_dev(md);
263 return ret;
264}
265
266static int device_not_zone_append_capable(struct dm_target *ti,
267 struct dm_dev *dev, sector_t start,
268 sector_t len, void *data)
269{
270 return !bdev_is_zoned(dev->bdev);
271}
272
273static bool dm_table_supports_zone_append(struct dm_table *t)
274{
275 for (unsigned int i = 0; i < t->num_targets; i++) {
276 struct dm_target *ti = dm_table_get_target(t, i);
277
278 if (ti->emulate_zone_append)
279 return false;
280
281 if (!ti->type->iterate_devices ||
282 ti->type->iterate_devices(ti, device_not_zone_append_capable, NULL))
283 return false;
284 }
285
286 return true;
287}
288
289int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q)
290{
291 struct mapped_device *md = t->md;
292
293 /*
294 * For a zoned target, the number of zones should be updated for the
295 * correct value to be exposed in sysfs queue/nr_zones.
296 */
297 WARN_ON_ONCE(queue_is_mq(q));
298 md->disk->nr_zones = bdev_nr_zones(md->disk->part0);
299
300 /* Check if zone append is natively supported */
301 if (dm_table_supports_zone_append(t)) {
302 clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
303 dm_cleanup_zoned_dev(md);
304 return 0;
305 }
306
307 /*
308 * Mark the mapped device as needing zone append emulation and
309 * initialize the emulation resources once the capacity is set.
310 */
311 set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
312 if (!get_capacity(md->disk))
313 return 0;
314
315 return dm_revalidate_zones(md, t);
316}
317
318static int dm_update_zone_wp_offset_cb(struct blk_zone *zone, unsigned int idx,
319 void *data)
320{
321 unsigned int *wp_offset = data;
322
323 *wp_offset = dm_get_zone_wp_offset(zone);
324
325 return 0;
326}
327
328static int dm_update_zone_wp_offset(struct mapped_device *md, unsigned int zno,
329 unsigned int *wp_ofst)
330{
331 sector_t sector = zno * bdev_zone_sectors(md->disk->part0);
332 unsigned int noio_flag;
333 struct dm_table *t;
334 int srcu_idx, ret;
335
336 t = dm_get_live_table(md, &srcu_idx);
337 if (!t)
338 return -EIO;
339
340 /*
341 * Ensure that all memory allocations in this context are done as if
342 * GFP_NOIO was specified.
343 */
344 noio_flag = memalloc_noio_save();
345 ret = dm_blk_do_report_zones(md, t, sector, 1,
346 dm_update_zone_wp_offset_cb, wp_ofst);
347 memalloc_noio_restore(noio_flag);
348
349 dm_put_live_table(md, srcu_idx);
350
351 if (ret != 1)
352 return -EIO;
353
354 return 0;
355}
356
357struct orig_bio_details {
358 enum req_op op;
359 unsigned int nr_sectors;
360};
361
362/*
363 * First phase of BIO mapping for targets with zone append emulation:
364 * check all BIO that change a zone writer pointer and change zone
365 * append operations into regular write operations.
366 */
367static bool dm_zone_map_bio_begin(struct mapped_device *md,
368 unsigned int zno, struct bio *clone)
369{
370 sector_t zsectors = bdev_zone_sectors(md->disk->part0);
371 unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]);
372
373 /*
374 * If the target zone is in an error state, recover by inspecting the
375 * zone to get its current write pointer position. Note that since the
376 * target zone is already locked, a BIO issuing context should never
377 * see the zone write in the DM_ZONE_UPDATING_WP_OFST state.
378 */
379 if (zwp_offset == DM_ZONE_INVALID_WP_OFST) {
380 if (dm_update_zone_wp_offset(md, zno, &zwp_offset))
381 return false;
382 WRITE_ONCE(md->zwp_offset[zno], zwp_offset);
383 }
384
385 switch (bio_op(clone)) {
386 case REQ_OP_ZONE_RESET:
387 case REQ_OP_ZONE_FINISH:
388 return true;
389 case REQ_OP_WRITE_ZEROES:
390 case REQ_OP_WRITE:
391 /* Writes must be aligned to the zone write pointer */
392 if ((clone->bi_iter.bi_sector & (zsectors - 1)) != zwp_offset)
393 return false;
394 break;
395 case REQ_OP_ZONE_APPEND:
396 /*
397 * Change zone append operations into a non-mergeable regular
398 * writes directed at the current write pointer position of the
399 * target zone.
400 */
401 clone->bi_opf = REQ_OP_WRITE | REQ_NOMERGE |
402 (clone->bi_opf & (~REQ_OP_MASK));
403 clone->bi_iter.bi_sector += zwp_offset;
404 break;
405 default:
406 DMWARN_LIMIT("Invalid BIO operation");
407 return false;
408 }
409
410 /* Cannot write to a full zone */
411 if (zwp_offset >= zsectors)
412 return false;
413
414 return true;
415}
416
417/*
418 * Second phase of BIO mapping for targets with zone append emulation:
419 * update the zone write pointer offset array to account for the additional
420 * data written to a zone. Note that at this point, the remapped clone BIO
421 * may already have completed, so we do not touch it.
422 */
423static blk_status_t dm_zone_map_bio_end(struct mapped_device *md, unsigned int zno,
424 struct orig_bio_details *orig_bio_details,
425 unsigned int nr_sectors)
426{
427 unsigned int zwp_offset = READ_ONCE(md->zwp_offset[zno]);
428
429 /* The clone BIO may already have been completed and failed */
430 if (zwp_offset == DM_ZONE_INVALID_WP_OFST)
431 return BLK_STS_IOERR;
432
433 /* Update the zone wp offset */
434 switch (orig_bio_details->op) {
435 case REQ_OP_ZONE_RESET:
436 WRITE_ONCE(md->zwp_offset[zno], 0);
437 return BLK_STS_OK;
438 case REQ_OP_ZONE_FINISH:
439 WRITE_ONCE(md->zwp_offset[zno],
440 bdev_zone_sectors(md->disk->part0));
441 return BLK_STS_OK;
442 case REQ_OP_WRITE_ZEROES:
443 case REQ_OP_WRITE:
444 WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors);
445 return BLK_STS_OK;
446 case REQ_OP_ZONE_APPEND:
447 /*
448 * Check that the target did not truncate the write operation
449 * emulating a zone append.
450 */
451 if (nr_sectors != orig_bio_details->nr_sectors) {
452 DMWARN_LIMIT("Truncated write for zone append");
453 return BLK_STS_IOERR;
454 }
455 WRITE_ONCE(md->zwp_offset[zno], zwp_offset + nr_sectors);
456 return BLK_STS_OK;
457 default:
458 DMWARN_LIMIT("Invalid BIO operation");
459 return BLK_STS_IOERR;
460 }
461}
462
463static inline void dm_zone_lock(struct gendisk *disk, unsigned int zno,
464 struct bio *clone)
465{
466 if (WARN_ON_ONCE(bio_flagged(clone, BIO_ZONE_WRITE_LOCKED)))
467 return;
468
469 wait_on_bit_lock_io(disk->seq_zones_wlock, zno, TASK_UNINTERRUPTIBLE);
470 bio_set_flag(clone, BIO_ZONE_WRITE_LOCKED);
471}
472
473static inline void dm_zone_unlock(struct gendisk *disk, unsigned int zno,
474 struct bio *clone)
475{
476 if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))
477 return;
478
479 WARN_ON_ONCE(!test_bit(zno, disk->seq_zones_wlock));
480 clear_bit_unlock(zno, disk->seq_zones_wlock);
481 smp_mb__after_atomic();
482 wake_up_bit(disk->seq_zones_wlock, zno);
483
484 bio_clear_flag(clone, BIO_ZONE_WRITE_LOCKED);
485}
486
487static bool dm_need_zone_wp_tracking(struct bio *bio)
488{
489 /*
490 * Special processing is not needed for operations that do not need the
491 * zone write lock, that is, all operations that target conventional
492 * zones and all operations that do not modify directly a sequential
493 * zone write pointer.
494 */
495 if (op_is_flush(bio->bi_opf) && !bio_sectors(bio))
496 return false;
497 switch (bio_op(bio)) {
498 case REQ_OP_WRITE_ZEROES:
499 case REQ_OP_WRITE:
500 case REQ_OP_ZONE_RESET:
501 case REQ_OP_ZONE_FINISH:
502 case REQ_OP_ZONE_APPEND:
503 return bio_zone_is_seq(bio);
504 default:
505 return false;
506 }
507}
508
509/*
510 * Special IO mapping for targets needing zone append emulation.
511 */
512int dm_zone_map_bio(struct dm_target_io *tio)
513{
514 struct dm_io *io = tio->io;
515 struct dm_target *ti = tio->ti;
516 struct mapped_device *md = io->md;
517 struct bio *clone = &tio->clone;
518 struct orig_bio_details orig_bio_details;
519 unsigned int zno;
520 blk_status_t sts;
521 int r;
522
523 /*
524 * IOs that do not change a zone write pointer do not need
525 * any additional special processing.
526 */
527 if (!dm_need_zone_wp_tracking(clone))
528 return ti->type->map(ti, clone);
529
530 /* Lock the target zone */
531 zno = bio_zone_no(clone);
532 dm_zone_lock(md->disk, zno, clone);
533
534 orig_bio_details.nr_sectors = bio_sectors(clone);
535 orig_bio_details.op = bio_op(clone);
536
537 /*
538 * Check that the bio and the target zone write pointer offset are
539 * both valid, and if the bio is a zone append, remap it to a write.
540 */
541 if (!dm_zone_map_bio_begin(md, zno, clone)) {
542 dm_zone_unlock(md->disk, zno, clone);
543 return DM_MAPIO_KILL;
544 }
545
546 /* Let the target do its work */
547 r = ti->type->map(ti, clone);
548 switch (r) {
549 case DM_MAPIO_SUBMITTED:
550 /*
551 * The target submitted the clone BIO. The target zone will
552 * be unlocked on completion of the clone.
553 */
554 sts = dm_zone_map_bio_end(md, zno, &orig_bio_details,
555 *tio->len_ptr);
556 break;
557 case DM_MAPIO_REMAPPED:
558 /*
559 * The target only remapped the clone BIO. In case of error,
560 * unlock the target zone here as the clone will not be
561 * submitted.
562 */
563 sts = dm_zone_map_bio_end(md, zno, &orig_bio_details,
564 *tio->len_ptr);
565 if (sts != BLK_STS_OK)
566 dm_zone_unlock(md->disk, zno, clone);
567 break;
568 case DM_MAPIO_REQUEUE:
569 case DM_MAPIO_KILL:
570 default:
571 dm_zone_unlock(md->disk, zno, clone);
572 sts = BLK_STS_IOERR;
573 break;
574 }
575
576 if (sts != BLK_STS_OK)
577 return DM_MAPIO_KILL;
578
579 return r;
580}
581
582/*
583 * IO completion callback called from clone_endio().
584 */
585void dm_zone_endio(struct dm_io *io, struct bio *clone)
586{
587 struct mapped_device *md = io->md;
588 struct gendisk *disk = md->disk;
589 struct bio *orig_bio = io->orig_bio;
590 unsigned int zwp_offset;
591 unsigned int zno;
592
593 /*
594 * For targets that do not emulate zone append, we only need to
595 * handle native zone-append bios.
596 */
597 if (!dm_emulate_zone_append(md)) {
598 /*
599 * Get the offset within the zone of the written sector
600 * and add that to the original bio sector position.
601 */
602 if (clone->bi_status == BLK_STS_OK &&
603 bio_op(clone) == REQ_OP_ZONE_APPEND) {
604 sector_t mask =
605 (sector_t)bdev_zone_sectors(disk->part0) - 1;
606
607 orig_bio->bi_iter.bi_sector +=
608 clone->bi_iter.bi_sector & mask;
609 }
610
611 return;
612 }
613
614 /*
615 * For targets that do emulate zone append, if the clone BIO does not
616 * own the target zone write lock, we have nothing to do.
617 */
618 if (!bio_flagged(clone, BIO_ZONE_WRITE_LOCKED))
619 return;
620
621 zno = bio_zone_no(orig_bio);
622
623 if (clone->bi_status != BLK_STS_OK) {
624 /*
625 * BIOs that modify a zone write pointer may leave the zone
626 * in an unknown state in case of failure (e.g. the write
627 * pointer was only partially advanced). In this case, set
628 * the target zone write pointer as invalid unless it is
629 * already being updated.
630 */
631 WRITE_ONCE(md->zwp_offset[zno], DM_ZONE_INVALID_WP_OFST);
632 } else if (bio_op(orig_bio) == REQ_OP_ZONE_APPEND) {
633 /*
634 * Get the written sector for zone append operation that were
635 * emulated using regular write operations.
636 */
637 zwp_offset = READ_ONCE(md->zwp_offset[zno]);
638 if (WARN_ON_ONCE(zwp_offset < bio_sectors(orig_bio)))
639 WRITE_ONCE(md->zwp_offset[zno],
640 DM_ZONE_INVALID_WP_OFST);
641 else
642 orig_bio->bi_iter.bi_sector +=
643 zwp_offset - bio_sectors(orig_bio);
644 }
645
646 dm_zone_unlock(disk, zno, clone);
647}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 */
5
6#include <linux/blkdev.h>
7#include <linux/mm.h>
8#include <linux/sched/mm.h>
9#include <linux/slab.h>
10#include <linux/bitmap.h>
11
12#include "dm-core.h"
13
14#define DM_MSG_PREFIX "zone"
15
16/*
17 * For internal zone reports bypassing the top BIO submission path.
18 */
19static int dm_blk_do_report_zones(struct mapped_device *md, struct dm_table *t,
20 sector_t sector, unsigned int nr_zones,
21 report_zones_cb cb, void *data)
22{
23 struct gendisk *disk = md->disk;
24 int ret;
25 struct dm_report_zones_args args = {
26 .next_sector = sector,
27 .orig_data = data,
28 .orig_cb = cb,
29 };
30
31 do {
32 struct dm_target *tgt;
33
34 tgt = dm_table_find_target(t, args.next_sector);
35 if (WARN_ON_ONCE(!tgt->type->report_zones))
36 return -EIO;
37
38 args.tgt = tgt;
39 ret = tgt->type->report_zones(tgt, &args,
40 nr_zones - args.zone_idx);
41 if (ret < 0)
42 return ret;
43 } while (args.zone_idx < nr_zones &&
44 args.next_sector < get_capacity(disk));
45
46 return args.zone_idx;
47}
48
49/*
50 * User facing dm device block device report zone operation. This calls the
51 * report_zones operation for each target of a device table. This operation is
52 * generally implemented by targets using dm_report_zones().
53 */
54int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
55 unsigned int nr_zones, report_zones_cb cb, void *data)
56{
57 struct mapped_device *md = disk->private_data;
58 struct dm_table *map;
59 int srcu_idx, ret;
60
61 if (!md->zone_revalidate_map) {
62 /* Regular user context */
63 if (dm_suspended_md(md))
64 return -EAGAIN;
65
66 map = dm_get_live_table(md, &srcu_idx);
67 if (!map)
68 return -EIO;
69 } else {
70 /* Zone revalidation during __bind() */
71 map = md->zone_revalidate_map;
72 }
73
74 ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data);
75
76 if (!md->zone_revalidate_map)
77 dm_put_live_table(md, srcu_idx);
78
79 return ret;
80}
81
82static int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx,
83 void *data)
84{
85 struct dm_report_zones_args *args = data;
86 sector_t sector_diff = args->tgt->begin - args->start;
87
88 /*
89 * Ignore zones beyond the target range.
90 */
91 if (zone->start >= args->start + args->tgt->len)
92 return 0;
93
94 /*
95 * Remap the start sector and write pointer position of the zone
96 * to match its position in the target range.
97 */
98 zone->start += sector_diff;
99 if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
100 if (zone->cond == BLK_ZONE_COND_FULL)
101 zone->wp = zone->start + zone->len;
102 else if (zone->cond == BLK_ZONE_COND_EMPTY)
103 zone->wp = zone->start;
104 else
105 zone->wp += sector_diff;
106 }
107
108 args->next_sector = zone->start + zone->len;
109 return args->orig_cb(zone, args->zone_idx++, args->orig_data);
110}
111
112/*
113 * Helper for drivers of zoned targets to implement struct target_type
114 * report_zones operation.
115 */
116int dm_report_zones(struct block_device *bdev, sector_t start, sector_t sector,
117 struct dm_report_zones_args *args, unsigned int nr_zones)
118{
119 /*
120 * Set the target mapping start sector first so that
121 * dm_report_zones_cb() can correctly remap zone information.
122 */
123 args->start = start;
124
125 return blkdev_report_zones(bdev, sector, nr_zones,
126 dm_report_zones_cb, args);
127}
128EXPORT_SYMBOL_GPL(dm_report_zones);
129
130bool dm_is_zone_write(struct mapped_device *md, struct bio *bio)
131{
132 struct request_queue *q = md->queue;
133
134 if (!blk_queue_is_zoned(q))
135 return false;
136
137 switch (bio_op(bio)) {
138 case REQ_OP_WRITE_ZEROES:
139 case REQ_OP_WRITE:
140 return !op_is_flush(bio->bi_opf) && bio_sectors(bio);
141 default:
142 return false;
143 }
144}
145
146/*
147 * Revalidate the zones of a mapped device to initialize resource necessary
148 * for zone append emulation. Note that we cannot simply use the block layer
149 * blk_revalidate_disk_zones() function here as the mapped device is suspended
150 * (this is called from __bind() context).
151 */
152int dm_revalidate_zones(struct dm_table *t, struct request_queue *q)
153{
154 struct mapped_device *md = t->md;
155 struct gendisk *disk = md->disk;
156 int ret;
157
158 if (!get_capacity(disk))
159 return 0;
160
161 /* Revalidate only if something changed. */
162 if (!disk->nr_zones || disk->nr_zones != md->nr_zones) {
163 DMINFO("%s using %s zone append",
164 disk->disk_name,
165 queue_emulates_zone_append(q) ? "emulated" : "native");
166 md->nr_zones = 0;
167 }
168
169 if (md->nr_zones)
170 return 0;
171
172 /*
173 * Our table is not live yet. So the call to dm_get_live_table()
174 * in dm_blk_report_zones() will fail. Set a temporary pointer to
175 * our table for dm_blk_report_zones() to use directly.
176 */
177 md->zone_revalidate_map = t;
178 ret = blk_revalidate_disk_zones(disk);
179 md->zone_revalidate_map = NULL;
180
181 if (ret) {
182 DMERR("Revalidate zones failed %d", ret);
183 return ret;
184 }
185
186 md->nr_zones = disk->nr_zones;
187
188 return 0;
189}
190
191static int device_not_zone_append_capable(struct dm_target *ti,
192 struct dm_dev *dev, sector_t start,
193 sector_t len, void *data)
194{
195 return !bdev_is_zoned(dev->bdev);
196}
197
198static bool dm_table_supports_zone_append(struct dm_table *t)
199{
200 for (unsigned int i = 0; i < t->num_targets; i++) {
201 struct dm_target *ti = dm_table_get_target(t, i);
202
203 if (ti->emulate_zone_append)
204 return false;
205
206 if (!ti->type->iterate_devices ||
207 ti->type->iterate_devices(ti, device_not_zone_append_capable, NULL))
208 return false;
209 }
210
211 return true;
212}
213
214struct dm_device_zone_count {
215 sector_t start;
216 sector_t len;
217 unsigned int total_nr_seq_zones;
218 unsigned int target_nr_seq_zones;
219};
220
221/*
222 * Count the total number of and the number of mapped sequential zones of a
223 * target zoned device.
224 */
225static int dm_device_count_zones_cb(struct blk_zone *zone,
226 unsigned int idx, void *data)
227{
228 struct dm_device_zone_count *zc = data;
229
230 if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
231 zc->total_nr_seq_zones++;
232 if (zone->start >= zc->start &&
233 zone->start < zc->start + zc->len)
234 zc->target_nr_seq_zones++;
235 }
236
237 return 0;
238}
239
240static int dm_device_count_zones(struct dm_dev *dev,
241 struct dm_device_zone_count *zc)
242{
243 int ret;
244
245 ret = blkdev_report_zones(dev->bdev, 0, BLK_ALL_ZONES,
246 dm_device_count_zones_cb, zc);
247 if (ret < 0)
248 return ret;
249 if (!ret)
250 return -EIO;
251 return 0;
252}
253
254struct dm_zone_resource_limits {
255 unsigned int mapped_nr_seq_zones;
256 struct queue_limits *lim;
257 bool reliable_limits;
258};
259
260static int device_get_zone_resource_limits(struct dm_target *ti,
261 struct dm_dev *dev, sector_t start,
262 sector_t len, void *data)
263{
264 struct dm_zone_resource_limits *zlim = data;
265 struct gendisk *disk = dev->bdev->bd_disk;
266 unsigned int max_open_zones, max_active_zones;
267 int ret;
268 struct dm_device_zone_count zc = {
269 .start = start,
270 .len = len,
271 };
272
273 /*
274 * If the target is not the whole device, the device zone resources may
275 * be shared between different targets. Check this by counting the
276 * number of mapped sequential zones: if this number is smaller than the
277 * total number of sequential zones of the target device, then resource
278 * sharing may happen and the zone limits will not be reliable.
279 */
280 ret = dm_device_count_zones(dev, &zc);
281 if (ret) {
282 DMERR("Count %s zones failed %d", disk->disk_name, ret);
283 return ret;
284 }
285
286 /*
287 * If the target does not map any sequential zones, then we do not need
288 * any zone resource limits.
289 */
290 if (!zc.target_nr_seq_zones)
291 return 0;
292
293 /*
294 * If the target does not map all sequential zones, the limits
295 * will not be reliable and we cannot use REQ_OP_ZONE_RESET_ALL.
296 */
297 if (zc.target_nr_seq_zones < zc.total_nr_seq_zones) {
298 zlim->reliable_limits = false;
299 ti->zone_reset_all_supported = false;
300 }
301
302 /*
303 * If the target maps less sequential zones than the limit values, then
304 * we do not have limits for this target.
305 */
306 max_active_zones = disk->queue->limits.max_active_zones;
307 if (max_active_zones >= zc.target_nr_seq_zones)
308 max_active_zones = 0;
309 zlim->lim->max_active_zones =
310 min_not_zero(max_active_zones, zlim->lim->max_active_zones);
311
312 max_open_zones = disk->queue->limits.max_open_zones;
313 if (max_open_zones >= zc.target_nr_seq_zones)
314 max_open_zones = 0;
315 zlim->lim->max_open_zones =
316 min_not_zero(max_open_zones, zlim->lim->max_open_zones);
317
318 /*
319 * Also count the total number of sequential zones for the mapped
320 * device so that when we are done inspecting all its targets, we are
321 * able to check if the mapped device actually has any sequential zones.
322 */
323 zlim->mapped_nr_seq_zones += zc.target_nr_seq_zones;
324
325 return 0;
326}
327
328int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q,
329 struct queue_limits *lim)
330{
331 struct mapped_device *md = t->md;
332 struct gendisk *disk = md->disk;
333 struct dm_zone_resource_limits zlim = {
334 .reliable_limits = true,
335 .lim = lim,
336 };
337
338 /*
339 * Check if zone append is natively supported, and if not, set the
340 * mapped device queue as needing zone append emulation.
341 */
342 WARN_ON_ONCE(queue_is_mq(q));
343 if (dm_table_supports_zone_append(t)) {
344 clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
345 } else {
346 set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
347 lim->max_hw_zone_append_sectors = 0;
348 }
349
350 /*
351 * Determine the max open and max active zone limits for the mapped
352 * device by inspecting the zone resource limits and the zones mapped
353 * by each target.
354 */
355 for (unsigned int i = 0; i < t->num_targets; i++) {
356 struct dm_target *ti = dm_table_get_target(t, i);
357
358 /*
359 * Assume that the target can accept REQ_OP_ZONE_RESET_ALL.
360 * device_get_zone_resource_limits() may adjust this if one of
361 * the device used by the target does not have all its
362 * sequential write required zones mapped.
363 */
364 ti->zone_reset_all_supported = true;
365
366 if (!ti->type->iterate_devices ||
367 ti->type->iterate_devices(ti,
368 device_get_zone_resource_limits, &zlim)) {
369 DMERR("Could not determine %s zone resource limits",
370 disk->disk_name);
371 return -ENODEV;
372 }
373 }
374
375 /*
376 * If we only have conventional zones mapped, expose the mapped device
377 + as a regular device.
378 */
379 if (!zlim.mapped_nr_seq_zones) {
380 lim->max_open_zones = 0;
381 lim->max_active_zones = 0;
382 lim->max_hw_zone_append_sectors = 0;
383 lim->zone_write_granularity = 0;
384 lim->chunk_sectors = 0;
385 lim->features &= ~BLK_FEAT_ZONED;
386 clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
387 md->nr_zones = 0;
388 disk->nr_zones = 0;
389 return 0;
390 }
391
392 /*
393 * Warn once (when the capacity is not yet set) if the mapped device is
394 * partially using zone resources of the target devices as that leads to
395 * unreliable limits, i.e. if another mapped device uses the same
396 * underlying devices, we cannot enforce zone limits to guarantee that
397 * writing will not lead to errors. Note that we really should return
398 * an error for such case but there is no easy way to find out if
399 * another mapped device uses the same underlying zoned devices.
400 */
401 if (!get_capacity(disk) && !zlim.reliable_limits)
402 DMWARN("%s zone resource limits may be unreliable",
403 disk->disk_name);
404
405 if (lim->features & BLK_FEAT_ZONED &&
406 !static_key_enabled(&zoned_enabled.key))
407 static_branch_enable(&zoned_enabled);
408 return 0;
409}
410
411/*
412 * IO completion callback called from clone_endio().
413 */
414void dm_zone_endio(struct dm_io *io, struct bio *clone)
415{
416 struct mapped_device *md = io->md;
417 struct gendisk *disk = md->disk;
418 struct bio *orig_bio = io->orig_bio;
419
420 /*
421 * Get the offset within the zone of the written sector
422 * and add that to the original bio sector position.
423 */
424 if (clone->bi_status == BLK_STS_OK &&
425 bio_op(clone) == REQ_OP_ZONE_APPEND) {
426 sector_t mask = bdev_zone_sectors(disk->part0) - 1;
427
428 orig_bio->bi_iter.bi_sector += clone->bi_iter.bi_sector & mask;
429 }
430
431 return;
432}
433
434static int dm_zone_need_reset_cb(struct blk_zone *zone, unsigned int idx,
435 void *data)
436{
437 /*
438 * For an all-zones reset, ignore conventional, empty, read-only
439 * and offline zones.
440 */
441 switch (zone->cond) {
442 case BLK_ZONE_COND_NOT_WP:
443 case BLK_ZONE_COND_EMPTY:
444 case BLK_ZONE_COND_READONLY:
445 case BLK_ZONE_COND_OFFLINE:
446 return 0;
447 default:
448 set_bit(idx, (unsigned long *)data);
449 return 0;
450 }
451}
452
453int dm_zone_get_reset_bitmap(struct mapped_device *md, struct dm_table *t,
454 sector_t sector, unsigned int nr_zones,
455 unsigned long *need_reset)
456{
457 int ret;
458
459 ret = dm_blk_do_report_zones(md, t, sector, nr_zones,
460 dm_zone_need_reset_cb, need_reset);
461 if (ret != nr_zones) {
462 DMERR("Get %s zone reset bitmap failed\n",
463 md->disk->disk_name);
464 return -EIO;
465 }
466
467 return 0;
468}