Loading...
Note: File does not exist in v3.1.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright (C) 2021 Western Digital Corporation or its affiliates.
4 */
5
6#include <linux/blkdev.h>
7#include <linux/mm.h>
8#include <linux/sched/mm.h>
9#include <linux/slab.h>
10#include <linux/bitmap.h>
11
12#include "dm-core.h"
13
14#define DM_MSG_PREFIX "zone"
15
16/*
17 * For internal zone reports bypassing the top BIO submission path.
18 */
19static int dm_blk_do_report_zones(struct mapped_device *md, struct dm_table *t,
20 sector_t sector, unsigned int nr_zones,
21 report_zones_cb cb, void *data)
22{
23 struct gendisk *disk = md->disk;
24 int ret;
25 struct dm_report_zones_args args = {
26 .next_sector = sector,
27 .orig_data = data,
28 .orig_cb = cb,
29 };
30
31 do {
32 struct dm_target *tgt;
33
34 tgt = dm_table_find_target(t, args.next_sector);
35 if (WARN_ON_ONCE(!tgt->type->report_zones))
36 return -EIO;
37
38 args.tgt = tgt;
39 ret = tgt->type->report_zones(tgt, &args,
40 nr_zones - args.zone_idx);
41 if (ret < 0)
42 return ret;
43 } while (args.zone_idx < nr_zones &&
44 args.next_sector < get_capacity(disk));
45
46 return args.zone_idx;
47}
48
49/*
50 * User facing dm device block device report zone operation. This calls the
51 * report_zones operation for each target of a device table. This operation is
52 * generally implemented by targets using dm_report_zones().
53 */
54int dm_blk_report_zones(struct gendisk *disk, sector_t sector,
55 unsigned int nr_zones, report_zones_cb cb, void *data)
56{
57 struct mapped_device *md = disk->private_data;
58 struct dm_table *map;
59 int srcu_idx, ret;
60
61 if (!md->zone_revalidate_map) {
62 /* Regular user context */
63 if (dm_suspended_md(md))
64 return -EAGAIN;
65
66 map = dm_get_live_table(md, &srcu_idx);
67 if (!map)
68 return -EIO;
69 } else {
70 /* Zone revalidation during __bind() */
71 map = md->zone_revalidate_map;
72 }
73
74 ret = dm_blk_do_report_zones(md, map, sector, nr_zones, cb, data);
75
76 if (!md->zone_revalidate_map)
77 dm_put_live_table(md, srcu_idx);
78
79 return ret;
80}
81
82static int dm_report_zones_cb(struct blk_zone *zone, unsigned int idx,
83 void *data)
84{
85 struct dm_report_zones_args *args = data;
86 sector_t sector_diff = args->tgt->begin - args->start;
87
88 /*
89 * Ignore zones beyond the target range.
90 */
91 if (zone->start >= args->start + args->tgt->len)
92 return 0;
93
94 /*
95 * Remap the start sector and write pointer position of the zone
96 * to match its position in the target range.
97 */
98 zone->start += sector_diff;
99 if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
100 if (zone->cond == BLK_ZONE_COND_FULL)
101 zone->wp = zone->start + zone->len;
102 else if (zone->cond == BLK_ZONE_COND_EMPTY)
103 zone->wp = zone->start;
104 else
105 zone->wp += sector_diff;
106 }
107
108 args->next_sector = zone->start + zone->len;
109 return args->orig_cb(zone, args->zone_idx++, args->orig_data);
110}
111
112/*
113 * Helper for drivers of zoned targets to implement struct target_type
114 * report_zones operation.
115 */
116int dm_report_zones(struct block_device *bdev, sector_t start, sector_t sector,
117 struct dm_report_zones_args *args, unsigned int nr_zones)
118{
119 /*
120 * Set the target mapping start sector first so that
121 * dm_report_zones_cb() can correctly remap zone information.
122 */
123 args->start = start;
124
125 return blkdev_report_zones(bdev, sector, nr_zones,
126 dm_report_zones_cb, args);
127}
128EXPORT_SYMBOL_GPL(dm_report_zones);
129
130bool dm_is_zone_write(struct mapped_device *md, struct bio *bio)
131{
132 struct request_queue *q = md->queue;
133
134 if (!blk_queue_is_zoned(q))
135 return false;
136
137 switch (bio_op(bio)) {
138 case REQ_OP_WRITE_ZEROES:
139 case REQ_OP_WRITE:
140 return !op_is_flush(bio->bi_opf) && bio_sectors(bio);
141 default:
142 return false;
143 }
144}
145
146/*
147 * Revalidate the zones of a mapped device to initialize resource necessary
148 * for zone append emulation. Note that we cannot simply use the block layer
149 * blk_revalidate_disk_zones() function here as the mapped device is suspended
150 * (this is called from __bind() context).
151 */
152int dm_revalidate_zones(struct dm_table *t, struct request_queue *q)
153{
154 struct mapped_device *md = t->md;
155 struct gendisk *disk = md->disk;
156 int ret;
157
158 if (!get_capacity(disk))
159 return 0;
160
161 /* Revalidate only if something changed. */
162 if (!disk->nr_zones || disk->nr_zones != md->nr_zones) {
163 DMINFO("%s using %s zone append",
164 disk->disk_name,
165 queue_emulates_zone_append(q) ? "emulated" : "native");
166 md->nr_zones = 0;
167 }
168
169 if (md->nr_zones)
170 return 0;
171
172 /*
173 * Our table is not live yet. So the call to dm_get_live_table()
174 * in dm_blk_report_zones() will fail. Set a temporary pointer to
175 * our table for dm_blk_report_zones() to use directly.
176 */
177 md->zone_revalidate_map = t;
178 ret = blk_revalidate_disk_zones(disk);
179 md->zone_revalidate_map = NULL;
180
181 if (ret) {
182 DMERR("Revalidate zones failed %d", ret);
183 return ret;
184 }
185
186 md->nr_zones = disk->nr_zones;
187
188 return 0;
189}
190
191static int device_not_zone_append_capable(struct dm_target *ti,
192 struct dm_dev *dev, sector_t start,
193 sector_t len, void *data)
194{
195 return !bdev_is_zoned(dev->bdev);
196}
197
198static bool dm_table_supports_zone_append(struct dm_table *t)
199{
200 for (unsigned int i = 0; i < t->num_targets; i++) {
201 struct dm_target *ti = dm_table_get_target(t, i);
202
203 if (ti->emulate_zone_append)
204 return false;
205
206 if (!ti->type->iterate_devices ||
207 ti->type->iterate_devices(ti, device_not_zone_append_capable, NULL))
208 return false;
209 }
210
211 return true;
212}
213
214struct dm_device_zone_count {
215 sector_t start;
216 sector_t len;
217 unsigned int total_nr_seq_zones;
218 unsigned int target_nr_seq_zones;
219};
220
221/*
222 * Count the total number of and the number of mapped sequential zones of a
223 * target zoned device.
224 */
225static int dm_device_count_zones_cb(struct blk_zone *zone,
226 unsigned int idx, void *data)
227{
228 struct dm_device_zone_count *zc = data;
229
230 if (zone->type != BLK_ZONE_TYPE_CONVENTIONAL) {
231 zc->total_nr_seq_zones++;
232 if (zone->start >= zc->start &&
233 zone->start < zc->start + zc->len)
234 zc->target_nr_seq_zones++;
235 }
236
237 return 0;
238}
239
240static int dm_device_count_zones(struct dm_dev *dev,
241 struct dm_device_zone_count *zc)
242{
243 int ret;
244
245 ret = blkdev_report_zones(dev->bdev, 0, BLK_ALL_ZONES,
246 dm_device_count_zones_cb, zc);
247 if (ret < 0)
248 return ret;
249 if (!ret)
250 return -EIO;
251 return 0;
252}
253
254struct dm_zone_resource_limits {
255 unsigned int mapped_nr_seq_zones;
256 struct queue_limits *lim;
257 bool reliable_limits;
258};
259
260static int device_get_zone_resource_limits(struct dm_target *ti,
261 struct dm_dev *dev, sector_t start,
262 sector_t len, void *data)
263{
264 struct dm_zone_resource_limits *zlim = data;
265 struct gendisk *disk = dev->bdev->bd_disk;
266 unsigned int max_open_zones, max_active_zones;
267 int ret;
268 struct dm_device_zone_count zc = {
269 .start = start,
270 .len = len,
271 };
272
273 /*
274 * If the target is not the whole device, the device zone resources may
275 * be shared between different targets. Check this by counting the
276 * number of mapped sequential zones: if this number is smaller than the
277 * total number of sequential zones of the target device, then resource
278 * sharing may happen and the zone limits will not be reliable.
279 */
280 ret = dm_device_count_zones(dev, &zc);
281 if (ret) {
282 DMERR("Count %s zones failed %d", disk->disk_name, ret);
283 return ret;
284 }
285
286 /*
287 * If the target does not map any sequential zones, then we do not need
288 * any zone resource limits.
289 */
290 if (!zc.target_nr_seq_zones)
291 return 0;
292
293 /*
294 * If the target does not map all sequential zones, the limits
295 * will not be reliable and we cannot use REQ_OP_ZONE_RESET_ALL.
296 */
297 if (zc.target_nr_seq_zones < zc.total_nr_seq_zones) {
298 zlim->reliable_limits = false;
299 ti->zone_reset_all_supported = false;
300 }
301
302 /*
303 * If the target maps less sequential zones than the limit values, then
304 * we do not have limits for this target.
305 */
306 max_active_zones = disk->queue->limits.max_active_zones;
307 if (max_active_zones >= zc.target_nr_seq_zones)
308 max_active_zones = 0;
309 zlim->lim->max_active_zones =
310 min_not_zero(max_active_zones, zlim->lim->max_active_zones);
311
312 max_open_zones = disk->queue->limits.max_open_zones;
313 if (max_open_zones >= zc.target_nr_seq_zones)
314 max_open_zones = 0;
315 zlim->lim->max_open_zones =
316 min_not_zero(max_open_zones, zlim->lim->max_open_zones);
317
318 /*
319 * Also count the total number of sequential zones for the mapped
320 * device so that when we are done inspecting all its targets, we are
321 * able to check if the mapped device actually has any sequential zones.
322 */
323 zlim->mapped_nr_seq_zones += zc.target_nr_seq_zones;
324
325 return 0;
326}
327
328int dm_set_zones_restrictions(struct dm_table *t, struct request_queue *q,
329 struct queue_limits *lim)
330{
331 struct mapped_device *md = t->md;
332 struct gendisk *disk = md->disk;
333 struct dm_zone_resource_limits zlim = {
334 .reliable_limits = true,
335 .lim = lim,
336 };
337
338 /*
339 * Check if zone append is natively supported, and if not, set the
340 * mapped device queue as needing zone append emulation.
341 */
342 WARN_ON_ONCE(queue_is_mq(q));
343 if (dm_table_supports_zone_append(t)) {
344 clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
345 } else {
346 set_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
347 lim->max_hw_zone_append_sectors = 0;
348 }
349
350 /*
351 * Determine the max open and max active zone limits for the mapped
352 * device by inspecting the zone resource limits and the zones mapped
353 * by each target.
354 */
355 for (unsigned int i = 0; i < t->num_targets; i++) {
356 struct dm_target *ti = dm_table_get_target(t, i);
357
358 /*
359 * Assume that the target can accept REQ_OP_ZONE_RESET_ALL.
360 * device_get_zone_resource_limits() may adjust this if one of
361 * the device used by the target does not have all its
362 * sequential write required zones mapped.
363 */
364 ti->zone_reset_all_supported = true;
365
366 if (!ti->type->iterate_devices ||
367 ti->type->iterate_devices(ti,
368 device_get_zone_resource_limits, &zlim)) {
369 DMERR("Could not determine %s zone resource limits",
370 disk->disk_name);
371 return -ENODEV;
372 }
373 }
374
375 /*
376 * If we only have conventional zones mapped, expose the mapped device
377 + as a regular device.
378 */
379 if (!zlim.mapped_nr_seq_zones) {
380 lim->max_open_zones = 0;
381 lim->max_active_zones = 0;
382 lim->max_hw_zone_append_sectors = 0;
383 lim->zone_write_granularity = 0;
384 lim->chunk_sectors = 0;
385 lim->features &= ~BLK_FEAT_ZONED;
386 clear_bit(DMF_EMULATE_ZONE_APPEND, &md->flags);
387 md->nr_zones = 0;
388 disk->nr_zones = 0;
389 return 0;
390 }
391
392 /*
393 * Warn once (when the capacity is not yet set) if the mapped device is
394 * partially using zone resources of the target devices as that leads to
395 * unreliable limits, i.e. if another mapped device uses the same
396 * underlying devices, we cannot enforce zone limits to guarantee that
397 * writing will not lead to errors. Note that we really should return
398 * an error for such case but there is no easy way to find out if
399 * another mapped device uses the same underlying zoned devices.
400 */
401 if (!get_capacity(disk) && !zlim.reliable_limits)
402 DMWARN("%s zone resource limits may be unreliable",
403 disk->disk_name);
404
405 if (lim->features & BLK_FEAT_ZONED &&
406 !static_key_enabled(&zoned_enabled.key))
407 static_branch_enable(&zoned_enabled);
408 return 0;
409}
410
411/*
412 * IO completion callback called from clone_endio().
413 */
414void dm_zone_endio(struct dm_io *io, struct bio *clone)
415{
416 struct mapped_device *md = io->md;
417 struct gendisk *disk = md->disk;
418 struct bio *orig_bio = io->orig_bio;
419
420 /*
421 * Get the offset within the zone of the written sector
422 * and add that to the original bio sector position.
423 */
424 if (clone->bi_status == BLK_STS_OK &&
425 bio_op(clone) == REQ_OP_ZONE_APPEND) {
426 sector_t mask = bdev_zone_sectors(disk->part0) - 1;
427
428 orig_bio->bi_iter.bi_sector += clone->bi_iter.bi_sector & mask;
429 }
430
431 return;
432}
433
434static int dm_zone_need_reset_cb(struct blk_zone *zone, unsigned int idx,
435 void *data)
436{
437 /*
438 * For an all-zones reset, ignore conventional, empty, read-only
439 * and offline zones.
440 */
441 switch (zone->cond) {
442 case BLK_ZONE_COND_NOT_WP:
443 case BLK_ZONE_COND_EMPTY:
444 case BLK_ZONE_COND_READONLY:
445 case BLK_ZONE_COND_OFFLINE:
446 return 0;
447 default:
448 set_bit(idx, (unsigned long *)data);
449 return 0;
450 }
451}
452
453int dm_zone_get_reset_bitmap(struct mapped_device *md, struct dm_table *t,
454 sector_t sector, unsigned int nr_zones,
455 unsigned long *need_reset)
456{
457 int ret;
458
459 ret = dm_blk_do_report_zones(md, t, sector, nr_zones,
460 dm_zone_need_reset_cb, need_reset);
461 if (ret != nr_zones) {
462 DMERR("Get %s zone reset bitmap failed\n",
463 md->disk->disk_name);
464 return -EIO;
465 }
466
467 return 0;
468}