1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (C) 2017 Western Digital Corporation or its affiliates.
   4 *
   5 * This file is released under the GPL.
   6 */
   7
   8#include "dm-zoned.h"
   9
  10#include <linux/module.h>
  11
  12#define	DM_MSG_PREFIX		"zoned"
  13
  14#define DMZ_MIN_BIOS		8192
  15
  16/*
  17 * Zone BIO context.
  18 */
  19struct dmz_bioctx {
  20	struct dmz_dev		*dev;
  21	struct dm_zone		*zone;
  22	struct bio		*bio;
  23	refcount_t		ref;
  24};
  25
  26/*
  27 * Chunk work descriptor.
  28 */
  29struct dm_chunk_work {
  30	struct work_struct	work;
  31	refcount_t		refcount;
  32	struct dmz_target	*target;
  33	unsigned int		chunk;
  34	struct bio_list		bio_list;
  35};
  36
  37/*
  38 * Target descriptor.
  39 */
  40struct dmz_target {
  41	struct dm_dev		**ddev;
  42	unsigned int		nr_ddevs;
  43
  44	unsigned int		flags;
  45
  46	/* Zoned block device information */
  47	struct dmz_dev		*dev;
  48
  49	/* For metadata handling */
  50	struct dmz_metadata     *metadata;
  51
  52	/* For chunk work */
  53	struct radix_tree_root	chunk_rxtree;
  54	struct workqueue_struct *chunk_wq;
  55	struct mutex		chunk_lock;
  56
  57	/* For cloned BIOs to zones */
  58	struct bio_set		bio_set;
  59
  60	/* For flush */
  61	spinlock_t		flush_lock;
  62	struct bio_list		flush_list;
  63	struct delayed_work	flush_work;
  64	struct workqueue_struct *flush_wq;
  65};
  66
  67/*
  68 * Flush intervals (seconds).
  69 */
  70#define DMZ_FLUSH_PERIOD	(10 * HZ)
  71
  72/*
  73 * Target BIO completion.
  74 */
  75static inline void dmz_bio_endio(struct bio *bio, blk_status_t status)
  76{
  77	struct dmz_bioctx *bioctx =
  78		dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
  79
  80	if (status != BLK_STS_OK && bio->bi_status == BLK_STS_OK)
  81		bio->bi_status = status;
  82	if (bioctx->dev && bio->bi_status != BLK_STS_OK)
  83		bioctx->dev->flags |= DMZ_CHECK_BDEV;
  84
  85	if (refcount_dec_and_test(&bioctx->ref)) {
  86		struct dm_zone *zone = bioctx->zone;
  87
  88		if (zone) {
  89			if (bio->bi_status != BLK_STS_OK &&
  90			    bio_op(bio) == REQ_OP_WRITE &&
  91			    dmz_is_seq(zone))
  92				set_bit(DMZ_SEQ_WRITE_ERR, &zone->flags);
  93			dmz_deactivate_zone(zone);
  94		}
  95		bio_endio(bio);
  96	}
  97}
  98
  99/*
 100 * Completion callback for an internally cloned target BIO. This terminates the
 101 * target BIO when there are no more references to its context.
 102 */
 103static void dmz_clone_endio(struct bio *clone)
 104{
 105	struct dmz_bioctx *bioctx = clone->bi_private;
 106	blk_status_t status = clone->bi_status;
 107
 108	bio_put(clone);
 109	dmz_bio_endio(bioctx->bio, status);
 110}
 111
 112/*
 113 * Issue a clone of a target BIO. The clone may only partially process the
 114 * original target BIO.
 115 */
 116static int dmz_submit_bio(struct dmz_target *dmz, struct dm_zone *zone,
 117			  struct bio *bio, sector_t chunk_block,
 118			  unsigned int nr_blocks)
 119{
 120	struct dmz_bioctx *bioctx =
 121		dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
 122	struct dmz_dev *dev = zone->dev;
 123	struct bio *clone;
 124
 125	if (dev->flags & DMZ_BDEV_DYING)
 126		return -EIO;
 127
 128	clone = bio_alloc_clone(dev->bdev, bio, GFP_NOIO, &dmz->bio_set);
 129	if (!clone)
 130		return -ENOMEM;
 131
 132	bioctx->dev = dev;
 133	clone->bi_iter.bi_sector =
 134		dmz_start_sect(dmz->metadata, zone) + dmz_blk2sect(chunk_block);
 135	clone->bi_iter.bi_size = dmz_blk2sect(nr_blocks) << SECTOR_SHIFT;
 136	clone->bi_end_io = dmz_clone_endio;
 137	clone->bi_private = bioctx;
 138
 139	bio_advance(bio, clone->bi_iter.bi_size);
 140
 141	refcount_inc(&bioctx->ref);
 142	submit_bio_noacct(clone);
 143
 144	if (bio_op(bio) == REQ_OP_WRITE && dmz_is_seq(zone))
 145		zone->wp_block += nr_blocks;
 146
 147	return 0;
 148}
 149
 150/*
 151 * Zero out pages of discarded blocks accessed by a read BIO.
 152 */
 153static void dmz_handle_read_zero(struct dmz_target *dmz, struct bio *bio,
 154				 sector_t chunk_block, unsigned int nr_blocks)
 155{
 156	unsigned int size = nr_blocks << DMZ_BLOCK_SHIFT;
 157
 158	/* Clear nr_blocks */
 159	swap(bio->bi_iter.bi_size, size);
 160	zero_fill_bio(bio);
 161	swap(bio->bi_iter.bi_size, size);
 162
 163	bio_advance(bio, size);
 164}
 165
 166/*
 167 * Process a read BIO.
 168 */
 169static int dmz_handle_read(struct dmz_target *dmz, struct dm_zone *zone,
 170			   struct bio *bio)
 171{
 172	struct dmz_metadata *zmd = dmz->metadata;
 173	sector_t chunk_block = dmz_chunk_block(zmd, dmz_bio_block(bio));
 174	unsigned int nr_blocks = dmz_bio_blocks(bio);
 175	sector_t end_block = chunk_block + nr_blocks;
 176	struct dm_zone *rzone, *bzone;
 177	int ret;
 178
 179	/* Read into unmapped chunks need only zeroing the BIO buffer */
 180	if (!zone) {
 181		zero_fill_bio(bio);
 182		return 0;
 183	}
 184
 185	DMDEBUG("(%s): READ chunk %llu -> %s zone %u, block %llu, %u blocks",
 186		dmz_metadata_label(zmd),
 187		(unsigned long long)dmz_bio_chunk(zmd, bio),
 188		(dmz_is_rnd(zone) ? "RND" :
 189		 (dmz_is_cache(zone) ? "CACHE" : "SEQ")),
 190		zone->id,
 191		(unsigned long long)chunk_block, nr_blocks);
 192
 193	/* Check block validity to determine the read location */
 194	bzone = zone->bzone;
 195	while (chunk_block < end_block) {
 196		nr_blocks = 0;
 197		if (dmz_is_rnd(zone) || dmz_is_cache(zone) ||
 198		    chunk_block < zone->wp_block) {
 199			/* Test block validity in the data zone */
 200			ret = dmz_block_valid(zmd, zone, chunk_block);
 201			if (ret < 0)
 202				return ret;
 203			if (ret > 0) {
 204				/* Read data zone blocks */
 205				nr_blocks = ret;
 206				rzone = zone;
 207			}
 208		}
 209
 210		/*
 211		 * No valid blocks found in the data zone.
 212		 * Check the buffer zone, if there is one.
 213		 */
 214		if (!nr_blocks && bzone) {
 215			ret = dmz_block_valid(zmd, bzone, chunk_block);
 216			if (ret < 0)
 217				return ret;
 218			if (ret > 0) {
 219				/* Read buffer zone blocks */
 220				nr_blocks = ret;
 221				rzone = bzone;
 222			}
 223		}
 224
 225		if (nr_blocks) {
 226			/* Valid blocks found: read them */
 227			nr_blocks = min_t(unsigned int, nr_blocks,
 228					  end_block - chunk_block);
 229			ret = dmz_submit_bio(dmz, rzone, bio,
 230					     chunk_block, nr_blocks);
 231			if (ret)
 232				return ret;
 233			chunk_block += nr_blocks;
 234		} else {
 235			/* No valid block: zeroout the current BIO block */
 236			dmz_handle_read_zero(dmz, bio, chunk_block, 1);
 237			chunk_block++;
 238		}
 239	}
 240
 241	return 0;
 242}
 243
 244/*
 245 * Write blocks directly in a data zone, at the write pointer.
 246 * If a buffer zone is assigned, invalidate the blocks written
 247 * in place.
 248 */
 249static int dmz_handle_direct_write(struct dmz_target *dmz,
 250				   struct dm_zone *zone, struct bio *bio,
 251				   sector_t chunk_block,
 252				   unsigned int nr_blocks)
 253{
 254	struct dmz_metadata *zmd = dmz->metadata;
 255	struct dm_zone *bzone = zone->bzone;
 256	int ret;
 257
 258	if (dmz_is_readonly(zone))
 259		return -EROFS;
 260
 261	/* Submit write */
 262	ret = dmz_submit_bio(dmz, zone, bio, chunk_block, nr_blocks);
 263	if (ret)
 264		return ret;
 265
 266	/*
 267	 * Validate the blocks in the data zone and invalidate
 268	 * in the buffer zone, if there is one.
 269	 */
 270	ret = dmz_validate_blocks(zmd, zone, chunk_block, nr_blocks);
 271	if (ret == 0 && bzone)
 272		ret = dmz_invalidate_blocks(zmd, bzone, chunk_block, nr_blocks);
 273
 274	return ret;
 275}
 276
 277/*
 278 * Write blocks in the buffer zone of @zone.
 279 * If no buffer zone is assigned yet, get one.
 280 * Called with @zone write locked.
 281 */
 282static int dmz_handle_buffered_write(struct dmz_target *dmz,
 283				     struct dm_zone *zone, struct bio *bio,
 284				     sector_t chunk_block,
 285				     unsigned int nr_blocks)
 286{
 287	struct dmz_metadata *zmd = dmz->metadata;
 288	struct dm_zone *bzone;
 289	int ret;
 290
 291	/* Get the buffer zone. One will be allocated if needed */
 292	bzone = dmz_get_chunk_buffer(zmd, zone);
 293	if (IS_ERR(bzone))
 294		return PTR_ERR(bzone);
 295
 296	if (dmz_is_readonly(bzone))
 297		return -EROFS;
 298
 299	/* Submit write */
 300	ret = dmz_submit_bio(dmz, bzone, bio, chunk_block, nr_blocks);
 301	if (ret)
 302		return ret;
 303
 304	/*
 305	 * Validate the blocks in the buffer zone
 306	 * and invalidate in the data zone.
 307	 */
 308	ret = dmz_validate_blocks(zmd, bzone, chunk_block, nr_blocks);
 309	if (ret == 0 && chunk_block < zone->wp_block)
 310		ret = dmz_invalidate_blocks(zmd, zone, chunk_block, nr_blocks);
 311
 312	return ret;
 313}
 314
 315/*
 316 * Process a write BIO.
 317 */
 318static int dmz_handle_write(struct dmz_target *dmz, struct dm_zone *zone,
 319			    struct bio *bio)
 320{
 321	struct dmz_metadata *zmd = dmz->metadata;
 322	sector_t chunk_block = dmz_chunk_block(zmd, dmz_bio_block(bio));
 323	unsigned int nr_blocks = dmz_bio_blocks(bio);
 324
 325	if (!zone)
 326		return -ENOSPC;
 327
 328	DMDEBUG("(%s): WRITE chunk %llu -> %s zone %u, block %llu, %u blocks",
 329		dmz_metadata_label(zmd),
 330		(unsigned long long)dmz_bio_chunk(zmd, bio),
 331		(dmz_is_rnd(zone) ? "RND" :
 332		 (dmz_is_cache(zone) ? "CACHE" : "SEQ")),
 333		zone->id,
 334		(unsigned long long)chunk_block, nr_blocks);
 335
 336	if (dmz_is_rnd(zone) || dmz_is_cache(zone) ||
 337	    chunk_block == zone->wp_block) {
 338		/*
 339		 * zone is a random zone or it is a sequential zone
 340		 * and the BIO is aligned to the zone write pointer:
 341		 * direct write the zone.
 342		 */
 343		return dmz_handle_direct_write(dmz, zone, bio,
 344					       chunk_block, nr_blocks);
 345	}
 346
 347	/*
 348	 * This is an unaligned write in a sequential zone:
 349	 * use buffered write.
 350	 */
 351	return dmz_handle_buffered_write(dmz, zone, bio, chunk_block, nr_blocks);
 352}
 353
 354/*
 355 * Process a discard BIO.
 356 */
 357static int dmz_handle_discard(struct dmz_target *dmz, struct dm_zone *zone,
 358			      struct bio *bio)
 359{
 360	struct dmz_metadata *zmd = dmz->metadata;
 361	sector_t block = dmz_bio_block(bio);
 362	unsigned int nr_blocks = dmz_bio_blocks(bio);
 363	sector_t chunk_block = dmz_chunk_block(zmd, block);
 364	int ret = 0;
 365
 366	/* For unmapped chunks, there is nothing to do */
 367	if (!zone)
 368		return 0;
 369
 370	if (dmz_is_readonly(zone))
 371		return -EROFS;
 372
 373	DMDEBUG("(%s): DISCARD chunk %llu -> zone %u, block %llu, %u blocks",
 374		dmz_metadata_label(dmz->metadata),
 375		(unsigned long long)dmz_bio_chunk(zmd, bio),
 376		zone->id,
 377		(unsigned long long)chunk_block, nr_blocks);
 378
 379	/*
 380	 * Invalidate blocks in the data zone and its
 381	 * buffer zone if one is mapped.
 382	 */
 383	if (dmz_is_rnd(zone) || dmz_is_cache(zone) ||
 384	    chunk_block < zone->wp_block)
 385		ret = dmz_invalidate_blocks(zmd, zone, chunk_block, nr_blocks);
 386	if (ret == 0 && zone->bzone)
 387		ret = dmz_invalidate_blocks(zmd, zone->bzone,
 388					    chunk_block, nr_blocks);
 389	return ret;
 390}
 391
 392/*
 393 * Process a BIO.
 394 */
 395static void dmz_handle_bio(struct dmz_target *dmz, struct dm_chunk_work *cw,
 396			   struct bio *bio)
 397{
 398	struct dmz_bioctx *bioctx =
 399		dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
 400	struct dmz_metadata *zmd = dmz->metadata;
 401	struct dm_zone *zone;
 402	int ret;
 403
 404	dmz_lock_metadata(zmd);
 405
 406	/*
 407	 * Get the data zone mapping the chunk. There may be no
 408	 * mapping for read and discard. If a mapping is obtained,
 409	 + the zone returned will be set to active state.
 410	 */
 411	zone = dmz_get_chunk_mapping(zmd, dmz_bio_chunk(zmd, bio),
 412				     bio_op(bio));
 413	if (IS_ERR(zone)) {
 414		ret = PTR_ERR(zone);
 415		goto out;
 416	}
 417
 418	/* Process the BIO */
 419	if (zone) {
 420		dmz_activate_zone(zone);
 421		bioctx->zone = zone;
 422		dmz_reclaim_bio_acc(zone->dev->reclaim);
 423	}
 424
 425	switch (bio_op(bio)) {
 426	case REQ_OP_READ:
 427		ret = dmz_handle_read(dmz, zone, bio);
 428		break;
 429	case REQ_OP_WRITE:
 430		ret = dmz_handle_write(dmz, zone, bio);
 431		break;
 432	case REQ_OP_DISCARD:
 433	case REQ_OP_WRITE_ZEROES:
 434		ret = dmz_handle_discard(dmz, zone, bio);
 435		break;
 436	default:
 437		DMERR("(%s): Unsupported BIO operation 0x%x",
 438		      dmz_metadata_label(dmz->metadata), bio_op(bio));
 439		ret = -EIO;
 440	}
 441
 442	/*
 443	 * Release the chunk mapping. This will check that the mapping
 444	 * is still valid, that is, that the zone used still has valid blocks.
 445	 */
 446	if (zone)
 447		dmz_put_chunk_mapping(zmd, zone);
 448out:
 449	dmz_bio_endio(bio, errno_to_blk_status(ret));
 450
 451	dmz_unlock_metadata(zmd);
 452}
 453
 454/*
 455 * Increment a chunk reference counter.
 456 */
 457static inline void dmz_get_chunk_work(struct dm_chunk_work *cw)
 458{
 459	refcount_inc(&cw->refcount);
 460}
 461
 462/*
 463 * Decrement a chunk work reference count and
 464 * free it if it becomes 0.
 465 */
 466static void dmz_put_chunk_work(struct dm_chunk_work *cw)
 467{
 468	if (refcount_dec_and_test(&cw->refcount)) {
 469		WARN_ON(!bio_list_empty(&cw->bio_list));
 470		radix_tree_delete(&cw->target->chunk_rxtree, cw->chunk);
 471		kfree(cw);
 472	}
 473}
 474
 475/*
 476 * Chunk BIO work function.
 477 */
 478static void dmz_chunk_work(struct work_struct *work)
 479{
 480	struct dm_chunk_work *cw = container_of(work, struct dm_chunk_work, work);
 481	struct dmz_target *dmz = cw->target;
 482	struct bio *bio;
 483
 484	mutex_lock(&dmz->chunk_lock);
 485
 486	/* Process the chunk BIOs */
 487	while ((bio = bio_list_pop(&cw->bio_list))) {
 488		mutex_unlock(&dmz->chunk_lock);
 489		dmz_handle_bio(dmz, cw, bio);
 490		mutex_lock(&dmz->chunk_lock);
 491		dmz_put_chunk_work(cw);
 492	}
 493
 494	/* Queueing the work incremented the work refcount */
 495	dmz_put_chunk_work(cw);
 496
 497	mutex_unlock(&dmz->chunk_lock);
 498}
 499
 500/*
 501 * Flush work.
 502 */
 503static void dmz_flush_work(struct work_struct *work)
 504{
 505	struct dmz_target *dmz = container_of(work, struct dmz_target, flush_work.work);
 506	struct bio *bio;
 507	int ret;
 508
 509	/* Flush dirty metadata blocks */
 510	ret = dmz_flush_metadata(dmz->metadata);
 511	if (ret)
 512		DMDEBUG("(%s): Metadata flush failed, rc=%d",
 513			dmz_metadata_label(dmz->metadata), ret);
 514
 515	/* Process queued flush requests */
 516	while (1) {
 517		spin_lock(&dmz->flush_lock);
 518		bio = bio_list_pop(&dmz->flush_list);
 519		spin_unlock(&dmz->flush_lock);
 520
 521		if (!bio)
 522			break;
 523
 524		dmz_bio_endio(bio, errno_to_blk_status(ret));
 525	}
 526
 527	queue_delayed_work(dmz->flush_wq, &dmz->flush_work, DMZ_FLUSH_PERIOD);
 528}
 529
 530/*
 531 * Get a chunk work and start it to process a new BIO.
 532 * If the BIO chunk has no work yet, create one.
 533 */
 534static int dmz_queue_chunk_work(struct dmz_target *dmz, struct bio *bio)
 535{
 536	unsigned int chunk = dmz_bio_chunk(dmz->metadata, bio);
 537	struct dm_chunk_work *cw;
 538	int ret = 0;
 539
 540	mutex_lock(&dmz->chunk_lock);
 541
 542	/* Get the BIO chunk work. If one is not active yet, create one */
 543	cw = radix_tree_lookup(&dmz->chunk_rxtree, chunk);
 544	if (cw) {
 545		dmz_get_chunk_work(cw);
 546	} else {
 547		/* Create a new chunk work */
 548		cw = kmalloc(sizeof(struct dm_chunk_work), GFP_NOIO);
 549		if (unlikely(!cw)) {
 550			ret = -ENOMEM;
 551			goto out;
 552		}
 553
 554		INIT_WORK(&cw->work, dmz_chunk_work);
 555		refcount_set(&cw->refcount, 1);
 556		cw->target = dmz;
 557		cw->chunk = chunk;
 558		bio_list_init(&cw->bio_list);
 559
 560		ret = radix_tree_insert(&dmz->chunk_rxtree, chunk, cw);
 561		if (unlikely(ret)) {
 562			kfree(cw);
 563			goto out;
 564		}
 565	}
 566
 567	bio_list_add(&cw->bio_list, bio);
 568
 569	if (queue_work(dmz->chunk_wq, &cw->work))
 570		dmz_get_chunk_work(cw);
 571out:
 572	mutex_unlock(&dmz->chunk_lock);
 573	return ret;
 574}
 575
 576/*
 577 * Check if the backing device is being removed. If it's on the way out,
 578 * start failing I/O. Reclaim and metadata components also call this
 579 * function to cleanly abort operation in the event of such failure.
 580 */
 581bool dmz_bdev_is_dying(struct dmz_dev *dmz_dev)
 582{
 583	if (dmz_dev->flags & DMZ_BDEV_DYING)
 584		return true;
 585
 586	if (dmz_dev->flags & DMZ_CHECK_BDEV)
 587		return !dmz_check_bdev(dmz_dev);
 588
 589	if (blk_queue_dying(bdev_get_queue(dmz_dev->bdev))) {
 590		dmz_dev_warn(dmz_dev, "Backing device queue dying");
 591		dmz_dev->flags |= DMZ_BDEV_DYING;
 592	}
 593
 594	return dmz_dev->flags & DMZ_BDEV_DYING;
 595}
 596
 597/*
 598 * Check the backing device availability. This detects such events as
 599 * backing device going offline due to errors, media removals, etc.
 600 * This check is less efficient than dmz_bdev_is_dying() and should
 601 * only be performed as a part of error handling.
 602 */
 603bool dmz_check_bdev(struct dmz_dev *dmz_dev)
 604{
 605	struct gendisk *disk;
 606
 607	dmz_dev->flags &= ~DMZ_CHECK_BDEV;
 608
 609	if (dmz_bdev_is_dying(dmz_dev))
 610		return false;
 611
 612	disk = dmz_dev->bdev->bd_disk;
 613	if (disk->fops->check_events &&
 614	    disk->fops->check_events(disk, 0) & DISK_EVENT_MEDIA_CHANGE) {
 615		dmz_dev_warn(dmz_dev, "Backing device offline");
 616		dmz_dev->flags |= DMZ_BDEV_DYING;
 617	}
 618
 619	return !(dmz_dev->flags & DMZ_BDEV_DYING);
 620}
 621
 622/*
 623 * Process a new BIO.
 624 */
 625static int dmz_map(struct dm_target *ti, struct bio *bio)
 626{
 627	struct dmz_target *dmz = ti->private;
 628	struct dmz_metadata *zmd = dmz->metadata;
 629	struct dmz_bioctx *bioctx = dm_per_bio_data(bio, sizeof(struct dmz_bioctx));
 630	sector_t sector = bio->bi_iter.bi_sector;
 631	unsigned int nr_sectors = bio_sectors(bio);
 632	sector_t chunk_sector;
 633	int ret;
 634
 635	if (dmz_dev_is_dying(zmd))
 636		return DM_MAPIO_KILL;
 637
 638	DMDEBUG("(%s): BIO op %d sector %llu + %u => chunk %llu, block %llu, %u blocks",
 639		dmz_metadata_label(zmd),
 640		bio_op(bio), (unsigned long long)sector, nr_sectors,
 641		(unsigned long long)dmz_bio_chunk(zmd, bio),
 642		(unsigned long long)dmz_chunk_block(zmd, dmz_bio_block(bio)),
 643		(unsigned int)dmz_bio_blocks(bio));
 644
 645	if (!nr_sectors && bio_op(bio) != REQ_OP_WRITE)
 646		return DM_MAPIO_REMAPPED;
 647
 648	/* The BIO should be block aligned */
 649	if ((nr_sectors & DMZ_BLOCK_SECTORS_MASK) || (sector & DMZ_BLOCK_SECTORS_MASK))
 650		return DM_MAPIO_KILL;
 651
 652	/* Initialize the BIO context */
 653	bioctx->dev = NULL;
 654	bioctx->zone = NULL;
 655	bioctx->bio = bio;
 656	refcount_set(&bioctx->ref, 1);
 657
 658	/* Set the BIO pending in the flush list */
 659	if (!nr_sectors && bio_op(bio) == REQ_OP_WRITE) {
 660		spin_lock(&dmz->flush_lock);
 661		bio_list_add(&dmz->flush_list, bio);
 662		spin_unlock(&dmz->flush_lock);
 663		mod_delayed_work(dmz->flush_wq, &dmz->flush_work, 0);
 664		return DM_MAPIO_SUBMITTED;
 665	}
 666
 667	/* Split zone BIOs to fit entirely into a zone */
 668	chunk_sector = sector & (dmz_zone_nr_sectors(zmd) - 1);
 669	if (chunk_sector + nr_sectors > dmz_zone_nr_sectors(zmd))
 670		dm_accept_partial_bio(bio, dmz_zone_nr_sectors(zmd) - chunk_sector);
 671
 672	/* Now ready to handle this BIO */
 673	ret = dmz_queue_chunk_work(dmz, bio);
 674	if (ret) {
 675		DMDEBUG("(%s): BIO op %d, can't process chunk %llu, err %i",
 676			dmz_metadata_label(zmd),
 677			bio_op(bio), (u64)dmz_bio_chunk(zmd, bio),
 678			ret);
 679		return DM_MAPIO_REQUEUE;
 680	}
 681
 682	return DM_MAPIO_SUBMITTED;
 683}
 684
 685/*
 686 * Get zoned device information.
 687 */
 688static int dmz_get_zoned_device(struct dm_target *ti, char *path,
 689				int idx, int nr_devs)
 690{
 691	struct dmz_target *dmz = ti->private;
 692	struct dm_dev *ddev;
 693	struct dmz_dev *dev;
 694	int ret;
 695	struct block_device *bdev;
 696
 697	/* Get the target device */
 698	ret = dm_get_device(ti, path, dm_table_get_mode(ti->table), &ddev);
 699	if (ret) {
 700		ti->error = "Get target device failed";
 701		return ret;
 702	}
 703
 704	bdev = ddev->bdev;
 705	if (!bdev_is_zoned(bdev)) {
 706		if (nr_devs == 1) {
 707			ti->error = "Invalid regular device";
 708			goto err;
 709		}
 710		if (idx != 0) {
 711			ti->error = "First device must be a regular device";
 712			goto err;
 713		}
 714		if (dmz->ddev[0]) {
 715			ti->error = "Too many regular devices";
 716			goto err;
 717		}
 718		dev = &dmz->dev[idx];
 719		dev->flags = DMZ_BDEV_REGULAR;
 720	} else {
 721		if (dmz->ddev[idx]) {
 722			ti->error = "Too many zoned devices";
 723			goto err;
 724		}
 725		if (nr_devs > 1 && idx == 0) {
 726			ti->error = "First device must be a regular device";
 727			goto err;
 728		}
 729		dev = &dmz->dev[idx];
 730	}
 731	dev->bdev = bdev;
 732	dev->dev_idx = idx;
 733
 734	dev->capacity = bdev_nr_sectors(bdev);
 735	if (ti->begin) {
 736		ti->error = "Partial mapping is not supported";
 737		goto err;
 738	}
 739
 740	dmz->ddev[idx] = ddev;
 741
 742	return 0;
 743err:
 744	dm_put_device(ti, ddev);
 745	return -EINVAL;
 746}
 747
 748/*
 749 * Cleanup zoned device information.
 750 */
 751static void dmz_put_zoned_devices(struct dm_target *ti)
 752{
 753	struct dmz_target *dmz = ti->private;
 754	int i;
 755
 756	for (i = 0; i < dmz->nr_ddevs; i++)
 757		if (dmz->ddev[i])
 758			dm_put_device(ti, dmz->ddev[i]);
 759
 760	kfree(dmz->ddev);
 761}
 762
 763static int dmz_fixup_devices(struct dm_target *ti)
 764{
 765	struct dmz_target *dmz = ti->private;
 766	struct dmz_dev *reg_dev = NULL;
 767	sector_t zone_nr_sectors = 0;
 768	int i;
 769
 770	/*
 771	 * When we have more than on devices, the first one must be a
 772	 * regular block device and the others zoned block devices.
 773	 */
 774	if (dmz->nr_ddevs > 1) {
 775		reg_dev = &dmz->dev[0];
 776		if (!(reg_dev->flags & DMZ_BDEV_REGULAR)) {
 777			ti->error = "Primary disk is not a regular device";
 778			return -EINVAL;
 779		}
 780		for (i = 1; i < dmz->nr_ddevs; i++) {
 781			struct dmz_dev *zoned_dev = &dmz->dev[i];
 782			struct block_device *bdev = zoned_dev->bdev;
 783
 784			if (zoned_dev->flags & DMZ_BDEV_REGULAR) {
 785				ti->error = "Secondary disk is not a zoned device";
 786				return -EINVAL;
 787			}
 788			if (zone_nr_sectors &&
 789			    zone_nr_sectors != bdev_zone_sectors(bdev)) {
 790				ti->error = "Zone nr sectors mismatch";
 791				return -EINVAL;
 792			}
 793			zone_nr_sectors = bdev_zone_sectors(bdev);
 794			zoned_dev->zone_nr_sectors = zone_nr_sectors;
 795			zoned_dev->nr_zones = bdev_nr_zones(bdev);
 796		}
 797	} else {
 798		struct dmz_dev *zoned_dev = &dmz->dev[0];
 799		struct block_device *bdev = zoned_dev->bdev;
 800
 801		if (zoned_dev->flags & DMZ_BDEV_REGULAR) {
 802			ti->error = "Disk is not a zoned device";
 803			return -EINVAL;
 804		}
 805		zoned_dev->zone_nr_sectors = bdev_zone_sectors(bdev);
 806		zoned_dev->nr_zones = bdev_nr_zones(bdev);
 807	}
 808
 809	if (reg_dev) {
 810		sector_t zone_offset;
 811
 812		reg_dev->zone_nr_sectors = zone_nr_sectors;
 813		reg_dev->nr_zones =
 814			DIV_ROUND_UP_SECTOR_T(reg_dev->capacity,
 815					      reg_dev->zone_nr_sectors);
 816		reg_dev->zone_offset = 0;
 817		zone_offset = reg_dev->nr_zones;
 818		for (i = 1; i < dmz->nr_ddevs; i++) {
 819			dmz->dev[i].zone_offset = zone_offset;
 820			zone_offset += dmz->dev[i].nr_zones;
 821		}
 822	}
 823	return 0;
 824}
 825
 826/*
 827 * Setup target.
 828 */
 829static int dmz_ctr(struct dm_target *ti, unsigned int argc, char **argv)
 830{
 831	struct dmz_target *dmz;
 832	int ret, i;
 833
 834	/* Check arguments */
 835	if (argc < 1) {
 836		ti->error = "Invalid argument count";
 837		return -EINVAL;
 838	}
 839
 840	/* Allocate and initialize the target descriptor */
 841	dmz = kzalloc(sizeof(struct dmz_target), GFP_KERNEL);
 842	if (!dmz) {
 843		ti->error = "Unable to allocate the zoned target descriptor";
 844		return -ENOMEM;
 845	}
 846	dmz->dev = kcalloc(argc, sizeof(struct dmz_dev), GFP_KERNEL);
 847	if (!dmz->dev) {
 848		ti->error = "Unable to allocate the zoned device descriptors";
 849		kfree(dmz);
 850		return -ENOMEM;
 851	}
 852	dmz->ddev = kcalloc(argc, sizeof(struct dm_dev *), GFP_KERNEL);
 853	if (!dmz->ddev) {
 854		ti->error = "Unable to allocate the dm device descriptors";
 855		ret = -ENOMEM;
 856		goto err;
 857	}
 858	dmz->nr_ddevs = argc;
 859
 860	ti->private = dmz;
 861
 862	/* Get the target zoned block device */
 863	for (i = 0; i < argc; i++) {
 864		ret = dmz_get_zoned_device(ti, argv[i], i, argc);
 865		if (ret)
 866			goto err_dev;
 867	}
 868	ret = dmz_fixup_devices(ti);
 869	if (ret)
 870		goto err_dev;
 871
 872	/* Initialize metadata */
 873	ret = dmz_ctr_metadata(dmz->dev, argc, &dmz->metadata,
 874			       dm_table_device_name(ti->table));
 875	if (ret) {
 876		ti->error = "Metadata initialization failed";
 877		goto err_dev;
 878	}
 879
 880	/* Set target (no write same support) */
 881	ti->max_io_len = dmz_zone_nr_sectors(dmz->metadata);
 882	ti->num_flush_bios = 1;
 883	ti->num_discard_bios = 1;
 884	ti->num_write_zeroes_bios = 1;
 885	ti->per_io_data_size = sizeof(struct dmz_bioctx);
 886	ti->flush_supported = true;
 887	ti->discards_supported = true;
 888
 889	/* The exposed capacity is the number of chunks that can be mapped */
 890	ti->len = (sector_t)dmz_nr_chunks(dmz->metadata) <<
 891		dmz_zone_nr_sectors_shift(dmz->metadata);
 892
 893	/* Zone BIO */
 894	ret = bioset_init(&dmz->bio_set, DMZ_MIN_BIOS, 0, 0);
 895	if (ret) {
 896		ti->error = "Create BIO set failed";
 897		goto err_meta;
 898	}
 899
 900	/* Chunk BIO work */
 901	mutex_init(&dmz->chunk_lock);
 902	INIT_RADIX_TREE(&dmz->chunk_rxtree, GFP_NOIO);
 903	dmz->chunk_wq = alloc_workqueue("dmz_cwq_%s",
 904					WQ_MEM_RECLAIM | WQ_UNBOUND, 0,
 905					dmz_metadata_label(dmz->metadata));
 906	if (!dmz->chunk_wq) {
 907		ti->error = "Create chunk workqueue failed";
 908		ret = -ENOMEM;
 909		goto err_bio;
 910	}
 911
 912	/* Flush work */
 913	spin_lock_init(&dmz->flush_lock);
 914	bio_list_init(&dmz->flush_list);
 915	INIT_DELAYED_WORK(&dmz->flush_work, dmz_flush_work);
 916	dmz->flush_wq = alloc_ordered_workqueue("dmz_fwq_%s", WQ_MEM_RECLAIM,
 917						dmz_metadata_label(dmz->metadata));
 918	if (!dmz->flush_wq) {
 919		ti->error = "Create flush workqueue failed";
 920		ret = -ENOMEM;
 921		goto err_cwq;
 922	}
 923	mod_delayed_work(dmz->flush_wq, &dmz->flush_work, DMZ_FLUSH_PERIOD);
 924
 925	/* Initialize reclaim */
 926	for (i = 0; i < dmz->nr_ddevs; i++) {
 927		ret = dmz_ctr_reclaim(dmz->metadata, &dmz->dev[i].reclaim, i);
 928		if (ret) {
 929			ti->error = "Zone reclaim initialization failed";
 930			goto err_fwq;
 931		}
 932	}
 933
 934	DMINFO("(%s): Target device: %llu 512-byte logical sectors (%llu blocks)",
 935	       dmz_metadata_label(dmz->metadata),
 936	       (unsigned long long)ti->len,
 937	       (unsigned long long)dmz_sect2blk(ti->len));
 938
 939	return 0;
 940err_fwq:
 941	destroy_workqueue(dmz->flush_wq);
 942err_cwq:
 943	destroy_workqueue(dmz->chunk_wq);
 944err_bio:
 945	mutex_destroy(&dmz->chunk_lock);
 946	bioset_exit(&dmz->bio_set);
 947err_meta:
 948	dmz_dtr_metadata(dmz->metadata);
 949err_dev:
 950	dmz_put_zoned_devices(ti);
 951err:
 952	kfree(dmz->dev);
 953	kfree(dmz);
 954
 955	return ret;
 956}
 957
 958/*
 959 * Cleanup target.
 960 */
 961static void dmz_dtr(struct dm_target *ti)
 962{
 963	struct dmz_target *dmz = ti->private;
 964	int i;
 965
 966	destroy_workqueue(dmz->chunk_wq);
 967
 968	for (i = 0; i < dmz->nr_ddevs; i++)
 969		dmz_dtr_reclaim(dmz->dev[i].reclaim);
 970
 971	cancel_delayed_work_sync(&dmz->flush_work);
 972	destroy_workqueue(dmz->flush_wq);
 973
 974	(void) dmz_flush_metadata(dmz->metadata);
 975
 976	dmz_dtr_metadata(dmz->metadata);
 977
 978	bioset_exit(&dmz->bio_set);
 979
 980	dmz_put_zoned_devices(ti);
 981
 982	mutex_destroy(&dmz->chunk_lock);
 983
 984	kfree(dmz->dev);
 985	kfree(dmz);
 986}
 987
 988/*
 989 * Setup target request queue limits.
 990 */
 991static void dmz_io_hints(struct dm_target *ti, struct queue_limits *limits)
 992{
 993	struct dmz_target *dmz = ti->private;
 994	unsigned int chunk_sectors = dmz_zone_nr_sectors(dmz->metadata);
 995
 996	limits->logical_block_size = DMZ_BLOCK_SIZE;
 997	limits->physical_block_size = DMZ_BLOCK_SIZE;
 998
 999	blk_limits_io_min(limits, DMZ_BLOCK_SIZE);
1000	blk_limits_io_opt(limits, DMZ_BLOCK_SIZE);
1001
1002	limits->discard_alignment = 0;
1003	limits->discard_granularity = DMZ_BLOCK_SIZE;
1004	limits->max_discard_sectors = chunk_sectors;
1005	limits->max_hw_discard_sectors = chunk_sectors;
1006	limits->max_write_zeroes_sectors = chunk_sectors;
1007
1008	/* FS hint to try to align to the device zone size */
1009	limits->chunk_sectors = chunk_sectors;
1010	limits->max_sectors = chunk_sectors;
1011
1012	/* We are exposing a drive-managed zoned block device */
1013	limits->zoned = false;
1014}
1015
1016/*
1017 * Pass on ioctl to the backend device.
1018 */
1019static int dmz_prepare_ioctl(struct dm_target *ti, struct block_device **bdev)
1020{
1021	struct dmz_target *dmz = ti->private;
1022	struct dmz_dev *dev = &dmz->dev[0];
1023
1024	if (!dmz_check_bdev(dev))
1025		return -EIO;
1026
1027	*bdev = dev->bdev;
1028
1029	return 0;
1030}
1031
1032/*
1033 * Stop works on suspend.
1034 */
1035static void dmz_suspend(struct dm_target *ti)
1036{
1037	struct dmz_target *dmz = ti->private;
1038	int i;
1039
1040	flush_workqueue(dmz->chunk_wq);
1041	for (i = 0; i < dmz->nr_ddevs; i++)
1042		dmz_suspend_reclaim(dmz->dev[i].reclaim);
1043	cancel_delayed_work_sync(&dmz->flush_work);
1044}
1045
1046/*
1047 * Restart works on resume or if suspend failed.
1048 */
1049static void dmz_resume(struct dm_target *ti)
1050{
1051	struct dmz_target *dmz = ti->private;
1052	int i;
1053
1054	queue_delayed_work(dmz->flush_wq, &dmz->flush_work, DMZ_FLUSH_PERIOD);
1055	for (i = 0; i < dmz->nr_ddevs; i++)
1056		dmz_resume_reclaim(dmz->dev[i].reclaim);
1057}
1058
1059static int dmz_iterate_devices(struct dm_target *ti,
1060			       iterate_devices_callout_fn fn, void *data)
1061{
1062	struct dmz_target *dmz = ti->private;
1063	unsigned int zone_nr_sectors = dmz_zone_nr_sectors(dmz->metadata);
1064	sector_t capacity;
1065	int i, r;
1066
1067	for (i = 0; i < dmz->nr_ddevs; i++) {
1068		capacity = dmz->dev[i].capacity & ~(zone_nr_sectors - 1);
1069		r = fn(ti, dmz->ddev[i], 0, capacity, data);
1070		if (r)
1071			break;
1072	}
1073	return r;
1074}
1075
1076static void dmz_status(struct dm_target *ti, status_type_t type,
1077		       unsigned int status_flags, char *result,
1078		       unsigned int maxlen)
1079{
1080	struct dmz_target *dmz = ti->private;
1081	ssize_t sz = 0;
1082	char buf[BDEVNAME_SIZE];
1083	struct dmz_dev *dev;
1084	int i;
1085
1086	switch (type) {
1087	case STATUSTYPE_INFO:
1088		DMEMIT("%u zones %u/%u cache",
1089		       dmz_nr_zones(dmz->metadata),
1090		       dmz_nr_unmap_cache_zones(dmz->metadata),
1091		       dmz_nr_cache_zones(dmz->metadata));
1092		for (i = 0; i < dmz->nr_ddevs; i++) {
1093			/*
1094			 * For a multi-device setup the first device
1095			 * contains only cache zones.
1096			 */
1097			if ((i == 0) &&
1098			    (dmz_nr_cache_zones(dmz->metadata) > 0))
1099				continue;
1100			DMEMIT(" %u/%u random %u/%u sequential",
1101			       dmz_nr_unmap_rnd_zones(dmz->metadata, i),
1102			       dmz_nr_rnd_zones(dmz->metadata, i),
1103			       dmz_nr_unmap_seq_zones(dmz->metadata, i),
1104			       dmz_nr_seq_zones(dmz->metadata, i));
1105		}
1106		break;
1107	case STATUSTYPE_TABLE:
1108		dev = &dmz->dev[0];
1109		format_dev_t(buf, dev->bdev->bd_dev);
1110		DMEMIT("%s", buf);
1111		for (i = 1; i < dmz->nr_ddevs; i++) {
1112			dev = &dmz->dev[i];
1113			format_dev_t(buf, dev->bdev->bd_dev);
1114			DMEMIT(" %s", buf);
1115		}
1116		break;
1117	case STATUSTYPE_IMA:
1118		*result = '\0';
1119		break;
1120	}
1121}
1122
1123static int dmz_message(struct dm_target *ti, unsigned int argc, char **argv,
1124		       char *result, unsigned int maxlen)
1125{
1126	struct dmz_target *dmz = ti->private;
1127	int r = -EINVAL;
1128
1129	if (!strcasecmp(argv[0], "reclaim")) {
1130		int i;
1131
1132		for (i = 0; i < dmz->nr_ddevs; i++)
1133			dmz_schedule_reclaim(dmz->dev[i].reclaim);
1134		r = 0;
1135	} else
1136		DMERR("unrecognized message %s", argv[0]);
1137	return r;
1138}
1139
1140static struct target_type zoned_target = {
1141	.name		 = "zoned",
1142	.version	 = {2, 0, 0},
1143	.features	 = DM_TARGET_SINGLETON | DM_TARGET_MIXED_ZONED_MODEL,
1144	.module		 = THIS_MODULE,
1145	.ctr		 = dmz_ctr,
1146	.dtr		 = dmz_dtr,
1147	.map		 = dmz_map,
1148	.io_hints	 = dmz_io_hints,
1149	.prepare_ioctl	 = dmz_prepare_ioctl,
1150	.postsuspend	 = dmz_suspend,
1151	.resume		 = dmz_resume,
1152	.iterate_devices = dmz_iterate_devices,
1153	.status		 = dmz_status,
1154	.message	 = dmz_message,
1155};
1156module_dm(zoned);
1157
1158MODULE_DESCRIPTION(DM_NAME " target for zoned block devices");
1159MODULE_AUTHOR("Damien Le Moal <damien.lemoal@wdc.com>");
1160MODULE_LICENSE("GPL");