Linux Audio

Check our new training course

Loading...
v4.6
   1/*
   2 * Copyright (C) 2001 Sistina Software (UK) Limited.
   3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
   4 *
   5 * This file is released under the GPL.
   6 */
   7
   8#include "dm.h"
   9
  10#include <linux/module.h>
  11#include <linux/vmalloc.h>
  12#include <linux/blkdev.h>
  13#include <linux/namei.h>
  14#include <linux/ctype.h>
  15#include <linux/string.h>
  16#include <linux/slab.h>
  17#include <linux/interrupt.h>
  18#include <linux/mutex.h>
  19#include <linux/delay.h>
  20#include <linux/atomic.h>
  21#include <linux/blk-mq.h>
  22#include <linux/mount.h>
 
  23
  24#define DM_MSG_PREFIX "table"
  25
  26#define MAX_DEPTH 16
  27#define NODE_SIZE L1_CACHE_BYTES
  28#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))
  29#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1)
  30
  31struct dm_table {
  32	struct mapped_device *md;
  33	unsigned type;
  34
  35	/* btree table */
  36	unsigned int depth;
  37	unsigned int counts[MAX_DEPTH];	/* in nodes */
  38	sector_t *index[MAX_DEPTH];
  39
  40	unsigned int num_targets;
  41	unsigned int num_allocated;
  42	sector_t *highs;
  43	struct dm_target *targets;
  44
  45	struct target_type *immutable_target_type;
  46	unsigned integrity_supported:1;
  47	unsigned singleton:1;
 
 
 
  48
  49	/*
  50	 * Indicates the rw permissions for the new logical
  51	 * device.  This should be a combination of FMODE_READ
  52	 * and FMODE_WRITE.
  53	 */
  54	fmode_t mode;
  55
  56	/* a list of devices used by this table */
  57	struct list_head devices;
  58
  59	/* events get handed up using this callback */
  60	void (*event_fn)(void *);
  61	void *event_context;
  62
  63	struct dm_md_mempools *mempools;
  64
  65	struct list_head target_callbacks;
  66};
  67
  68/*
  69 * Similar to ceiling(log_size(n))
  70 */
  71static unsigned int int_log(unsigned int n, unsigned int base)
  72{
  73	int result = 0;
  74
  75	while (n > 1) {
  76		n = dm_div_up(n, base);
  77		result++;
  78	}
  79
  80	return result;
  81}
  82
  83/*
  84 * Calculate the index of the child node of the n'th node k'th key.
  85 */
  86static inline unsigned int get_child(unsigned int n, unsigned int k)
  87{
  88	return (n * CHILDREN_PER_NODE) + k;
  89}
  90
  91/*
  92 * Return the n'th node of level l from table t.
  93 */
  94static inline sector_t *get_node(struct dm_table *t,
  95				 unsigned int l, unsigned int n)
  96{
  97	return t->index[l] + (n * KEYS_PER_NODE);
  98}
  99
 100/*
 101 * Return the highest key that you could lookup from the n'th
 102 * node on level l of the btree.
 103 */
 104static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)
 105{
 106	for (; l < t->depth - 1; l++)
 107		n = get_child(n, CHILDREN_PER_NODE - 1);
 108
 109	if (n >= t->counts[l])
 110		return (sector_t) - 1;
 111
 112	return get_node(t, l, n)[KEYS_PER_NODE - 1];
 113}
 114
 115/*
 116 * Fills in a level of the btree based on the highs of the level
 117 * below it.
 118 */
 119static int setup_btree_index(unsigned int l, struct dm_table *t)
 120{
 121	unsigned int n, k;
 122	sector_t *node;
 123
 124	for (n = 0U; n < t->counts[l]; n++) {
 125		node = get_node(t, l, n);
 126
 127		for (k = 0U; k < KEYS_PER_NODE; k++)
 128			node[k] = high(t, l + 1, get_child(n, k));
 129	}
 130
 131	return 0;
 132}
 133
 134void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size)
 135{
 136	unsigned long size;
 137	void *addr;
 138
 139	/*
 140	 * Check that we're not going to overflow.
 141	 */
 142	if (nmemb > (ULONG_MAX / elem_size))
 143		return NULL;
 144
 145	size = nmemb * elem_size;
 146	addr = vzalloc(size);
 147
 148	return addr;
 149}
 150EXPORT_SYMBOL(dm_vcalloc);
 151
 152/*
 153 * highs, and targets are managed as dynamic arrays during a
 154 * table load.
 155 */
 156static int alloc_targets(struct dm_table *t, unsigned int num)
 157{
 158	sector_t *n_highs;
 159	struct dm_target *n_targets;
 160
 161	/*
 162	 * Allocate both the target array and offset array at once.
 163	 * Append an empty entry to catch sectors beyond the end of
 164	 * the device.
 165	 */
 166	n_highs = (sector_t *) dm_vcalloc(num + 1, sizeof(struct dm_target) +
 167					  sizeof(sector_t));
 168	if (!n_highs)
 169		return -ENOMEM;
 170
 171	n_targets = (struct dm_target *) (n_highs + num);
 172
 173	memset(n_highs, -1, sizeof(*n_highs) * num);
 174	vfree(t->highs);
 175
 176	t->num_allocated = num;
 177	t->highs = n_highs;
 178	t->targets = n_targets;
 179
 180	return 0;
 181}
 182
 183int dm_table_create(struct dm_table **result, fmode_t mode,
 184		    unsigned num_targets, struct mapped_device *md)
 185{
 186	struct dm_table *t = kzalloc(sizeof(*t), GFP_KERNEL);
 187
 188	if (!t)
 189		return -ENOMEM;
 190
 191	INIT_LIST_HEAD(&t->devices);
 192	INIT_LIST_HEAD(&t->target_callbacks);
 193
 194	if (!num_targets)
 195		num_targets = KEYS_PER_NODE;
 196
 197	num_targets = dm_round_up(num_targets, KEYS_PER_NODE);
 198
 199	if (!num_targets) {
 200		kfree(t);
 201		return -ENOMEM;
 202	}
 203
 204	if (alloc_targets(t, num_targets)) {
 205		kfree(t);
 206		return -ENOMEM;
 207	}
 208
 
 209	t->mode = mode;
 210	t->md = md;
 211	*result = t;
 212	return 0;
 213}
 214
 215static void free_devices(struct list_head *devices, struct mapped_device *md)
 216{
 217	struct list_head *tmp, *next;
 218
 219	list_for_each_safe(tmp, next, devices) {
 220		struct dm_dev_internal *dd =
 221		    list_entry(tmp, struct dm_dev_internal, list);
 222		DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s",
 223		       dm_device_name(md), dd->dm_dev->name);
 224		dm_put_table_device(md, dd->dm_dev);
 225		kfree(dd);
 226	}
 227}
 228
 229void dm_table_destroy(struct dm_table *t)
 230{
 231	unsigned int i;
 232
 233	if (!t)
 234		return;
 235
 236	/* free the indexes */
 237	if (t->depth >= 2)
 238		vfree(t->index[t->depth - 2]);
 239
 240	/* free the targets */
 241	for (i = 0; i < t->num_targets; i++) {
 242		struct dm_target *tgt = t->targets + i;
 243
 244		if (tgt->type->dtr)
 245			tgt->type->dtr(tgt);
 246
 247		dm_put_target_type(tgt->type);
 248	}
 249
 250	vfree(t->highs);
 251
 252	/* free the device list */
 253	free_devices(&t->devices, t->md);
 254
 255	dm_free_md_mempools(t->mempools);
 256
 257	kfree(t);
 258}
 259
 260/*
 261 * See if we've already got a device in the list.
 262 */
 263static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev)
 264{
 265	struct dm_dev_internal *dd;
 266
 267	list_for_each_entry (dd, l, list)
 268		if (dd->dm_dev->bdev->bd_dev == dev)
 269			return dd;
 270
 271	return NULL;
 272}
 273
 274/*
 275 * If possible, this checks an area of a destination device is invalid.
 276 */
 277static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev,
 278				  sector_t start, sector_t len, void *data)
 279{
 280	struct request_queue *q;
 281	struct queue_limits *limits = data;
 282	struct block_device *bdev = dev->bdev;
 283	sector_t dev_size =
 284		i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
 285	unsigned short logical_block_size_sectors =
 286		limits->logical_block_size >> SECTOR_SHIFT;
 287	char b[BDEVNAME_SIZE];
 288
 289	/*
 290	 * Some devices exist without request functions,
 291	 * such as loop devices not yet bound to backing files.
 292	 * Forbid the use of such devices.
 293	 */
 294	q = bdev_get_queue(bdev);
 295	if (!q || !q->make_request_fn) {
 296		DMWARN("%s: %s is not yet initialised: "
 297		       "start=%llu, len=%llu, dev_size=%llu",
 298		       dm_device_name(ti->table->md), bdevname(bdev, b),
 299		       (unsigned long long)start,
 300		       (unsigned long long)len,
 301		       (unsigned long long)dev_size);
 302		return 1;
 303	}
 304
 305	if (!dev_size)
 306		return 0;
 307
 308	if ((start >= dev_size) || (start + len > dev_size)) {
 309		DMWARN("%s: %s too small for target: "
 310		       "start=%llu, len=%llu, dev_size=%llu",
 311		       dm_device_name(ti->table->md), bdevname(bdev, b),
 312		       (unsigned long long)start,
 313		       (unsigned long long)len,
 314		       (unsigned long long)dev_size);
 315		return 1;
 316	}
 317
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 318	if (logical_block_size_sectors <= 1)
 319		return 0;
 320
 321	if (start & (logical_block_size_sectors - 1)) {
 322		DMWARN("%s: start=%llu not aligned to h/w "
 323		       "logical block size %u of %s",
 324		       dm_device_name(ti->table->md),
 325		       (unsigned long long)start,
 326		       limits->logical_block_size, bdevname(bdev, b));
 327		return 1;
 328	}
 329
 330	if (len & (logical_block_size_sectors - 1)) {
 331		DMWARN("%s: len=%llu not aligned to h/w "
 332		       "logical block size %u of %s",
 333		       dm_device_name(ti->table->md),
 334		       (unsigned long long)len,
 335		       limits->logical_block_size, bdevname(bdev, b));
 336		return 1;
 337	}
 338
 339	return 0;
 340}
 341
 342/*
 343 * This upgrades the mode on an already open dm_dev, being
 344 * careful to leave things as they were if we fail to reopen the
 345 * device and not to touch the existing bdev field in case
 346 * it is accessed concurrently inside dm_table_any_congested().
 347 */
 348static int upgrade_mode(struct dm_dev_internal *dd, fmode_t new_mode,
 349			struct mapped_device *md)
 350{
 351	int r;
 352	struct dm_dev *old_dev, *new_dev;
 353
 354	old_dev = dd->dm_dev;
 355
 356	r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev,
 357				dd->dm_dev->mode | new_mode, &new_dev);
 358	if (r)
 359		return r;
 360
 361	dd->dm_dev = new_dev;
 362	dm_put_table_device(md, old_dev);
 363
 364	return 0;
 365}
 366
 367/*
 368 * Convert the path to a device
 369 */
 370dev_t dm_get_dev_t(const char *path)
 371{
 372	dev_t uninitialized_var(dev);
 373	struct block_device *bdev;
 374
 375	bdev = lookup_bdev(path);
 376	if (IS_ERR(bdev))
 377		dev = name_to_dev_t(path);
 378	else {
 379		dev = bdev->bd_dev;
 380		bdput(bdev);
 381	}
 382
 383	return dev;
 384}
 385EXPORT_SYMBOL_GPL(dm_get_dev_t);
 386
 387/*
 388 * Add a device to the list, or just increment the usage count if
 389 * it's already present.
 390 */
 391int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode,
 392		  struct dm_dev **result)
 393{
 394	int r;
 395	dev_t dev;
 396	struct dm_dev_internal *dd;
 397	struct dm_table *t = ti->table;
 398
 399	BUG_ON(!t);
 400
 401	dev = dm_get_dev_t(path);
 402	if (!dev)
 403		return -ENODEV;
 404
 405	dd = find_device(&t->devices, dev);
 406	if (!dd) {
 407		dd = kmalloc(sizeof(*dd), GFP_KERNEL);
 408		if (!dd)
 409			return -ENOMEM;
 410
 411		if ((r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev))) {
 412			kfree(dd);
 413			return r;
 414		}
 415
 416		atomic_set(&dd->count, 0);
 417		list_add(&dd->list, &t->devices);
 
 418
 419	} else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) {
 420		r = upgrade_mode(dd, mode, t->md);
 421		if (r)
 422			return r;
 423	}
 424	atomic_inc(&dd->count);
 425
 426	*result = dd->dm_dev;
 427	return 0;
 428}
 429EXPORT_SYMBOL(dm_get_device);
 430
 431static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev,
 432				sector_t start, sector_t len, void *data)
 433{
 434	struct queue_limits *limits = data;
 435	struct block_device *bdev = dev->bdev;
 436	struct request_queue *q = bdev_get_queue(bdev);
 437	char b[BDEVNAME_SIZE];
 438
 439	if (unlikely(!q)) {
 440		DMWARN("%s: Cannot set limits for nonexistent device %s",
 441		       dm_device_name(ti->table->md), bdevname(bdev, b));
 442		return 0;
 443	}
 444
 445	if (bdev_stack_limits(limits, bdev, start) < 0)
 446		DMWARN("%s: adding target device %s caused an alignment inconsistency: "
 447		       "physical_block_size=%u, logical_block_size=%u, "
 448		       "alignment_offset=%u, start=%llu",
 449		       dm_device_name(ti->table->md), bdevname(bdev, b),
 450		       q->limits.physical_block_size,
 451		       q->limits.logical_block_size,
 452		       q->limits.alignment_offset,
 453		       (unsigned long long) start << SECTOR_SHIFT);
 454
 
 
 455	return 0;
 456}
 457
 458/*
 459 * Decrement a device's use count and remove it if necessary.
 460 */
 461void dm_put_device(struct dm_target *ti, struct dm_dev *d)
 462{
 463	int found = 0;
 464	struct list_head *devices = &ti->table->devices;
 465	struct dm_dev_internal *dd;
 466
 467	list_for_each_entry(dd, devices, list) {
 468		if (dd->dm_dev == d) {
 469			found = 1;
 470			break;
 471		}
 472	}
 473	if (!found) {
 474		DMWARN("%s: device %s not in table devices list",
 475		       dm_device_name(ti->table->md), d->name);
 476		return;
 477	}
 478	if (atomic_dec_and_test(&dd->count)) {
 479		dm_put_table_device(ti->table->md, d);
 480		list_del(&dd->list);
 481		kfree(dd);
 482	}
 483}
 484EXPORT_SYMBOL(dm_put_device);
 485
 486/*
 487 * Checks to see if the target joins onto the end of the table.
 488 */
 489static int adjoin(struct dm_table *table, struct dm_target *ti)
 490{
 491	struct dm_target *prev;
 492
 493	if (!table->num_targets)
 494		return !ti->begin;
 495
 496	prev = &table->targets[table->num_targets - 1];
 497	return (ti->begin == (prev->begin + prev->len));
 498}
 499
 500/*
 501 * Used to dynamically allocate the arg array.
 502 *
 503 * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must
 504 * process messages even if some device is suspended. These messages have a
 505 * small fixed number of arguments.
 506 *
 507 * On the other hand, dm-switch needs to process bulk data using messages and
 508 * excessive use of GFP_NOIO could cause trouble.
 509 */
 510static char **realloc_argv(unsigned *array_size, char **old_argv)
 511{
 512	char **argv;
 513	unsigned new_size;
 514	gfp_t gfp;
 515
 516	if (*array_size) {
 517		new_size = *array_size * 2;
 518		gfp = GFP_KERNEL;
 519	} else {
 520		new_size = 8;
 521		gfp = GFP_NOIO;
 522	}
 523	argv = kmalloc(new_size * sizeof(*argv), gfp);
 524	if (argv) {
 525		memcpy(argv, old_argv, *array_size * sizeof(*argv));
 526		*array_size = new_size;
 527	}
 528
 529	kfree(old_argv);
 530	return argv;
 531}
 532
 533/*
 534 * Destructively splits up the argument list to pass to ctr.
 535 */
 536int dm_split_args(int *argc, char ***argvp, char *input)
 537{
 538	char *start, *end = input, *out, **argv = NULL;
 539	unsigned array_size = 0;
 540
 541	*argc = 0;
 542
 543	if (!input) {
 544		*argvp = NULL;
 545		return 0;
 546	}
 547
 548	argv = realloc_argv(&array_size, argv);
 549	if (!argv)
 550		return -ENOMEM;
 551
 552	while (1) {
 553		/* Skip whitespace */
 554		start = skip_spaces(end);
 555
 556		if (!*start)
 557			break;	/* success, we hit the end */
 558
 559		/* 'out' is used to remove any back-quotes */
 560		end = out = start;
 561		while (*end) {
 562			/* Everything apart from '\0' can be quoted */
 563			if (*end == '\\' && *(end + 1)) {
 564				*out++ = *(end + 1);
 565				end += 2;
 566				continue;
 567			}
 568
 569			if (isspace(*end))
 570				break;	/* end of token */
 571
 572			*out++ = *end++;
 573		}
 574
 575		/* have we already filled the array ? */
 576		if ((*argc + 1) > array_size) {
 577			argv = realloc_argv(&array_size, argv);
 578			if (!argv)
 579				return -ENOMEM;
 580		}
 581
 582		/* we know this is whitespace */
 583		if (*end)
 584			end++;
 585
 586		/* terminate the string and put it in the array */
 587		*out = '\0';
 588		argv[*argc] = start;
 589		(*argc)++;
 590	}
 591
 592	*argvp = argv;
 593	return 0;
 594}
 595
 596/*
 597 * Impose necessary and sufficient conditions on a devices's table such
 598 * that any incoming bio which respects its logical_block_size can be
 599 * processed successfully.  If it falls across the boundary between
 600 * two or more targets, the size of each piece it gets split into must
 601 * be compatible with the logical_block_size of the target processing it.
 602 */
 603static int validate_hardware_logical_block_alignment(struct dm_table *table,
 604						 struct queue_limits *limits)
 605{
 606	/*
 607	 * This function uses arithmetic modulo the logical_block_size
 608	 * (in units of 512-byte sectors).
 609	 */
 610	unsigned short device_logical_block_size_sects =
 611		limits->logical_block_size >> SECTOR_SHIFT;
 612
 613	/*
 614	 * Offset of the start of the next table entry, mod logical_block_size.
 615	 */
 616	unsigned short next_target_start = 0;
 617
 618	/*
 619	 * Given an aligned bio that extends beyond the end of a
 620	 * target, how many sectors must the next target handle?
 621	 */
 622	unsigned short remaining = 0;
 623
 624	struct dm_target *uninitialized_var(ti);
 625	struct queue_limits ti_limits;
 626	unsigned i = 0;
 627
 628	/*
 629	 * Check each entry in the table in turn.
 630	 */
 631	while (i < dm_table_get_num_targets(table)) {
 632		ti = dm_table_get_target(table, i++);
 633
 634		blk_set_stacking_limits(&ti_limits);
 635
 636		/* combine all target devices' limits */
 637		if (ti->type->iterate_devices)
 638			ti->type->iterate_devices(ti, dm_set_device_limits,
 639						  &ti_limits);
 640
 641		/*
 642		 * If the remaining sectors fall entirely within this
 643		 * table entry are they compatible with its logical_block_size?
 644		 */
 645		if (remaining < ti->len &&
 646		    remaining & ((ti_limits.logical_block_size >>
 647				  SECTOR_SHIFT) - 1))
 648			break;	/* Error */
 649
 650		next_target_start =
 651		    (unsigned short) ((next_target_start + ti->len) &
 652				      (device_logical_block_size_sects - 1));
 653		remaining = next_target_start ?
 654		    device_logical_block_size_sects - next_target_start : 0;
 655	}
 656
 657	if (remaining) {
 658		DMWARN("%s: table line %u (start sect %llu len %llu) "
 659		       "not aligned to h/w logical block size %u",
 660		       dm_device_name(table->md), i,
 661		       (unsigned long long) ti->begin,
 662		       (unsigned long long) ti->len,
 663		       limits->logical_block_size);
 664		return -EINVAL;
 665	}
 666
 667	return 0;
 668}
 669
 670int dm_table_add_target(struct dm_table *t, const char *type,
 671			sector_t start, sector_t len, char *params)
 672{
 673	int r = -EINVAL, argc;
 674	char **argv;
 675	struct dm_target *tgt;
 676
 677	if (t->singleton) {
 678		DMERR("%s: target type %s must appear alone in table",
 679		      dm_device_name(t->md), t->targets->type->name);
 680		return -EINVAL;
 681	}
 682
 683	BUG_ON(t->num_targets >= t->num_allocated);
 684
 685	tgt = t->targets + t->num_targets;
 686	memset(tgt, 0, sizeof(*tgt));
 687
 688	if (!len) {
 689		DMERR("%s: zero-length target", dm_device_name(t->md));
 690		return -EINVAL;
 691	}
 692
 693	tgt->type = dm_get_target_type(type);
 694	if (!tgt->type) {
 695		DMERR("%s: %s: unknown target type", dm_device_name(t->md),
 696		      type);
 697		return -EINVAL;
 698	}
 699
 700	if (dm_target_needs_singleton(tgt->type)) {
 701		if (t->num_targets) {
 702			DMERR("%s: target type %s must appear alone in table",
 703			      dm_device_name(t->md), type);
 704			return -EINVAL;
 705		}
 706		t->singleton = 1;
 707	}
 708
 709	if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) {
 710		DMERR("%s: target type %s may not be included in read-only tables",
 711		      dm_device_name(t->md), type);
 712		return -EINVAL;
 713	}
 714
 715	if (t->immutable_target_type) {
 716		if (t->immutable_target_type != tgt->type) {
 717			DMERR("%s: immutable target type %s cannot be mixed with other target types",
 718			      dm_device_name(t->md), t->immutable_target_type->name);
 719			return -EINVAL;
 720		}
 721	} else if (dm_target_is_immutable(tgt->type)) {
 722		if (t->num_targets) {
 723			DMERR("%s: immutable target type %s cannot be mixed with other target types",
 724			      dm_device_name(t->md), tgt->type->name);
 725			return -EINVAL;
 726		}
 727		t->immutable_target_type = tgt->type;
 728	}
 729
 
 
 
 730	tgt->table = t;
 731	tgt->begin = start;
 732	tgt->len = len;
 733	tgt->error = "Unknown error";
 734
 735	/*
 736	 * Does this target adjoin the previous one ?
 737	 */
 738	if (!adjoin(t, tgt)) {
 739		tgt->error = "Gap in table";
 740		r = -EINVAL;
 741		goto bad;
 742	}
 743
 744	r = dm_split_args(&argc, &argv, params);
 745	if (r) {
 746		tgt->error = "couldn't split parameters (insufficient memory)";
 747		goto bad;
 748	}
 749
 750	r = tgt->type->ctr(tgt, argc, argv);
 751	kfree(argv);
 752	if (r)
 753		goto bad;
 754
 755	t->highs[t->num_targets++] = tgt->begin + tgt->len - 1;
 756
 757	if (!tgt->num_discard_bios && tgt->discards_supported)
 758		DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",
 759		       dm_device_name(t->md), type);
 760
 761	return 0;
 762
 763 bad:
 764	DMERR("%s: %s: %s", dm_device_name(t->md), type, tgt->error);
 765	dm_put_target_type(tgt->type);
 766	return r;
 767}
 768
 769/*
 770 * Target argument parsing helpers.
 771 */
 772static int validate_next_arg(struct dm_arg *arg, struct dm_arg_set *arg_set,
 
 773			     unsigned *value, char **error, unsigned grouped)
 774{
 775	const char *arg_str = dm_shift_arg(arg_set);
 776	char dummy;
 777
 778	if (!arg_str ||
 779	    (sscanf(arg_str, "%u%c", value, &dummy) != 1) ||
 780	    (*value < arg->min) ||
 781	    (*value > arg->max) ||
 782	    (grouped && arg_set->argc < *value)) {
 783		*error = arg->error;
 784		return -EINVAL;
 785	}
 786
 787	return 0;
 788}
 789
 790int dm_read_arg(struct dm_arg *arg, struct dm_arg_set *arg_set,
 791		unsigned *value, char **error)
 792{
 793	return validate_next_arg(arg, arg_set, value, error, 0);
 794}
 795EXPORT_SYMBOL(dm_read_arg);
 796
 797int dm_read_arg_group(struct dm_arg *arg, struct dm_arg_set *arg_set,
 798		      unsigned *value, char **error)
 799{
 800	return validate_next_arg(arg, arg_set, value, error, 1);
 801}
 802EXPORT_SYMBOL(dm_read_arg_group);
 803
 804const char *dm_shift_arg(struct dm_arg_set *as)
 805{
 806	char *r;
 807
 808	if (as->argc) {
 809		as->argc--;
 810		r = *as->argv;
 811		as->argv++;
 812		return r;
 813	}
 814
 815	return NULL;
 816}
 817EXPORT_SYMBOL(dm_shift_arg);
 818
 819void dm_consume_args(struct dm_arg_set *as, unsigned num_args)
 820{
 821	BUG_ON(as->argc < num_args);
 822	as->argc -= num_args;
 823	as->argv += num_args;
 824}
 825EXPORT_SYMBOL(dm_consume_args);
 826
 827static bool __table_type_request_based(unsigned table_type)
 
 
 
 
 
 
 
 828{
 829	return (table_type == DM_TYPE_REQUEST_BASED ||
 830		table_type == DM_TYPE_MQ_REQUEST_BASED);
 831}
 832
 833static int dm_table_set_type(struct dm_table *t)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 834{
 835	unsigned i;
 836	unsigned bio_based = 0, request_based = 0, hybrid = 0;
 837	bool use_blk_mq = false;
 838	struct dm_target *tgt;
 839	struct dm_dev_internal *dd;
 840	struct list_head *devices;
 841	unsigned live_md_type = dm_get_md_type(t->md);
 
 
 
 
 
 
 
 
 
 
 842
 843	for (i = 0; i < t->num_targets; i++) {
 844		tgt = t->targets + i;
 845		if (dm_target_hybrid(tgt))
 846			hybrid = 1;
 847		else if (dm_target_request_based(tgt))
 848			request_based = 1;
 849		else
 850			bio_based = 1;
 851
 852		if (bio_based && request_based) {
 853			DMWARN("Inconsistent table: different target types"
 854			       " can't be mixed up");
 855			return -EINVAL;
 856		}
 857	}
 858
 859	if (hybrid && !bio_based && !request_based) {
 860		/*
 861		 * The targets can work either way.
 862		 * Determine the type from the live device.
 863		 * Default to bio-based if device is new.
 864		 */
 865		if (__table_type_request_based(live_md_type))
 866			request_based = 1;
 867		else
 868			bio_based = 1;
 869	}
 870
 871	if (bio_based) {
 
 872		/* We must use this table as bio-based */
 873		t->type = DM_TYPE_BIO_BASED;
 
 
 
 
 
 
 
 
 
 
 
 
 
 874		return 0;
 875	}
 876
 877	BUG_ON(!request_based); /* No targets in this table */
 878
 879	/*
 
 
 
 
 
 
 
 880	 * Request-based dm supports only tables that have a single target now.
 881	 * To support multiple targets, request splitting support is needed,
 882	 * and that needs lots of changes in the block-layer.
 883	 * (e.g. request completion process for partial completion.)
 884	 */
 885	if (t->num_targets > 1) {
 886		DMWARN("Request-based dm doesn't support multiple targets yet");
 
 887		return -EINVAL;
 888	}
 889
 890	/* Non-request-stackable devices can't be used for request-based dm */
 891	devices = dm_table_get_devices(t);
 892	list_for_each_entry(dd, devices, list) {
 893		struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev);
 894
 895		if (!blk_queue_stackable(q)) {
 896			DMERR("table load rejected: including"
 897			      " non-request-stackable devices");
 898			return -EINVAL;
 899		}
 900
 901		if (q->mq_ops)
 902			use_blk_mq = true;
 903	}
 904
 905	if (use_blk_mq) {
 906		/* verify _all_ devices in the table are blk-mq devices */
 907		list_for_each_entry(dd, devices, list)
 908			if (!bdev_get_queue(dd->dm_dev->bdev)->mq_ops) {
 909				DMERR("table load rejected: not all devices"
 910				      " are blk-mq request-stackable");
 911				return -EINVAL;
 912			}
 913		t->type = DM_TYPE_MQ_REQUEST_BASED;
 914
 915	} else if (list_empty(devices) && __table_type_request_based(live_md_type)) {
 916		/* inherit live MD type */
 917		t->type = live_md_type;
 
 
 
 
 
 
 
 
 918
 919	} else
 920		t->type = DM_TYPE_REQUEST_BASED;
 
 
 
 921
 922	return 0;
 923}
 924
 925unsigned dm_table_get_type(struct dm_table *t)
 926{
 927	return t->type;
 928}
 929
 930struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)
 931{
 932	return t->immutable_target_type;
 933}
 934
 935struct dm_target *dm_table_get_immutable_target(struct dm_table *t)
 936{
 937	/* Immutable target is implicitly a singleton */
 938	if (t->num_targets > 1 ||
 939	    !dm_target_is_immutable(t->targets[0].type))
 940		return NULL;
 941
 942	return t->targets;
 943}
 944
 945struct dm_target *dm_table_get_wildcard_target(struct dm_table *t)
 946{
 947	struct dm_target *uninitialized_var(ti);
 948	unsigned i = 0;
 949
 950	while (i < dm_table_get_num_targets(t)) {
 951		ti = dm_table_get_target(t, i++);
 952		if (dm_target_is_wildcard(ti->type))
 953			return ti;
 954	}
 955
 956	return NULL;
 957}
 958
 
 
 
 
 
 959bool dm_table_request_based(struct dm_table *t)
 960{
 961	return __table_type_request_based(dm_table_get_type(t));
 962}
 963
 964bool dm_table_mq_request_based(struct dm_table *t)
 965{
 966	return dm_table_get_type(t) == DM_TYPE_MQ_REQUEST_BASED;
 967}
 968
 969static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md)
 970{
 971	unsigned type = dm_table_get_type(t);
 972	unsigned per_io_data_size = 0;
 973	struct dm_target *tgt;
 
 974	unsigned i;
 975
 976	if (unlikely(type == DM_TYPE_NONE)) {
 977		DMWARN("no table type is set, can't allocate mempools");
 978		return -EINVAL;
 979	}
 980
 981	if (type == DM_TYPE_BIO_BASED)
 982		for (i = 0; i < t->num_targets; i++) {
 983			tgt = t->targets + i;
 984			per_io_data_size = max(per_io_data_size, tgt->per_io_data_size);
 
 985		}
 986
 987	t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported, per_io_data_size);
 
 988	if (!t->mempools)
 989		return -ENOMEM;
 990
 991	return 0;
 992}
 993
 994void dm_table_free_md_mempools(struct dm_table *t)
 995{
 996	dm_free_md_mempools(t->mempools);
 997	t->mempools = NULL;
 998}
 999
1000struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t)
1001{
1002	return t->mempools;
1003}
1004
1005static int setup_indexes(struct dm_table *t)
1006{
1007	int i;
1008	unsigned int total = 0;
1009	sector_t *indexes;
1010
1011	/* allocate the space for *all* the indexes */
1012	for (i = t->depth - 2; i >= 0; i--) {
1013		t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);
1014		total += t->counts[i];
1015	}
1016
1017	indexes = (sector_t *) dm_vcalloc(total, (unsigned long) NODE_SIZE);
1018	if (!indexes)
1019		return -ENOMEM;
1020
1021	/* set up internal nodes, bottom-up */
1022	for (i = t->depth - 2; i >= 0; i--) {
1023		t->index[i] = indexes;
1024		indexes += (KEYS_PER_NODE * t->counts[i]);
1025		setup_btree_index(i, t);
1026	}
1027
1028	return 0;
1029}
1030
1031/*
1032 * Builds the btree to index the map.
1033 */
1034static int dm_table_build_index(struct dm_table *t)
1035{
1036	int r = 0;
1037	unsigned int leaf_nodes;
1038
1039	/* how many indexes will the btree have ? */
1040	leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
1041	t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);
1042
1043	/* leaf layer has already been set up */
1044	t->counts[t->depth - 1] = leaf_nodes;
1045	t->index[t->depth - 1] = t->highs;
1046
1047	if (t->depth >= 2)
1048		r = setup_indexes(t);
1049
1050	return r;
1051}
1052
1053static bool integrity_profile_exists(struct gendisk *disk)
1054{
1055	return !!blk_get_integrity(disk);
1056}
1057
1058/*
1059 * Get a disk whose integrity profile reflects the table's profile.
1060 * Returns NULL if integrity support was inconsistent or unavailable.
1061 */
1062static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t)
1063{
1064	struct list_head *devices = dm_table_get_devices(t);
1065	struct dm_dev_internal *dd = NULL;
1066	struct gendisk *prev_disk = NULL, *template_disk = NULL;
 
 
 
 
 
 
 
1067
1068	list_for_each_entry(dd, devices, list) {
1069		template_disk = dd->dm_dev->bdev->bd_disk;
1070		if (!integrity_profile_exists(template_disk))
1071			goto no_integrity;
1072		else if (prev_disk &&
1073			 blk_integrity_compare(prev_disk, template_disk) < 0)
1074			goto no_integrity;
1075		prev_disk = template_disk;
1076	}
1077
1078	return template_disk;
1079
1080no_integrity:
1081	if (prev_disk)
1082		DMWARN("%s: integrity not set: %s and %s profile mismatch",
1083		       dm_device_name(t->md),
1084		       prev_disk->disk_name,
1085		       template_disk->disk_name);
1086	return NULL;
1087}
1088
1089/*
1090 * Register the mapped device for blk_integrity support if the
1091 * underlying devices have an integrity profile.  But all devices may
1092 * not have matching profiles (checking all devices isn't reliable
1093 * during table load because this table may use other DM device(s) which
1094 * must be resumed before they will have an initialized integity
1095 * profile).  Consequently, stacked DM devices force a 2 stage integrity
1096 * profile validation: First pass during table load, final pass during
1097 * resume.
1098 */
1099static int dm_table_register_integrity(struct dm_table *t)
1100{
1101	struct mapped_device *md = t->md;
1102	struct gendisk *template_disk = NULL;
1103
 
 
 
 
1104	template_disk = dm_table_get_integrity_disk(t);
1105	if (!template_disk)
1106		return 0;
1107
1108	if (!integrity_profile_exists(dm_disk(md))) {
1109		t->integrity_supported = 1;
1110		/*
1111		 * Register integrity profile during table load; we can do
1112		 * this because the final profile must match during resume.
1113		 */
1114		blk_integrity_register(dm_disk(md),
1115				       blk_get_integrity(template_disk));
1116		return 0;
1117	}
1118
1119	/*
1120	 * If DM device already has an initialized integrity
1121	 * profile the new profile should not conflict.
1122	 */
1123	if (blk_integrity_compare(dm_disk(md), template_disk) < 0) {
1124		DMWARN("%s: conflict with existing integrity profile: "
1125		       "%s profile mismatch",
1126		       dm_device_name(t->md),
1127		       template_disk->disk_name);
1128		return 1;
1129	}
1130
1131	/* Preserve existing integrity profile */
1132	t->integrity_supported = 1;
1133	return 0;
1134}
1135
1136/*
1137 * Prepares the table for use by building the indices,
1138 * setting the type, and allocating mempools.
1139 */
1140int dm_table_complete(struct dm_table *t)
1141{
1142	int r;
1143
1144	r = dm_table_set_type(t);
1145	if (r) {
1146		DMERR("unable to set table type");
1147		return r;
1148	}
1149
1150	r = dm_table_build_index(t);
1151	if (r) {
1152		DMERR("unable to build btrees");
1153		return r;
1154	}
1155
1156	r = dm_table_register_integrity(t);
1157	if (r) {
1158		DMERR("could not register integrity profile.");
1159		return r;
1160	}
1161
1162	r = dm_table_alloc_md_mempools(t, t->md);
1163	if (r)
1164		DMERR("unable to allocate mempools");
1165
1166	return r;
1167}
1168
1169static DEFINE_MUTEX(_event_lock);
1170void dm_table_event_callback(struct dm_table *t,
1171			     void (*fn)(void *), void *context)
1172{
1173	mutex_lock(&_event_lock);
1174	t->event_fn = fn;
1175	t->event_context = context;
1176	mutex_unlock(&_event_lock);
1177}
1178
1179void dm_table_event(struct dm_table *t)
1180{
1181	/*
1182	 * You can no longer call dm_table_event() from interrupt
1183	 * context, use a bottom half instead.
1184	 */
1185	BUG_ON(in_interrupt());
1186
1187	mutex_lock(&_event_lock);
1188	if (t->event_fn)
1189		t->event_fn(t->event_context);
1190	mutex_unlock(&_event_lock);
1191}
1192EXPORT_SYMBOL(dm_table_event);
1193
1194sector_t dm_table_get_size(struct dm_table *t)
1195{
1196	return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;
1197}
1198EXPORT_SYMBOL(dm_table_get_size);
1199
1200struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index)
1201{
1202	if (index >= t->num_targets)
1203		return NULL;
1204
1205	return t->targets + index;
1206}
1207
1208/*
1209 * Search the btree for the correct target.
1210 *
1211 * Caller should check returned pointer with dm_target_is_valid()
1212 * to trap I/O beyond end of device.
1213 */
1214struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
1215{
1216	unsigned int l, n = 0, k = 0;
1217	sector_t *node;
1218
1219	for (l = 0; l < t->depth; l++) {
1220		n = get_child(n, k);
1221		node = get_node(t, l, n);
1222
1223		for (k = 0; k < KEYS_PER_NODE; k++)
1224			if (node[k] >= sector)
1225				break;
1226	}
1227
1228	return &t->targets[(KEYS_PER_NODE * n) + k];
1229}
1230
1231static int count_device(struct dm_target *ti, struct dm_dev *dev,
1232			sector_t start, sector_t len, void *data)
1233{
1234	unsigned *num_devices = data;
1235
1236	(*num_devices)++;
1237
1238	return 0;
1239}
1240
1241/*
1242 * Check whether a table has no data devices attached using each
1243 * target's iterate_devices method.
1244 * Returns false if the result is unknown because a target doesn't
1245 * support iterate_devices.
1246 */
1247bool dm_table_has_no_data_devices(struct dm_table *table)
1248{
1249	struct dm_target *uninitialized_var(ti);
1250	unsigned i = 0, num_devices = 0;
1251
1252	while (i < dm_table_get_num_targets(table)) {
1253		ti = dm_table_get_target(table, i++);
1254
1255		if (!ti->type->iterate_devices)
1256			return false;
1257
 
1258		ti->type->iterate_devices(ti, count_device, &num_devices);
1259		if (num_devices)
1260			return false;
1261	}
1262
1263	return true;
1264}
1265
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1266/*
1267 * Establish the new table's queue_limits and validate them.
1268 */
1269int dm_calculate_queue_limits(struct dm_table *table,
1270			      struct queue_limits *limits)
1271{
1272	struct dm_target *uninitialized_var(ti);
1273	struct queue_limits ti_limits;
1274	unsigned i = 0;
 
 
1275
1276	blk_set_stacking_limits(limits);
1277
1278	while (i < dm_table_get_num_targets(table)) {
1279		blk_set_stacking_limits(&ti_limits);
1280
1281		ti = dm_table_get_target(table, i++);
1282
1283		if (!ti->type->iterate_devices)
1284			goto combine_limits;
1285
1286		/*
1287		 * Combine queue limits of all the devices this target uses.
1288		 */
1289		ti->type->iterate_devices(ti, dm_set_device_limits,
1290					  &ti_limits);
1291
 
 
 
 
 
 
 
 
 
1292		/* Set I/O hints portion of queue limits */
1293		if (ti->type->io_hints)
1294			ti->type->io_hints(ti, &ti_limits);
1295
1296		/*
1297		 * Check each device area is consistent with the target's
1298		 * overall queue limits.
1299		 */
1300		if (ti->type->iterate_devices(ti, device_area_is_invalid,
1301					      &ti_limits))
1302			return -EINVAL;
1303
1304combine_limits:
1305		/*
1306		 * Merge this target's queue limits into the overall limits
1307		 * for the table.
1308		 */
1309		if (blk_stack_limits(limits, &ti_limits, 0) < 0)
1310			DMWARN("%s: adding target device "
1311			       "(start sect %llu len %llu) "
1312			       "caused an alignment inconsistency",
1313			       dm_device_name(table->md),
1314			       (unsigned long long) ti->begin,
1315			       (unsigned long long) ti->len);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1316	}
 
 
1317
1318	return validate_hardware_logical_block_alignment(table, limits);
1319}
1320
1321/*
1322 * Verify that all devices have an integrity profile that matches the
1323 * DM device's registered integrity profile.  If the profiles don't
1324 * match then unregister the DM device's integrity profile.
1325 */
1326static void dm_table_verify_integrity(struct dm_table *t)
1327{
1328	struct gendisk *template_disk = NULL;
1329
 
 
 
1330	if (t->integrity_supported) {
1331		/*
1332		 * Verify that the original integrity profile
1333		 * matches all the devices in this table.
1334		 */
1335		template_disk = dm_table_get_integrity_disk(t);
1336		if (template_disk &&
1337		    blk_integrity_compare(dm_disk(t->md), template_disk) >= 0)
1338			return;
1339	}
1340
1341	if (integrity_profile_exists(dm_disk(t->md))) {
1342		DMWARN("%s: unable to establish an integrity profile",
1343		       dm_device_name(t->md));
1344		blk_integrity_unregister(dm_disk(t->md));
1345	}
1346}
1347
1348static int device_flush_capable(struct dm_target *ti, struct dm_dev *dev,
1349				sector_t start, sector_t len, void *data)
1350{
1351	unsigned flush = (*(unsigned *)data);
1352	struct request_queue *q = bdev_get_queue(dev->bdev);
1353
1354	return q && (q->flush_flags & flush);
1355}
1356
1357static bool dm_table_supports_flush(struct dm_table *t, unsigned flush)
1358{
1359	struct dm_target *ti;
1360	unsigned i = 0;
1361
1362	/*
1363	 * Require at least one underlying device to support flushes.
1364	 * t->devices includes internal dm devices such as mirror logs
1365	 * so we need to use iterate_devices here, which targets
1366	 * supporting flushes must provide.
1367	 */
1368	while (i < dm_table_get_num_targets(t)) {
1369		ti = dm_table_get_target(t, i++);
1370
1371		if (!ti->num_flush_bios)
1372			continue;
1373
1374		if (ti->flush_supported)
1375			return true;
1376
1377		if (ti->type->iterate_devices &&
1378		    ti->type->iterate_devices(ti, device_flush_capable, &flush))
1379			return true;
1380	}
1381
1382	return false;
1383}
1384
1385static bool dm_table_discard_zeroes_data(struct dm_table *t)
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1386{
1387	struct dm_target *ti;
1388	unsigned i = 0;
1389
1390	/* Ensure that all targets supports discard_zeroes_data. */
1391	while (i < dm_table_get_num_targets(t)) {
1392		ti = dm_table_get_target(t, i++);
1393
1394		if (ti->discard_zeroes_data_unsupported)
1395			return false;
 
 
1396	}
1397
1398	return true;
1399}
1400
1401static int device_is_nonrot(struct dm_target *ti, struct dm_dev *dev,
1402			    sector_t start, sector_t len, void *data)
1403{
1404	struct request_queue *q = bdev_get_queue(dev->bdev);
1405
1406	return q && blk_queue_nonrot(q);
1407}
1408
1409static int device_is_not_random(struct dm_target *ti, struct dm_dev *dev,
1410			     sector_t start, sector_t len, void *data)
1411{
1412	struct request_queue *q = bdev_get_queue(dev->bdev);
1413
1414	return q && !blk_queue_add_random(q);
1415}
1416
1417static int queue_supports_sg_merge(struct dm_target *ti, struct dm_dev *dev,
1418				   sector_t start, sector_t len, void *data)
1419{
1420	struct request_queue *q = bdev_get_queue(dev->bdev);
1421
1422	return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags);
1423}
1424
1425static bool dm_table_all_devices_attribute(struct dm_table *t,
1426					   iterate_devices_callout_fn func)
1427{
1428	struct dm_target *ti;
1429	unsigned i = 0;
1430
1431	while (i < dm_table_get_num_targets(t)) {
1432		ti = dm_table_get_target(t, i++);
1433
1434		if (!ti->type->iterate_devices ||
1435		    !ti->type->iterate_devices(ti, func, NULL))
1436			return false;
1437	}
1438
1439	return true;
1440}
1441
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1442static int device_not_write_same_capable(struct dm_target *ti, struct dm_dev *dev,
1443					 sector_t start, sector_t len, void *data)
1444{
1445	struct request_queue *q = bdev_get_queue(dev->bdev);
1446
1447	return q && !q->limits.max_write_same_sectors;
1448}
1449
1450static bool dm_table_supports_write_same(struct dm_table *t)
1451{
1452	struct dm_target *ti;
1453	unsigned i = 0;
1454
1455	while (i < dm_table_get_num_targets(t)) {
1456		ti = dm_table_get_target(t, i++);
1457
1458		if (!ti->num_write_same_bios)
1459			return false;
1460
1461		if (!ti->type->iterate_devices ||
1462		    ti->type->iterate_devices(ti, device_not_write_same_capable, NULL))
1463			return false;
1464	}
1465
1466	return true;
1467}
1468
1469static int device_discard_capable(struct dm_target *ti, struct dm_dev *dev,
1470				  sector_t start, sector_t len, void *data)
1471{
1472	struct request_queue *q = bdev_get_queue(dev->bdev);
1473
1474	return q && blk_queue_discard(q);
1475}
1476
1477static bool dm_table_supports_discards(struct dm_table *t)
1478{
1479	struct dm_target *ti;
1480	unsigned i = 0;
1481
1482	/*
1483	 * Unless any target used by the table set discards_supported,
1484	 * require at least one underlying device to support discards.
1485	 * t->devices includes internal dm devices such as mirror logs
1486	 * so we need to use iterate_devices here, which targets
1487	 * supporting discard selectively must provide.
1488	 */
1489	while (i < dm_table_get_num_targets(t)) {
1490		ti = dm_table_get_target(t, i++);
1491
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1492		if (!ti->num_discard_bios)
1493			continue;
1494
1495		if (ti->discards_supported)
1496			return true;
 
 
 
 
 
 
 
 
1497
1498		if (ti->type->iterate_devices &&
1499		    ti->type->iterate_devices(ti, device_discard_capable, NULL))
1500			return true;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1501	}
1502
1503	return false;
1504}
1505
1506void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
1507			       struct queue_limits *limits)
1508{
1509	unsigned flush = 0;
1510
1511	/*
1512	 * Copy table's limits to the DM device's request_queue
1513	 */
1514	q->limits = *limits;
1515
1516	if (!dm_table_supports_discards(t))
1517		queue_flag_clear_unlocked(QUEUE_FLAG_DISCARD, q);
1518	else
1519		queue_flag_set_unlocked(QUEUE_FLAG_DISCARD, q);
 
 
 
 
 
 
1520
1521	if (dm_table_supports_flush(t, REQ_FLUSH)) {
1522		flush |= REQ_FLUSH;
1523		if (dm_table_supports_flush(t, REQ_FUA))
1524			flush |= REQ_FUA;
1525	}
1526	blk_queue_flush(q, flush);
1527
1528	if (!dm_table_discard_zeroes_data(t))
1529		q->limits.discard_zeroes_data = 0;
 
 
 
 
 
 
 
 
 
1530
1531	/* Ensure that all underlying devices are non-rotational. */
1532	if (dm_table_all_devices_attribute(t, device_is_nonrot))
1533		queue_flag_set_unlocked(QUEUE_FLAG_NONROT, q);
1534	else
1535		queue_flag_clear_unlocked(QUEUE_FLAG_NONROT, q);
1536
1537	if (!dm_table_supports_write_same(t))
1538		q->limits.max_write_same_sectors = 0;
 
 
1539
1540	if (dm_table_all_devices_attribute(t, queue_supports_sg_merge))
1541		queue_flag_clear_unlocked(QUEUE_FLAG_NO_SG_MERGE, q);
1542	else
1543		queue_flag_set_unlocked(QUEUE_FLAG_NO_SG_MERGE, q);
1544
1545	dm_table_verify_integrity(t);
1546
1547	/*
1548	 * Determine whether or not this queue's I/O timings contribute
1549	 * to the entropy pool, Only request-based targets use this.
1550	 * Clear QUEUE_FLAG_ADD_RANDOM if any underlying device does not
1551	 * have it set.
1552	 */
1553	if (blk_queue_add_random(q) && dm_table_all_devices_attribute(t, device_is_not_random))
1554		queue_flag_clear_unlocked(QUEUE_FLAG_ADD_RANDOM, q);
1555
1556	/*
1557	 * QUEUE_FLAG_STACKABLE must be set after all queue settings are
1558	 * visible to other CPUs because, once the flag is set, incoming bios
1559	 * are processed by request-based dm, which refers to the queue
1560	 * settings.
1561	 * Until the flag set, bios are passed to bio-based dm and queued to
1562	 * md->deferred where queue settings are not needed yet.
1563	 * Those bios are passed to request-based dm at the resume time.
1564	 */
1565	smp_mb();
1566	if (dm_table_request_based(t))
1567		queue_flag_set_unlocked(QUEUE_FLAG_STACKABLE, q);
1568}
1569
1570unsigned int dm_table_get_num_targets(struct dm_table *t)
1571{
1572	return t->num_targets;
1573}
1574
1575struct list_head *dm_table_get_devices(struct dm_table *t)
1576{
1577	return &t->devices;
1578}
1579
1580fmode_t dm_table_get_mode(struct dm_table *t)
1581{
1582	return t->mode;
1583}
1584EXPORT_SYMBOL(dm_table_get_mode);
1585
1586enum suspend_mode {
1587	PRESUSPEND,
1588	PRESUSPEND_UNDO,
1589	POSTSUSPEND,
1590};
1591
1592static void suspend_targets(struct dm_table *t, enum suspend_mode mode)
1593{
1594	int i = t->num_targets;
1595	struct dm_target *ti = t->targets;
1596
 
 
1597	while (i--) {
1598		switch (mode) {
1599		case PRESUSPEND:
1600			if (ti->type->presuspend)
1601				ti->type->presuspend(ti);
1602			break;
1603		case PRESUSPEND_UNDO:
1604			if (ti->type->presuspend_undo)
1605				ti->type->presuspend_undo(ti);
1606			break;
1607		case POSTSUSPEND:
1608			if (ti->type->postsuspend)
1609				ti->type->postsuspend(ti);
1610			break;
1611		}
1612		ti++;
1613	}
1614}
1615
1616void dm_table_presuspend_targets(struct dm_table *t)
1617{
1618	if (!t)
1619		return;
1620
1621	suspend_targets(t, PRESUSPEND);
1622}
1623
1624void dm_table_presuspend_undo_targets(struct dm_table *t)
1625{
1626	if (!t)
1627		return;
1628
1629	suspend_targets(t, PRESUSPEND_UNDO);
1630}
1631
1632void dm_table_postsuspend_targets(struct dm_table *t)
1633{
1634	if (!t)
1635		return;
1636
1637	suspend_targets(t, POSTSUSPEND);
1638}
1639
1640int dm_table_resume_targets(struct dm_table *t)
1641{
1642	int i, r = 0;
1643
 
 
1644	for (i = 0; i < t->num_targets; i++) {
1645		struct dm_target *ti = t->targets + i;
1646
1647		if (!ti->type->preresume)
1648			continue;
1649
1650		r = ti->type->preresume(ti);
1651		if (r) {
1652			DMERR("%s: %s: preresume failed, error = %d",
1653			      dm_device_name(t->md), ti->type->name, r);
1654			return r;
1655		}
1656	}
1657
1658	for (i = 0; i < t->num_targets; i++) {
1659		struct dm_target *ti = t->targets + i;
1660
1661		if (ti->type->resume)
1662			ti->type->resume(ti);
1663	}
1664
1665	return 0;
1666}
1667
1668void dm_table_add_target_callbacks(struct dm_table *t, struct dm_target_callbacks *cb)
1669{
1670	list_add(&cb->list, &t->target_callbacks);
1671}
1672EXPORT_SYMBOL_GPL(dm_table_add_target_callbacks);
1673
1674int dm_table_any_congested(struct dm_table *t, int bdi_bits)
1675{
1676	struct dm_dev_internal *dd;
1677	struct list_head *devices = dm_table_get_devices(t);
1678	struct dm_target_callbacks *cb;
1679	int r = 0;
1680
1681	list_for_each_entry(dd, devices, list) {
1682		struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev);
1683		char b[BDEVNAME_SIZE];
1684
1685		if (likely(q))
1686			r |= bdi_congested(&q->backing_dev_info, bdi_bits);
1687		else
1688			DMWARN_LIMIT("%s: any_congested: nonexistent device %s",
1689				     dm_device_name(t->md),
1690				     bdevname(dd->dm_dev->bdev, b));
1691	}
1692
1693	list_for_each_entry(cb, &t->target_callbacks, list)
1694		if (cb->congested_fn)
1695			r |= cb->congested_fn(cb, bdi_bits);
1696
1697	return r;
1698}
1699
1700struct mapped_device *dm_table_get_md(struct dm_table *t)
1701{
1702	return t->md;
1703}
1704EXPORT_SYMBOL(dm_table_get_md);
1705
1706void dm_table_run_md_queue_async(struct dm_table *t)
1707{
1708	struct mapped_device *md;
1709	struct request_queue *queue;
1710	unsigned long flags;
1711
1712	if (!dm_table_request_based(t))
1713		return;
1714
1715	md = dm_table_get_md(t);
1716	queue = dm_get_md_queue(md);
1717	if (queue) {
1718		if (queue->mq_ops)
1719			blk_mq_run_hw_queues(queue, true);
1720		else {
1721			spin_lock_irqsave(queue->queue_lock, flags);
1722			blk_run_queue_async(queue);
1723			spin_unlock_irqrestore(queue->queue_lock, flags);
1724		}
1725	}
1726}
1727EXPORT_SYMBOL(dm_table_run_md_queue_async);
1728
v4.17
   1/*
   2 * Copyright (C) 2001 Sistina Software (UK) Limited.
   3 * Copyright (C) 2004-2008 Red Hat, Inc. All rights reserved.
   4 *
   5 * This file is released under the GPL.
   6 */
   7
   8#include "dm-core.h"
   9
  10#include <linux/module.h>
  11#include <linux/vmalloc.h>
  12#include <linux/blkdev.h>
  13#include <linux/namei.h>
  14#include <linux/ctype.h>
  15#include <linux/string.h>
  16#include <linux/slab.h>
  17#include <linux/interrupt.h>
  18#include <linux/mutex.h>
  19#include <linux/delay.h>
  20#include <linux/atomic.h>
  21#include <linux/blk-mq.h>
  22#include <linux/mount.h>
  23#include <linux/dax.h>
  24
  25#define DM_MSG_PREFIX "table"
  26
  27#define MAX_DEPTH 16
  28#define NODE_SIZE L1_CACHE_BYTES
  29#define KEYS_PER_NODE (NODE_SIZE / sizeof(sector_t))
  30#define CHILDREN_PER_NODE (KEYS_PER_NODE + 1)
  31
  32struct dm_table {
  33	struct mapped_device *md;
  34	enum dm_queue_mode type;
  35
  36	/* btree table */
  37	unsigned int depth;
  38	unsigned int counts[MAX_DEPTH];	/* in nodes */
  39	sector_t *index[MAX_DEPTH];
  40
  41	unsigned int num_targets;
  42	unsigned int num_allocated;
  43	sector_t *highs;
  44	struct dm_target *targets;
  45
  46	struct target_type *immutable_target_type;
  47
  48	bool integrity_supported:1;
  49	bool singleton:1;
  50	bool all_blk_mq:1;
  51	unsigned integrity_added:1;
  52
  53	/*
  54	 * Indicates the rw permissions for the new logical
  55	 * device.  This should be a combination of FMODE_READ
  56	 * and FMODE_WRITE.
  57	 */
  58	fmode_t mode;
  59
  60	/* a list of devices used by this table */
  61	struct list_head devices;
  62
  63	/* events get handed up using this callback */
  64	void (*event_fn)(void *);
  65	void *event_context;
  66
  67	struct dm_md_mempools *mempools;
  68
  69	struct list_head target_callbacks;
  70};
  71
  72/*
  73 * Similar to ceiling(log_size(n))
  74 */
  75static unsigned int int_log(unsigned int n, unsigned int base)
  76{
  77	int result = 0;
  78
  79	while (n > 1) {
  80		n = dm_div_up(n, base);
  81		result++;
  82	}
  83
  84	return result;
  85}
  86
  87/*
  88 * Calculate the index of the child node of the n'th node k'th key.
  89 */
  90static inline unsigned int get_child(unsigned int n, unsigned int k)
  91{
  92	return (n * CHILDREN_PER_NODE) + k;
  93}
  94
  95/*
  96 * Return the n'th node of level l from table t.
  97 */
  98static inline sector_t *get_node(struct dm_table *t,
  99				 unsigned int l, unsigned int n)
 100{
 101	return t->index[l] + (n * KEYS_PER_NODE);
 102}
 103
 104/*
 105 * Return the highest key that you could lookup from the n'th
 106 * node on level l of the btree.
 107 */
 108static sector_t high(struct dm_table *t, unsigned int l, unsigned int n)
 109{
 110	for (; l < t->depth - 1; l++)
 111		n = get_child(n, CHILDREN_PER_NODE - 1);
 112
 113	if (n >= t->counts[l])
 114		return (sector_t) - 1;
 115
 116	return get_node(t, l, n)[KEYS_PER_NODE - 1];
 117}
 118
 119/*
 120 * Fills in a level of the btree based on the highs of the level
 121 * below it.
 122 */
 123static int setup_btree_index(unsigned int l, struct dm_table *t)
 124{
 125	unsigned int n, k;
 126	sector_t *node;
 127
 128	for (n = 0U; n < t->counts[l]; n++) {
 129		node = get_node(t, l, n);
 130
 131		for (k = 0U; k < KEYS_PER_NODE; k++)
 132			node[k] = high(t, l + 1, get_child(n, k));
 133	}
 134
 135	return 0;
 136}
 137
 138void *dm_vcalloc(unsigned long nmemb, unsigned long elem_size)
 139{
 140	unsigned long size;
 141	void *addr;
 142
 143	/*
 144	 * Check that we're not going to overflow.
 145	 */
 146	if (nmemb > (ULONG_MAX / elem_size))
 147		return NULL;
 148
 149	size = nmemb * elem_size;
 150	addr = vzalloc(size);
 151
 152	return addr;
 153}
 154EXPORT_SYMBOL(dm_vcalloc);
 155
 156/*
 157 * highs, and targets are managed as dynamic arrays during a
 158 * table load.
 159 */
 160static int alloc_targets(struct dm_table *t, unsigned int num)
 161{
 162	sector_t *n_highs;
 163	struct dm_target *n_targets;
 164
 165	/*
 166	 * Allocate both the target array and offset array at once.
 167	 * Append an empty entry to catch sectors beyond the end of
 168	 * the device.
 169	 */
 170	n_highs = (sector_t *) dm_vcalloc(num + 1, sizeof(struct dm_target) +
 171					  sizeof(sector_t));
 172	if (!n_highs)
 173		return -ENOMEM;
 174
 175	n_targets = (struct dm_target *) (n_highs + num);
 176
 177	memset(n_highs, -1, sizeof(*n_highs) * num);
 178	vfree(t->highs);
 179
 180	t->num_allocated = num;
 181	t->highs = n_highs;
 182	t->targets = n_targets;
 183
 184	return 0;
 185}
 186
 187int dm_table_create(struct dm_table **result, fmode_t mode,
 188		    unsigned num_targets, struct mapped_device *md)
 189{
 190	struct dm_table *t = kzalloc(sizeof(*t), GFP_KERNEL);
 191
 192	if (!t)
 193		return -ENOMEM;
 194
 195	INIT_LIST_HEAD(&t->devices);
 196	INIT_LIST_HEAD(&t->target_callbacks);
 197
 198	if (!num_targets)
 199		num_targets = KEYS_PER_NODE;
 200
 201	num_targets = dm_round_up(num_targets, KEYS_PER_NODE);
 202
 203	if (!num_targets) {
 204		kfree(t);
 205		return -ENOMEM;
 206	}
 207
 208	if (alloc_targets(t, num_targets)) {
 209		kfree(t);
 210		return -ENOMEM;
 211	}
 212
 213	t->type = DM_TYPE_NONE;
 214	t->mode = mode;
 215	t->md = md;
 216	*result = t;
 217	return 0;
 218}
 219
 220static void free_devices(struct list_head *devices, struct mapped_device *md)
 221{
 222	struct list_head *tmp, *next;
 223
 224	list_for_each_safe(tmp, next, devices) {
 225		struct dm_dev_internal *dd =
 226		    list_entry(tmp, struct dm_dev_internal, list);
 227		DMWARN("%s: dm_table_destroy: dm_put_device call missing for %s",
 228		       dm_device_name(md), dd->dm_dev->name);
 229		dm_put_table_device(md, dd->dm_dev);
 230		kfree(dd);
 231	}
 232}
 233
 234void dm_table_destroy(struct dm_table *t)
 235{
 236	unsigned int i;
 237
 238	if (!t)
 239		return;
 240
 241	/* free the indexes */
 242	if (t->depth >= 2)
 243		vfree(t->index[t->depth - 2]);
 244
 245	/* free the targets */
 246	for (i = 0; i < t->num_targets; i++) {
 247		struct dm_target *tgt = t->targets + i;
 248
 249		if (tgt->type->dtr)
 250			tgt->type->dtr(tgt);
 251
 252		dm_put_target_type(tgt->type);
 253	}
 254
 255	vfree(t->highs);
 256
 257	/* free the device list */
 258	free_devices(&t->devices, t->md);
 259
 260	dm_free_md_mempools(t->mempools);
 261
 262	kfree(t);
 263}
 264
 265/*
 266 * See if we've already got a device in the list.
 267 */
 268static struct dm_dev_internal *find_device(struct list_head *l, dev_t dev)
 269{
 270	struct dm_dev_internal *dd;
 271
 272	list_for_each_entry (dd, l, list)
 273		if (dd->dm_dev->bdev->bd_dev == dev)
 274			return dd;
 275
 276	return NULL;
 277}
 278
 279/*
 280 * If possible, this checks an area of a destination device is invalid.
 281 */
 282static int device_area_is_invalid(struct dm_target *ti, struct dm_dev *dev,
 283				  sector_t start, sector_t len, void *data)
 284{
 285	struct request_queue *q;
 286	struct queue_limits *limits = data;
 287	struct block_device *bdev = dev->bdev;
 288	sector_t dev_size =
 289		i_size_read(bdev->bd_inode) >> SECTOR_SHIFT;
 290	unsigned short logical_block_size_sectors =
 291		limits->logical_block_size >> SECTOR_SHIFT;
 292	char b[BDEVNAME_SIZE];
 293
 294	/*
 295	 * Some devices exist without request functions,
 296	 * such as loop devices not yet bound to backing files.
 297	 * Forbid the use of such devices.
 298	 */
 299	q = bdev_get_queue(bdev);
 300	if (!q || !q->make_request_fn) {
 301		DMWARN("%s: %s is not yet initialised: "
 302		       "start=%llu, len=%llu, dev_size=%llu",
 303		       dm_device_name(ti->table->md), bdevname(bdev, b),
 304		       (unsigned long long)start,
 305		       (unsigned long long)len,
 306		       (unsigned long long)dev_size);
 307		return 1;
 308	}
 309
 310	if (!dev_size)
 311		return 0;
 312
 313	if ((start >= dev_size) || (start + len > dev_size)) {
 314		DMWARN("%s: %s too small for target: "
 315		       "start=%llu, len=%llu, dev_size=%llu",
 316		       dm_device_name(ti->table->md), bdevname(bdev, b),
 317		       (unsigned long long)start,
 318		       (unsigned long long)len,
 319		       (unsigned long long)dev_size);
 320		return 1;
 321	}
 322
 323	/*
 324	 * If the target is mapped to zoned block device(s), check
 325	 * that the zones are not partially mapped.
 326	 */
 327	if (bdev_zoned_model(bdev) != BLK_ZONED_NONE) {
 328		unsigned int zone_sectors = bdev_zone_sectors(bdev);
 329
 330		if (start & (zone_sectors - 1)) {
 331			DMWARN("%s: start=%llu not aligned to h/w zone size %u of %s",
 332			       dm_device_name(ti->table->md),
 333			       (unsigned long long)start,
 334			       zone_sectors, bdevname(bdev, b));
 335			return 1;
 336		}
 337
 338		/*
 339		 * Note: The last zone of a zoned block device may be smaller
 340		 * than other zones. So for a target mapping the end of a
 341		 * zoned block device with such a zone, len would not be zone
 342		 * aligned. We do not allow such last smaller zone to be part
 343		 * of the mapping here to ensure that mappings with multiple
 344		 * devices do not end up with a smaller zone in the middle of
 345		 * the sector range.
 346		 */
 347		if (len & (zone_sectors - 1)) {
 348			DMWARN("%s: len=%llu not aligned to h/w zone size %u of %s",
 349			       dm_device_name(ti->table->md),
 350			       (unsigned long long)len,
 351			       zone_sectors, bdevname(bdev, b));
 352			return 1;
 353		}
 354	}
 355
 356	if (logical_block_size_sectors <= 1)
 357		return 0;
 358
 359	if (start & (logical_block_size_sectors - 1)) {
 360		DMWARN("%s: start=%llu not aligned to h/w "
 361		       "logical block size %u of %s",
 362		       dm_device_name(ti->table->md),
 363		       (unsigned long long)start,
 364		       limits->logical_block_size, bdevname(bdev, b));
 365		return 1;
 366	}
 367
 368	if (len & (logical_block_size_sectors - 1)) {
 369		DMWARN("%s: len=%llu not aligned to h/w "
 370		       "logical block size %u of %s",
 371		       dm_device_name(ti->table->md),
 372		       (unsigned long long)len,
 373		       limits->logical_block_size, bdevname(bdev, b));
 374		return 1;
 375	}
 376
 377	return 0;
 378}
 379
 380/*
 381 * This upgrades the mode on an already open dm_dev, being
 382 * careful to leave things as they were if we fail to reopen the
 383 * device and not to touch the existing bdev field in case
 384 * it is accessed concurrently inside dm_table_any_congested().
 385 */
 386static int upgrade_mode(struct dm_dev_internal *dd, fmode_t new_mode,
 387			struct mapped_device *md)
 388{
 389	int r;
 390	struct dm_dev *old_dev, *new_dev;
 391
 392	old_dev = dd->dm_dev;
 393
 394	r = dm_get_table_device(md, dd->dm_dev->bdev->bd_dev,
 395				dd->dm_dev->mode | new_mode, &new_dev);
 396	if (r)
 397		return r;
 398
 399	dd->dm_dev = new_dev;
 400	dm_put_table_device(md, old_dev);
 401
 402	return 0;
 403}
 404
 405/*
 406 * Convert the path to a device
 407 */
 408dev_t dm_get_dev_t(const char *path)
 409{
 410	dev_t dev;
 411	struct block_device *bdev;
 412
 413	bdev = lookup_bdev(path);
 414	if (IS_ERR(bdev))
 415		dev = name_to_dev_t(path);
 416	else {
 417		dev = bdev->bd_dev;
 418		bdput(bdev);
 419	}
 420
 421	return dev;
 422}
 423EXPORT_SYMBOL_GPL(dm_get_dev_t);
 424
 425/*
 426 * Add a device to the list, or just increment the usage count if
 427 * it's already present.
 428 */
 429int dm_get_device(struct dm_target *ti, const char *path, fmode_t mode,
 430		  struct dm_dev **result)
 431{
 432	int r;
 433	dev_t dev;
 434	struct dm_dev_internal *dd;
 435	struct dm_table *t = ti->table;
 436
 437	BUG_ON(!t);
 438
 439	dev = dm_get_dev_t(path);
 440	if (!dev)
 441		return -ENODEV;
 442
 443	dd = find_device(&t->devices, dev);
 444	if (!dd) {
 445		dd = kmalloc(sizeof(*dd), GFP_KERNEL);
 446		if (!dd)
 447			return -ENOMEM;
 448
 449		if ((r = dm_get_table_device(t->md, dev, mode, &dd->dm_dev))) {
 450			kfree(dd);
 451			return r;
 452		}
 453
 454		refcount_set(&dd->count, 1);
 455		list_add(&dd->list, &t->devices);
 456		goto out;
 457
 458	} else if (dd->dm_dev->mode != (mode | dd->dm_dev->mode)) {
 459		r = upgrade_mode(dd, mode, t->md);
 460		if (r)
 461			return r;
 462	}
 463	refcount_inc(&dd->count);
 464out:
 465	*result = dd->dm_dev;
 466	return 0;
 467}
 468EXPORT_SYMBOL(dm_get_device);
 469
 470static int dm_set_device_limits(struct dm_target *ti, struct dm_dev *dev,
 471				sector_t start, sector_t len, void *data)
 472{
 473	struct queue_limits *limits = data;
 474	struct block_device *bdev = dev->bdev;
 475	struct request_queue *q = bdev_get_queue(bdev);
 476	char b[BDEVNAME_SIZE];
 477
 478	if (unlikely(!q)) {
 479		DMWARN("%s: Cannot set limits for nonexistent device %s",
 480		       dm_device_name(ti->table->md), bdevname(bdev, b));
 481		return 0;
 482	}
 483
 484	if (bdev_stack_limits(limits, bdev, start) < 0)
 485		DMWARN("%s: adding target device %s caused an alignment inconsistency: "
 486		       "physical_block_size=%u, logical_block_size=%u, "
 487		       "alignment_offset=%u, start=%llu",
 488		       dm_device_name(ti->table->md), bdevname(bdev, b),
 489		       q->limits.physical_block_size,
 490		       q->limits.logical_block_size,
 491		       q->limits.alignment_offset,
 492		       (unsigned long long) start << SECTOR_SHIFT);
 493
 494	limits->zoned = blk_queue_zoned_model(q);
 495
 496	return 0;
 497}
 498
 499/*
 500 * Decrement a device's use count and remove it if necessary.
 501 */
 502void dm_put_device(struct dm_target *ti, struct dm_dev *d)
 503{
 504	int found = 0;
 505	struct list_head *devices = &ti->table->devices;
 506	struct dm_dev_internal *dd;
 507
 508	list_for_each_entry(dd, devices, list) {
 509		if (dd->dm_dev == d) {
 510			found = 1;
 511			break;
 512		}
 513	}
 514	if (!found) {
 515		DMWARN("%s: device %s not in table devices list",
 516		       dm_device_name(ti->table->md), d->name);
 517		return;
 518	}
 519	if (refcount_dec_and_test(&dd->count)) {
 520		dm_put_table_device(ti->table->md, d);
 521		list_del(&dd->list);
 522		kfree(dd);
 523	}
 524}
 525EXPORT_SYMBOL(dm_put_device);
 526
 527/*
 528 * Checks to see if the target joins onto the end of the table.
 529 */
 530static int adjoin(struct dm_table *table, struct dm_target *ti)
 531{
 532	struct dm_target *prev;
 533
 534	if (!table->num_targets)
 535		return !ti->begin;
 536
 537	prev = &table->targets[table->num_targets - 1];
 538	return (ti->begin == (prev->begin + prev->len));
 539}
 540
 541/*
 542 * Used to dynamically allocate the arg array.
 543 *
 544 * We do first allocation with GFP_NOIO because dm-mpath and dm-thin must
 545 * process messages even if some device is suspended. These messages have a
 546 * small fixed number of arguments.
 547 *
 548 * On the other hand, dm-switch needs to process bulk data using messages and
 549 * excessive use of GFP_NOIO could cause trouble.
 550 */
 551static char **realloc_argv(unsigned *array_size, char **old_argv)
 552{
 553	char **argv;
 554	unsigned new_size;
 555	gfp_t gfp;
 556
 557	if (*array_size) {
 558		new_size = *array_size * 2;
 559		gfp = GFP_KERNEL;
 560	} else {
 561		new_size = 8;
 562		gfp = GFP_NOIO;
 563	}
 564	argv = kmalloc(new_size * sizeof(*argv), gfp);
 565	if (argv) {
 566		memcpy(argv, old_argv, *array_size * sizeof(*argv));
 567		*array_size = new_size;
 568	}
 569
 570	kfree(old_argv);
 571	return argv;
 572}
 573
 574/*
 575 * Destructively splits up the argument list to pass to ctr.
 576 */
 577int dm_split_args(int *argc, char ***argvp, char *input)
 578{
 579	char *start, *end = input, *out, **argv = NULL;
 580	unsigned array_size = 0;
 581
 582	*argc = 0;
 583
 584	if (!input) {
 585		*argvp = NULL;
 586		return 0;
 587	}
 588
 589	argv = realloc_argv(&array_size, argv);
 590	if (!argv)
 591		return -ENOMEM;
 592
 593	while (1) {
 594		/* Skip whitespace */
 595		start = skip_spaces(end);
 596
 597		if (!*start)
 598			break;	/* success, we hit the end */
 599
 600		/* 'out' is used to remove any back-quotes */
 601		end = out = start;
 602		while (*end) {
 603			/* Everything apart from '\0' can be quoted */
 604			if (*end == '\\' && *(end + 1)) {
 605				*out++ = *(end + 1);
 606				end += 2;
 607				continue;
 608			}
 609
 610			if (isspace(*end))
 611				break;	/* end of token */
 612
 613			*out++ = *end++;
 614		}
 615
 616		/* have we already filled the array ? */
 617		if ((*argc + 1) > array_size) {
 618			argv = realloc_argv(&array_size, argv);
 619			if (!argv)
 620				return -ENOMEM;
 621		}
 622
 623		/* we know this is whitespace */
 624		if (*end)
 625			end++;
 626
 627		/* terminate the string and put it in the array */
 628		*out = '\0';
 629		argv[*argc] = start;
 630		(*argc)++;
 631	}
 632
 633	*argvp = argv;
 634	return 0;
 635}
 636
 637/*
 638 * Impose necessary and sufficient conditions on a devices's table such
 639 * that any incoming bio which respects its logical_block_size can be
 640 * processed successfully.  If it falls across the boundary between
 641 * two or more targets, the size of each piece it gets split into must
 642 * be compatible with the logical_block_size of the target processing it.
 643 */
 644static int validate_hardware_logical_block_alignment(struct dm_table *table,
 645						 struct queue_limits *limits)
 646{
 647	/*
 648	 * This function uses arithmetic modulo the logical_block_size
 649	 * (in units of 512-byte sectors).
 650	 */
 651	unsigned short device_logical_block_size_sects =
 652		limits->logical_block_size >> SECTOR_SHIFT;
 653
 654	/*
 655	 * Offset of the start of the next table entry, mod logical_block_size.
 656	 */
 657	unsigned short next_target_start = 0;
 658
 659	/*
 660	 * Given an aligned bio that extends beyond the end of a
 661	 * target, how many sectors must the next target handle?
 662	 */
 663	unsigned short remaining = 0;
 664
 665	struct dm_target *uninitialized_var(ti);
 666	struct queue_limits ti_limits;
 667	unsigned i;
 668
 669	/*
 670	 * Check each entry in the table in turn.
 671	 */
 672	for (i = 0; i < dm_table_get_num_targets(table); i++) {
 673		ti = dm_table_get_target(table, i);
 674
 675		blk_set_stacking_limits(&ti_limits);
 676
 677		/* combine all target devices' limits */
 678		if (ti->type->iterate_devices)
 679			ti->type->iterate_devices(ti, dm_set_device_limits,
 680						  &ti_limits);
 681
 682		/*
 683		 * If the remaining sectors fall entirely within this
 684		 * table entry are they compatible with its logical_block_size?
 685		 */
 686		if (remaining < ti->len &&
 687		    remaining & ((ti_limits.logical_block_size >>
 688				  SECTOR_SHIFT) - 1))
 689			break;	/* Error */
 690
 691		next_target_start =
 692		    (unsigned short) ((next_target_start + ti->len) &
 693				      (device_logical_block_size_sects - 1));
 694		remaining = next_target_start ?
 695		    device_logical_block_size_sects - next_target_start : 0;
 696	}
 697
 698	if (remaining) {
 699		DMWARN("%s: table line %u (start sect %llu len %llu) "
 700		       "not aligned to h/w logical block size %u",
 701		       dm_device_name(table->md), i,
 702		       (unsigned long long) ti->begin,
 703		       (unsigned long long) ti->len,
 704		       limits->logical_block_size);
 705		return -EINVAL;
 706	}
 707
 708	return 0;
 709}
 710
 711int dm_table_add_target(struct dm_table *t, const char *type,
 712			sector_t start, sector_t len, char *params)
 713{
 714	int r = -EINVAL, argc;
 715	char **argv;
 716	struct dm_target *tgt;
 717
 718	if (t->singleton) {
 719		DMERR("%s: target type %s must appear alone in table",
 720		      dm_device_name(t->md), t->targets->type->name);
 721		return -EINVAL;
 722	}
 723
 724	BUG_ON(t->num_targets >= t->num_allocated);
 725
 726	tgt = t->targets + t->num_targets;
 727	memset(tgt, 0, sizeof(*tgt));
 728
 729	if (!len) {
 730		DMERR("%s: zero-length target", dm_device_name(t->md));
 731		return -EINVAL;
 732	}
 733
 734	tgt->type = dm_get_target_type(type);
 735	if (!tgt->type) {
 736		DMERR("%s: %s: unknown target type", dm_device_name(t->md), type);
 
 737		return -EINVAL;
 738	}
 739
 740	if (dm_target_needs_singleton(tgt->type)) {
 741		if (t->num_targets) {
 742			tgt->error = "singleton target type must appear alone in table";
 743			goto bad;
 
 744		}
 745		t->singleton = true;
 746	}
 747
 748	if (dm_target_always_writeable(tgt->type) && !(t->mode & FMODE_WRITE)) {
 749		tgt->error = "target type may not be included in a read-only table";
 750		goto bad;
 
 751	}
 752
 753	if (t->immutable_target_type) {
 754		if (t->immutable_target_type != tgt->type) {
 755			tgt->error = "immutable target type cannot be mixed with other target types";
 756			goto bad;
 
 757		}
 758	} else if (dm_target_is_immutable(tgt->type)) {
 759		if (t->num_targets) {
 760			tgt->error = "immutable target type cannot be mixed with other target types";
 761			goto bad;
 
 762		}
 763		t->immutable_target_type = tgt->type;
 764	}
 765
 766	if (dm_target_has_integrity(tgt->type))
 767		t->integrity_added = 1;
 768
 769	tgt->table = t;
 770	tgt->begin = start;
 771	tgt->len = len;
 772	tgt->error = "Unknown error";
 773
 774	/*
 775	 * Does this target adjoin the previous one ?
 776	 */
 777	if (!adjoin(t, tgt)) {
 778		tgt->error = "Gap in table";
 
 779		goto bad;
 780	}
 781
 782	r = dm_split_args(&argc, &argv, params);
 783	if (r) {
 784		tgt->error = "couldn't split parameters (insufficient memory)";
 785		goto bad;
 786	}
 787
 788	r = tgt->type->ctr(tgt, argc, argv);
 789	kfree(argv);
 790	if (r)
 791		goto bad;
 792
 793	t->highs[t->num_targets++] = tgt->begin + tgt->len - 1;
 794
 795	if (!tgt->num_discard_bios && tgt->discards_supported)
 796		DMWARN("%s: %s: ignoring discards_supported because num_discard_bios is zero.",
 797		       dm_device_name(t->md), type);
 798
 799	return 0;
 800
 801 bad:
 802	DMERR("%s: %s: %s", dm_device_name(t->md), type, tgt->error);
 803	dm_put_target_type(tgt->type);
 804	return r;
 805}
 806
 807/*
 808 * Target argument parsing helpers.
 809 */
 810static int validate_next_arg(const struct dm_arg *arg,
 811			     struct dm_arg_set *arg_set,
 812			     unsigned *value, char **error, unsigned grouped)
 813{
 814	const char *arg_str = dm_shift_arg(arg_set);
 815	char dummy;
 816
 817	if (!arg_str ||
 818	    (sscanf(arg_str, "%u%c", value, &dummy) != 1) ||
 819	    (*value < arg->min) ||
 820	    (*value > arg->max) ||
 821	    (grouped && arg_set->argc < *value)) {
 822		*error = arg->error;
 823		return -EINVAL;
 824	}
 825
 826	return 0;
 827}
 828
 829int dm_read_arg(const struct dm_arg *arg, struct dm_arg_set *arg_set,
 830		unsigned *value, char **error)
 831{
 832	return validate_next_arg(arg, arg_set, value, error, 0);
 833}
 834EXPORT_SYMBOL(dm_read_arg);
 835
 836int dm_read_arg_group(const struct dm_arg *arg, struct dm_arg_set *arg_set,
 837		      unsigned *value, char **error)
 838{
 839	return validate_next_arg(arg, arg_set, value, error, 1);
 840}
 841EXPORT_SYMBOL(dm_read_arg_group);
 842
 843const char *dm_shift_arg(struct dm_arg_set *as)
 844{
 845	char *r;
 846
 847	if (as->argc) {
 848		as->argc--;
 849		r = *as->argv;
 850		as->argv++;
 851		return r;
 852	}
 853
 854	return NULL;
 855}
 856EXPORT_SYMBOL(dm_shift_arg);
 857
 858void dm_consume_args(struct dm_arg_set *as, unsigned num_args)
 859{
 860	BUG_ON(as->argc < num_args);
 861	as->argc -= num_args;
 862	as->argv += num_args;
 863}
 864EXPORT_SYMBOL(dm_consume_args);
 865
 866static bool __table_type_bio_based(enum dm_queue_mode table_type)
 867{
 868	return (table_type == DM_TYPE_BIO_BASED ||
 869		table_type == DM_TYPE_DAX_BIO_BASED ||
 870		table_type == DM_TYPE_NVME_BIO_BASED);
 871}
 872
 873static bool __table_type_request_based(enum dm_queue_mode table_type)
 874{
 875	return (table_type == DM_TYPE_REQUEST_BASED ||
 876		table_type == DM_TYPE_MQ_REQUEST_BASED);
 877}
 878
 879void dm_table_set_type(struct dm_table *t, enum dm_queue_mode type)
 880{
 881	t->type = type;
 882}
 883EXPORT_SYMBOL_GPL(dm_table_set_type);
 884
 885static int device_supports_dax(struct dm_target *ti, struct dm_dev *dev,
 886			       sector_t start, sector_t len, void *data)
 887{
 888	struct request_queue *q = bdev_get_queue(dev->bdev);
 889
 890	return q && blk_queue_dax(q);
 891}
 892
 893static bool dm_table_supports_dax(struct dm_table *t)
 894{
 895	struct dm_target *ti;
 896	unsigned i;
 897
 898	/* Ensure that all targets support DAX. */
 899	for (i = 0; i < dm_table_get_num_targets(t); i++) {
 900		ti = dm_table_get_target(t, i);
 901
 902		if (!ti->type->direct_access)
 903			return false;
 904
 905		if (!ti->type->iterate_devices ||
 906		    !ti->type->iterate_devices(ti, device_supports_dax, NULL))
 907			return false;
 908	}
 909
 910	return true;
 911}
 912
 913static bool dm_table_does_not_support_partial_completion(struct dm_table *t);
 914
 915struct verify_rq_based_data {
 916	unsigned sq_count;
 917	unsigned mq_count;
 918};
 919
 920static int device_is_rq_based(struct dm_target *ti, struct dm_dev *dev,
 921			      sector_t start, sector_t len, void *data)
 922{
 923	struct request_queue *q = bdev_get_queue(dev->bdev);
 924	struct verify_rq_based_data *v = data;
 925
 926	if (q->mq_ops)
 927		v->mq_count++;
 928	else
 929		v->sq_count++;
 930
 931	return queue_is_rq_based(q);
 932}
 933
 934static int dm_table_determine_type(struct dm_table *t)
 935{
 936	unsigned i;
 937	unsigned bio_based = 0, request_based = 0, hybrid = 0;
 938	struct verify_rq_based_data v = {.sq_count = 0, .mq_count = 0};
 939	struct dm_target *tgt;
 940	struct list_head *devices = dm_table_get_devices(t);
 941	enum dm_queue_mode live_md_type = dm_get_md_type(t->md);
 942
 943	if (t->type != DM_TYPE_NONE) {
 944		/* target already set the table's type */
 945		if (t->type == DM_TYPE_BIO_BASED) {
 946			/* possibly upgrade to a variant of bio-based */
 947			goto verify_bio_based;
 948		}
 949		BUG_ON(t->type == DM_TYPE_DAX_BIO_BASED);
 950		BUG_ON(t->type == DM_TYPE_NVME_BIO_BASED);
 951		goto verify_rq_based;
 952	}
 953
 954	for (i = 0; i < t->num_targets; i++) {
 955		tgt = t->targets + i;
 956		if (dm_target_hybrid(tgt))
 957			hybrid = 1;
 958		else if (dm_target_request_based(tgt))
 959			request_based = 1;
 960		else
 961			bio_based = 1;
 962
 963		if (bio_based && request_based) {
 964			DMERR("Inconsistent table: different target types"
 965			      " can't be mixed up");
 966			return -EINVAL;
 967		}
 968	}
 969
 970	if (hybrid && !bio_based && !request_based) {
 971		/*
 972		 * The targets can work either way.
 973		 * Determine the type from the live device.
 974		 * Default to bio-based if device is new.
 975		 */
 976		if (__table_type_request_based(live_md_type))
 977			request_based = 1;
 978		else
 979			bio_based = 1;
 980	}
 981
 982	if (bio_based) {
 983verify_bio_based:
 984		/* We must use this table as bio-based */
 985		t->type = DM_TYPE_BIO_BASED;
 986		if (dm_table_supports_dax(t) ||
 987		    (list_empty(devices) && live_md_type == DM_TYPE_DAX_BIO_BASED)) {
 988			t->type = DM_TYPE_DAX_BIO_BASED;
 989		} else {
 990			/* Check if upgrading to NVMe bio-based is valid or required */
 991			tgt = dm_table_get_immutable_target(t);
 992			if (tgt && !tgt->max_io_len && dm_table_does_not_support_partial_completion(t)) {
 993				t->type = DM_TYPE_NVME_BIO_BASED;
 994				goto verify_rq_based; /* must be stacked directly on NVMe (blk-mq) */
 995			} else if (list_empty(devices) && live_md_type == DM_TYPE_NVME_BIO_BASED) {
 996				t->type = DM_TYPE_NVME_BIO_BASED;
 997			}
 998		}
 999		return 0;
1000	}
1001
1002	BUG_ON(!request_based); /* No targets in this table */
1003
1004	/*
1005	 * The only way to establish DM_TYPE_MQ_REQUEST_BASED is by
1006	 * having a compatible target use dm_table_set_type.
1007	 */
1008	t->type = DM_TYPE_REQUEST_BASED;
1009
1010verify_rq_based:
1011	/*
1012	 * Request-based dm supports only tables that have a single target now.
1013	 * To support multiple targets, request splitting support is needed,
1014	 * and that needs lots of changes in the block-layer.
1015	 * (e.g. request completion process for partial completion.)
1016	 */
1017	if (t->num_targets > 1) {
1018		DMERR("%s DM doesn't support multiple targets",
1019		      t->type == DM_TYPE_NVME_BIO_BASED ? "nvme bio-based" : "request-based");
1020		return -EINVAL;
1021	}
1022
1023	if (list_empty(devices)) {
1024		int srcu_idx;
1025		struct dm_table *live_table = dm_get_live_table(t->md, &srcu_idx);
1026
1027		/* inherit live table's type and all_blk_mq */
1028		if (live_table) {
1029			t->type = live_table->type;
1030			t->all_blk_mq = live_table->all_blk_mq;
 
1031		}
1032		dm_put_live_table(t->md, srcu_idx);
1033		return 0;
 
1034	}
1035
1036	tgt = dm_table_get_immutable_target(t);
1037	if (!tgt) {
1038		DMERR("table load rejected: immutable target is required");
1039		return -EINVAL;
1040	} else if (tgt->max_io_len) {
1041		DMERR("table load rejected: immutable target that splits IO is not supported");
1042		return -EINVAL;
1043	}
 
1044
1045	/* Non-request-stackable devices can't be used for request-based dm */
1046	if (!tgt->type->iterate_devices ||
1047	    !tgt->type->iterate_devices(tgt, device_is_rq_based, &v)) {
1048		DMERR("table load rejected: including non-request-stackable devices");
1049		return -EINVAL;
1050	}
1051	if (v.sq_count && v.mq_count) {
1052		DMERR("table load rejected: not all devices are blk-mq request-stackable");
1053		return -EINVAL;
1054	}
1055	t->all_blk_mq = v.mq_count > 0;
1056
1057	if (!t->all_blk_mq &&
1058	    (t->type == DM_TYPE_MQ_REQUEST_BASED || t->type == DM_TYPE_NVME_BIO_BASED)) {
1059		DMERR("table load rejected: all devices are not blk-mq request-stackable");
1060		return -EINVAL;
1061	}
1062
1063	return 0;
1064}
1065
1066enum dm_queue_mode dm_table_get_type(struct dm_table *t)
1067{
1068	return t->type;
1069}
1070
1071struct target_type *dm_table_get_immutable_target_type(struct dm_table *t)
1072{
1073	return t->immutable_target_type;
1074}
1075
1076struct dm_target *dm_table_get_immutable_target(struct dm_table *t)
1077{
1078	/* Immutable target is implicitly a singleton */
1079	if (t->num_targets > 1 ||
1080	    !dm_target_is_immutable(t->targets[0].type))
1081		return NULL;
1082
1083	return t->targets;
1084}
1085
1086struct dm_target *dm_table_get_wildcard_target(struct dm_table *t)
1087{
1088	struct dm_target *ti;
1089	unsigned i;
1090
1091	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1092		ti = dm_table_get_target(t, i);
1093		if (dm_target_is_wildcard(ti->type))
1094			return ti;
1095	}
1096
1097	return NULL;
1098}
1099
1100bool dm_table_bio_based(struct dm_table *t)
1101{
1102	return __table_type_bio_based(dm_table_get_type(t));
1103}
1104
1105bool dm_table_request_based(struct dm_table *t)
1106{
1107	return __table_type_request_based(dm_table_get_type(t));
1108}
1109
1110bool dm_table_all_blk_mq_devices(struct dm_table *t)
1111{
1112	return t->all_blk_mq;
1113}
1114
1115static int dm_table_alloc_md_mempools(struct dm_table *t, struct mapped_device *md)
1116{
1117	enum dm_queue_mode type = dm_table_get_type(t);
1118	unsigned per_io_data_size = 0;
1119	unsigned min_pool_size = 0;
1120	struct dm_target *ti;
1121	unsigned i;
1122
1123	if (unlikely(type == DM_TYPE_NONE)) {
1124		DMWARN("no table type is set, can't allocate mempools");
1125		return -EINVAL;
1126	}
1127
1128	if (__table_type_bio_based(type))
1129		for (i = 0; i < t->num_targets; i++) {
1130			ti = t->targets + i;
1131			per_io_data_size = max(per_io_data_size, ti->per_io_data_size);
1132			min_pool_size = max(min_pool_size, ti->num_flush_bios);
1133		}
1134
1135	t->mempools = dm_alloc_md_mempools(md, type, t->integrity_supported,
1136					   per_io_data_size, min_pool_size);
1137	if (!t->mempools)
1138		return -ENOMEM;
1139
1140	return 0;
1141}
1142
1143void dm_table_free_md_mempools(struct dm_table *t)
1144{
1145	dm_free_md_mempools(t->mempools);
1146	t->mempools = NULL;
1147}
1148
1149struct dm_md_mempools *dm_table_get_md_mempools(struct dm_table *t)
1150{
1151	return t->mempools;
1152}
1153
1154static int setup_indexes(struct dm_table *t)
1155{
1156	int i;
1157	unsigned int total = 0;
1158	sector_t *indexes;
1159
1160	/* allocate the space for *all* the indexes */
1161	for (i = t->depth - 2; i >= 0; i--) {
1162		t->counts[i] = dm_div_up(t->counts[i + 1], CHILDREN_PER_NODE);
1163		total += t->counts[i];
1164	}
1165
1166	indexes = (sector_t *) dm_vcalloc(total, (unsigned long) NODE_SIZE);
1167	if (!indexes)
1168		return -ENOMEM;
1169
1170	/* set up internal nodes, bottom-up */
1171	for (i = t->depth - 2; i >= 0; i--) {
1172		t->index[i] = indexes;
1173		indexes += (KEYS_PER_NODE * t->counts[i]);
1174		setup_btree_index(i, t);
1175	}
1176
1177	return 0;
1178}
1179
1180/*
1181 * Builds the btree to index the map.
1182 */
1183static int dm_table_build_index(struct dm_table *t)
1184{
1185	int r = 0;
1186	unsigned int leaf_nodes;
1187
1188	/* how many indexes will the btree have ? */
1189	leaf_nodes = dm_div_up(t->num_targets, KEYS_PER_NODE);
1190	t->depth = 1 + int_log(leaf_nodes, CHILDREN_PER_NODE);
1191
1192	/* leaf layer has already been set up */
1193	t->counts[t->depth - 1] = leaf_nodes;
1194	t->index[t->depth - 1] = t->highs;
1195
1196	if (t->depth >= 2)
1197		r = setup_indexes(t);
1198
1199	return r;
1200}
1201
1202static bool integrity_profile_exists(struct gendisk *disk)
1203{
1204	return !!blk_get_integrity(disk);
1205}
1206
1207/*
1208 * Get a disk whose integrity profile reflects the table's profile.
1209 * Returns NULL if integrity support was inconsistent or unavailable.
1210 */
1211static struct gendisk * dm_table_get_integrity_disk(struct dm_table *t)
1212{
1213	struct list_head *devices = dm_table_get_devices(t);
1214	struct dm_dev_internal *dd = NULL;
1215	struct gendisk *prev_disk = NULL, *template_disk = NULL;
1216	unsigned i;
1217
1218	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1219		struct dm_target *ti = dm_table_get_target(t, i);
1220		if (!dm_target_passes_integrity(ti->type))
1221			goto no_integrity;
1222	}
1223
1224	list_for_each_entry(dd, devices, list) {
1225		template_disk = dd->dm_dev->bdev->bd_disk;
1226		if (!integrity_profile_exists(template_disk))
1227			goto no_integrity;
1228		else if (prev_disk &&
1229			 blk_integrity_compare(prev_disk, template_disk) < 0)
1230			goto no_integrity;
1231		prev_disk = template_disk;
1232	}
1233
1234	return template_disk;
1235
1236no_integrity:
1237	if (prev_disk)
1238		DMWARN("%s: integrity not set: %s and %s profile mismatch",
1239		       dm_device_name(t->md),
1240		       prev_disk->disk_name,
1241		       template_disk->disk_name);
1242	return NULL;
1243}
1244
1245/*
1246 * Register the mapped device for blk_integrity support if the
1247 * underlying devices have an integrity profile.  But all devices may
1248 * not have matching profiles (checking all devices isn't reliable
1249 * during table load because this table may use other DM device(s) which
1250 * must be resumed before they will have an initialized integity
1251 * profile).  Consequently, stacked DM devices force a 2 stage integrity
1252 * profile validation: First pass during table load, final pass during
1253 * resume.
1254 */
1255static int dm_table_register_integrity(struct dm_table *t)
1256{
1257	struct mapped_device *md = t->md;
1258	struct gendisk *template_disk = NULL;
1259
1260	/* If target handles integrity itself do not register it here. */
1261	if (t->integrity_added)
1262		return 0;
1263
1264	template_disk = dm_table_get_integrity_disk(t);
1265	if (!template_disk)
1266		return 0;
1267
1268	if (!integrity_profile_exists(dm_disk(md))) {
1269		t->integrity_supported = true;
1270		/*
1271		 * Register integrity profile during table load; we can do
1272		 * this because the final profile must match during resume.
1273		 */
1274		blk_integrity_register(dm_disk(md),
1275				       blk_get_integrity(template_disk));
1276		return 0;
1277	}
1278
1279	/*
1280	 * If DM device already has an initialized integrity
1281	 * profile the new profile should not conflict.
1282	 */
1283	if (blk_integrity_compare(dm_disk(md), template_disk) < 0) {
1284		DMWARN("%s: conflict with existing integrity profile: "
1285		       "%s profile mismatch",
1286		       dm_device_name(t->md),
1287		       template_disk->disk_name);
1288		return 1;
1289	}
1290
1291	/* Preserve existing integrity profile */
1292	t->integrity_supported = true;
1293	return 0;
1294}
1295
1296/*
1297 * Prepares the table for use by building the indices,
1298 * setting the type, and allocating mempools.
1299 */
1300int dm_table_complete(struct dm_table *t)
1301{
1302	int r;
1303
1304	r = dm_table_determine_type(t);
1305	if (r) {
1306		DMERR("unable to determine table type");
1307		return r;
1308	}
1309
1310	r = dm_table_build_index(t);
1311	if (r) {
1312		DMERR("unable to build btrees");
1313		return r;
1314	}
1315
1316	r = dm_table_register_integrity(t);
1317	if (r) {
1318		DMERR("could not register integrity profile.");
1319		return r;
1320	}
1321
1322	r = dm_table_alloc_md_mempools(t, t->md);
1323	if (r)
1324		DMERR("unable to allocate mempools");
1325
1326	return r;
1327}
1328
1329static DEFINE_MUTEX(_event_lock);
1330void dm_table_event_callback(struct dm_table *t,
1331			     void (*fn)(void *), void *context)
1332{
1333	mutex_lock(&_event_lock);
1334	t->event_fn = fn;
1335	t->event_context = context;
1336	mutex_unlock(&_event_lock);
1337}
1338
1339void dm_table_event(struct dm_table *t)
1340{
1341	/*
1342	 * You can no longer call dm_table_event() from interrupt
1343	 * context, use a bottom half instead.
1344	 */
1345	BUG_ON(in_interrupt());
1346
1347	mutex_lock(&_event_lock);
1348	if (t->event_fn)
1349		t->event_fn(t->event_context);
1350	mutex_unlock(&_event_lock);
1351}
1352EXPORT_SYMBOL(dm_table_event);
1353
1354sector_t dm_table_get_size(struct dm_table *t)
1355{
1356	return t->num_targets ? (t->highs[t->num_targets - 1] + 1) : 0;
1357}
1358EXPORT_SYMBOL(dm_table_get_size);
1359
1360struct dm_target *dm_table_get_target(struct dm_table *t, unsigned int index)
1361{
1362	if (index >= t->num_targets)
1363		return NULL;
1364
1365	return t->targets + index;
1366}
1367
1368/*
1369 * Search the btree for the correct target.
1370 *
1371 * Caller should check returned pointer with dm_target_is_valid()
1372 * to trap I/O beyond end of device.
1373 */
1374struct dm_target *dm_table_find_target(struct dm_table *t, sector_t sector)
1375{
1376	unsigned int l, n = 0, k = 0;
1377	sector_t *node;
1378
1379	for (l = 0; l < t->depth; l++) {
1380		n = get_child(n, k);
1381		node = get_node(t, l, n);
1382
1383		for (k = 0; k < KEYS_PER_NODE; k++)
1384			if (node[k] >= sector)
1385				break;
1386	}
1387
1388	return &t->targets[(KEYS_PER_NODE * n) + k];
1389}
1390
1391static int count_device(struct dm_target *ti, struct dm_dev *dev,
1392			sector_t start, sector_t len, void *data)
1393{
1394	unsigned *num_devices = data;
1395
1396	(*num_devices)++;
1397
1398	return 0;
1399}
1400
1401/*
1402 * Check whether a table has no data devices attached using each
1403 * target's iterate_devices method.
1404 * Returns false if the result is unknown because a target doesn't
1405 * support iterate_devices.
1406 */
1407bool dm_table_has_no_data_devices(struct dm_table *table)
1408{
1409	struct dm_target *ti;
1410	unsigned i, num_devices;
1411
1412	for (i = 0; i < dm_table_get_num_targets(table); i++) {
1413		ti = dm_table_get_target(table, i);
1414
1415		if (!ti->type->iterate_devices)
1416			return false;
1417
1418		num_devices = 0;
1419		ti->type->iterate_devices(ti, count_device, &num_devices);
1420		if (num_devices)
1421			return false;
1422	}
1423
1424	return true;
1425}
1426
1427static int device_is_zoned_model(struct dm_target *ti, struct dm_dev *dev,
1428				 sector_t start, sector_t len, void *data)
1429{
1430	struct request_queue *q = bdev_get_queue(dev->bdev);
1431	enum blk_zoned_model *zoned_model = data;
1432
1433	return q && blk_queue_zoned_model(q) == *zoned_model;
1434}
1435
1436static bool dm_table_supports_zoned_model(struct dm_table *t,
1437					  enum blk_zoned_model zoned_model)
1438{
1439	struct dm_target *ti;
1440	unsigned i;
1441
1442	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1443		ti = dm_table_get_target(t, i);
1444
1445		if (zoned_model == BLK_ZONED_HM &&
1446		    !dm_target_supports_zoned_hm(ti->type))
1447			return false;
1448
1449		if (!ti->type->iterate_devices ||
1450		    !ti->type->iterate_devices(ti, device_is_zoned_model, &zoned_model))
1451			return false;
1452	}
1453
1454	return true;
1455}
1456
1457static int device_matches_zone_sectors(struct dm_target *ti, struct dm_dev *dev,
1458				       sector_t start, sector_t len, void *data)
1459{
1460	struct request_queue *q = bdev_get_queue(dev->bdev);
1461	unsigned int *zone_sectors = data;
1462
1463	return q && blk_queue_zone_sectors(q) == *zone_sectors;
1464}
1465
1466static bool dm_table_matches_zone_sectors(struct dm_table *t,
1467					  unsigned int zone_sectors)
1468{
1469	struct dm_target *ti;
1470	unsigned i;
1471
1472	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1473		ti = dm_table_get_target(t, i);
1474
1475		if (!ti->type->iterate_devices ||
1476		    !ti->type->iterate_devices(ti, device_matches_zone_sectors, &zone_sectors))
1477			return false;
1478	}
1479
1480	return true;
1481}
1482
1483static int validate_hardware_zoned_model(struct dm_table *table,
1484					 enum blk_zoned_model zoned_model,
1485					 unsigned int zone_sectors)
1486{
1487	if (zoned_model == BLK_ZONED_NONE)
1488		return 0;
1489
1490	if (!dm_table_supports_zoned_model(table, zoned_model)) {
1491		DMERR("%s: zoned model is not consistent across all devices",
1492		      dm_device_name(table->md));
1493		return -EINVAL;
1494	}
1495
1496	/* Check zone size validity and compatibility */
1497	if (!zone_sectors || !is_power_of_2(zone_sectors))
1498		return -EINVAL;
1499
1500	if (!dm_table_matches_zone_sectors(table, zone_sectors)) {
1501		DMERR("%s: zone sectors is not consistent across all devices",
1502		      dm_device_name(table->md));
1503		return -EINVAL;
1504	}
1505
1506	return 0;
1507}
1508
1509/*
1510 * Establish the new table's queue_limits and validate them.
1511 */
1512int dm_calculate_queue_limits(struct dm_table *table,
1513			      struct queue_limits *limits)
1514{
1515	struct dm_target *ti;
1516	struct queue_limits ti_limits;
1517	unsigned i;
1518	enum blk_zoned_model zoned_model = BLK_ZONED_NONE;
1519	unsigned int zone_sectors = 0;
1520
1521	blk_set_stacking_limits(limits);
1522
1523	for (i = 0; i < dm_table_get_num_targets(table); i++) {
1524		blk_set_stacking_limits(&ti_limits);
1525
1526		ti = dm_table_get_target(table, i);
1527
1528		if (!ti->type->iterate_devices)
1529			goto combine_limits;
1530
1531		/*
1532		 * Combine queue limits of all the devices this target uses.
1533		 */
1534		ti->type->iterate_devices(ti, dm_set_device_limits,
1535					  &ti_limits);
1536
1537		if (zoned_model == BLK_ZONED_NONE && ti_limits.zoned != BLK_ZONED_NONE) {
1538			/*
1539			 * After stacking all limits, validate all devices
1540			 * in table support this zoned model and zone sectors.
1541			 */
1542			zoned_model = ti_limits.zoned;
1543			zone_sectors = ti_limits.chunk_sectors;
1544		}
1545
1546		/* Set I/O hints portion of queue limits */
1547		if (ti->type->io_hints)
1548			ti->type->io_hints(ti, &ti_limits);
1549
1550		/*
1551		 * Check each device area is consistent with the target's
1552		 * overall queue limits.
1553		 */
1554		if (ti->type->iterate_devices(ti, device_area_is_invalid,
1555					      &ti_limits))
1556			return -EINVAL;
1557
1558combine_limits:
1559		/*
1560		 * Merge this target's queue limits into the overall limits
1561		 * for the table.
1562		 */
1563		if (blk_stack_limits(limits, &ti_limits, 0) < 0)
1564			DMWARN("%s: adding target device "
1565			       "(start sect %llu len %llu) "
1566			       "caused an alignment inconsistency",
1567			       dm_device_name(table->md),
1568			       (unsigned long long) ti->begin,
1569			       (unsigned long long) ti->len);
1570
1571		/*
1572		 * FIXME: this should likely be moved to blk_stack_limits(), would
1573		 * also eliminate limits->zoned stacking hack in dm_set_device_limits()
1574		 */
1575		if (limits->zoned == BLK_ZONED_NONE && ti_limits.zoned != BLK_ZONED_NONE) {
1576			/*
1577			 * By default, the stacked limits zoned model is set to
1578			 * BLK_ZONED_NONE in blk_set_stacking_limits(). Update
1579			 * this model using the first target model reported
1580			 * that is not BLK_ZONED_NONE. This will be either the
1581			 * first target device zoned model or the model reported
1582			 * by the target .io_hints.
1583			 */
1584			limits->zoned = ti_limits.zoned;
1585		}
1586	}
1587
1588	/*
1589	 * Verify that the zoned model and zone sectors, as determined before
1590	 * any .io_hints override, are the same across all devices in the table.
1591	 * - this is especially relevant if .io_hints is emulating a disk-managed
1592	 *   zoned model (aka BLK_ZONED_NONE) on host-managed zoned block devices.
1593	 * BUT...
1594	 */
1595	if (limits->zoned != BLK_ZONED_NONE) {
1596		/*
1597		 * ...IF the above limits stacking determined a zoned model
1598		 * validate that all of the table's devices conform to it.
1599		 */
1600		zoned_model = limits->zoned;
1601		zone_sectors = limits->chunk_sectors;
1602	}
1603	if (validate_hardware_zoned_model(table, zoned_model, zone_sectors))
1604		return -EINVAL;
1605
1606	return validate_hardware_logical_block_alignment(table, limits);
1607}
1608
1609/*
1610 * Verify that all devices have an integrity profile that matches the
1611 * DM device's registered integrity profile.  If the profiles don't
1612 * match then unregister the DM device's integrity profile.
1613 */
1614static void dm_table_verify_integrity(struct dm_table *t)
1615{
1616	struct gendisk *template_disk = NULL;
1617
1618	if (t->integrity_added)
1619		return;
1620
1621	if (t->integrity_supported) {
1622		/*
1623		 * Verify that the original integrity profile
1624		 * matches all the devices in this table.
1625		 */
1626		template_disk = dm_table_get_integrity_disk(t);
1627		if (template_disk &&
1628		    blk_integrity_compare(dm_disk(t->md), template_disk) >= 0)
1629			return;
1630	}
1631
1632	if (integrity_profile_exists(dm_disk(t->md))) {
1633		DMWARN("%s: unable to establish an integrity profile",
1634		       dm_device_name(t->md));
1635		blk_integrity_unregister(dm_disk(t->md));
1636	}
1637}
1638
1639static int device_flush_capable(struct dm_target *ti, struct dm_dev *dev,
1640				sector_t start, sector_t len, void *data)
1641{
1642	unsigned long flush = (unsigned long) data;
1643	struct request_queue *q = bdev_get_queue(dev->bdev);
1644
1645	return q && (q->queue_flags & flush);
1646}
1647
1648static bool dm_table_supports_flush(struct dm_table *t, unsigned long flush)
1649{
1650	struct dm_target *ti;
1651	unsigned i;
1652
1653	/*
1654	 * Require at least one underlying device to support flushes.
1655	 * t->devices includes internal dm devices such as mirror logs
1656	 * so we need to use iterate_devices here, which targets
1657	 * supporting flushes must provide.
1658	 */
1659	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1660		ti = dm_table_get_target(t, i);
1661
1662		if (!ti->num_flush_bios)
1663			continue;
1664
1665		if (ti->flush_supported)
1666			return true;
1667
1668		if (ti->type->iterate_devices &&
1669		    ti->type->iterate_devices(ti, device_flush_capable, (void *) flush))
1670			return true;
1671	}
1672
1673	return false;
1674}
1675
1676static int device_dax_write_cache_enabled(struct dm_target *ti,
1677					  struct dm_dev *dev, sector_t start,
1678					  sector_t len, void *data)
1679{
1680	struct dax_device *dax_dev = dev->dax_dev;
1681
1682	if (!dax_dev)
1683		return false;
1684
1685	if (dax_write_cache_enabled(dax_dev))
1686		return true;
1687	return false;
1688}
1689
1690static int dm_table_supports_dax_write_cache(struct dm_table *t)
1691{
1692	struct dm_target *ti;
1693	unsigned i;
1694
1695	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1696		ti = dm_table_get_target(t, i);
 
1697
1698		if (ti->type->iterate_devices &&
1699		    ti->type->iterate_devices(ti,
1700				device_dax_write_cache_enabled, NULL))
1701			return true;
1702	}
1703
1704	return false;
1705}
1706
1707static int device_is_nonrot(struct dm_target *ti, struct dm_dev *dev,
1708			    sector_t start, sector_t len, void *data)
1709{
1710	struct request_queue *q = bdev_get_queue(dev->bdev);
1711
1712	return q && blk_queue_nonrot(q);
1713}
1714
1715static int device_is_not_random(struct dm_target *ti, struct dm_dev *dev,
1716			     sector_t start, sector_t len, void *data)
1717{
1718	struct request_queue *q = bdev_get_queue(dev->bdev);
1719
1720	return q && !blk_queue_add_random(q);
1721}
1722
1723static int queue_supports_sg_merge(struct dm_target *ti, struct dm_dev *dev,
1724				   sector_t start, sector_t len, void *data)
1725{
1726	struct request_queue *q = bdev_get_queue(dev->bdev);
1727
1728	return q && !test_bit(QUEUE_FLAG_NO_SG_MERGE, &q->queue_flags);
1729}
1730
1731static bool dm_table_all_devices_attribute(struct dm_table *t,
1732					   iterate_devices_callout_fn func)
1733{
1734	struct dm_target *ti;
1735	unsigned i;
1736
1737	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1738		ti = dm_table_get_target(t, i);
1739
1740		if (!ti->type->iterate_devices ||
1741		    !ti->type->iterate_devices(ti, func, NULL))
1742			return false;
1743	}
1744
1745	return true;
1746}
1747
1748static int device_no_partial_completion(struct dm_target *ti, struct dm_dev *dev,
1749					sector_t start, sector_t len, void *data)
1750{
1751	char b[BDEVNAME_SIZE];
1752
1753	/* For now, NVMe devices are the only devices of this class */
1754	return (strncmp(bdevname(dev->bdev, b), "nvme", 4) == 0);
1755}
1756
1757static bool dm_table_does_not_support_partial_completion(struct dm_table *t)
1758{
1759	return dm_table_all_devices_attribute(t, device_no_partial_completion);
1760}
1761
1762static int device_not_write_same_capable(struct dm_target *ti, struct dm_dev *dev,
1763					 sector_t start, sector_t len, void *data)
1764{
1765	struct request_queue *q = bdev_get_queue(dev->bdev);
1766
1767	return q && !q->limits.max_write_same_sectors;
1768}
1769
1770static bool dm_table_supports_write_same(struct dm_table *t)
1771{
1772	struct dm_target *ti;
1773	unsigned i;
1774
1775	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1776		ti = dm_table_get_target(t, i);
1777
1778		if (!ti->num_write_same_bios)
1779			return false;
1780
1781		if (!ti->type->iterate_devices ||
1782		    ti->type->iterate_devices(ti, device_not_write_same_capable, NULL))
1783			return false;
1784	}
1785
1786	return true;
1787}
1788
1789static int device_not_write_zeroes_capable(struct dm_target *ti, struct dm_dev *dev,
1790					   sector_t start, sector_t len, void *data)
1791{
1792	struct request_queue *q = bdev_get_queue(dev->bdev);
1793
1794	return q && !q->limits.max_write_zeroes_sectors;
1795}
1796
1797static bool dm_table_supports_write_zeroes(struct dm_table *t)
1798{
1799	struct dm_target *ti;
1800	unsigned i = 0;
1801
 
 
 
 
 
 
 
1802	while (i < dm_table_get_num_targets(t)) {
1803		ti = dm_table_get_target(t, i++);
1804
1805		if (!ti->num_write_zeroes_bios)
1806			return false;
1807
1808		if (!ti->type->iterate_devices ||
1809		    ti->type->iterate_devices(ti, device_not_write_zeroes_capable, NULL))
1810			return false;
1811	}
1812
1813	return true;
1814}
1815
1816static int device_not_discard_capable(struct dm_target *ti, struct dm_dev *dev,
1817				      sector_t start, sector_t len, void *data)
1818{
1819	struct request_queue *q = bdev_get_queue(dev->bdev);
1820
1821	return q && !blk_queue_discard(q);
1822}
1823
1824static bool dm_table_supports_discards(struct dm_table *t)
1825{
1826	struct dm_target *ti;
1827	unsigned i;
1828
1829	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1830		ti = dm_table_get_target(t, i);
1831
1832		if (!ti->num_discard_bios)
1833			return false;
1834
1835		/*
1836		 * Either the target provides discard support (as implied by setting
1837		 * 'discards_supported') or it relies on _all_ data devices having
1838		 * discard support.
1839		 */
1840		if (!ti->discards_supported &&
1841		    (!ti->type->iterate_devices ||
1842		     ti->type->iterate_devices(ti, device_not_discard_capable, NULL)))
1843			return false;
1844	}
1845
1846	return true;
1847}
1848
1849static int device_not_secure_erase_capable(struct dm_target *ti,
1850					   struct dm_dev *dev, sector_t start,
1851					   sector_t len, void *data)
1852{
1853	struct request_queue *q = bdev_get_queue(dev->bdev);
1854
1855	return q && !blk_queue_secure_erase(q);
1856}
1857
1858static bool dm_table_supports_secure_erase(struct dm_table *t)
1859{
1860	struct dm_target *ti;
1861	unsigned int i;
1862
1863	for (i = 0; i < dm_table_get_num_targets(t); i++) {
1864		ti = dm_table_get_target(t, i);
1865
1866		if (!ti->num_secure_erase_bios)
1867			return false;
1868
1869		if (!ti->type->iterate_devices ||
1870		    ti->type->iterate_devices(ti, device_not_secure_erase_capable, NULL))
1871			return false;
1872	}
1873
1874	return true;
1875}
1876
1877void dm_table_set_restrictions(struct dm_table *t, struct request_queue *q,
1878			       struct queue_limits *limits)
1879{
1880	bool wc = false, fua = false;
1881
1882	/*
1883	 * Copy table's limits to the DM device's request_queue
1884	 */
1885	q->limits = *limits;
1886
1887	if (!dm_table_supports_discards(t)) {
1888		blk_queue_flag_clear(QUEUE_FLAG_DISCARD, q);
1889		/* Must also clear discard limits... */
1890		q->limits.max_discard_sectors = 0;
1891		q->limits.max_hw_discard_sectors = 0;
1892		q->limits.discard_granularity = 0;
1893		q->limits.discard_alignment = 0;
1894		q->limits.discard_misaligned = 0;
1895	} else
1896		blk_queue_flag_set(QUEUE_FLAG_DISCARD, q);
1897
1898	if (dm_table_supports_secure_erase(t))
1899		blk_queue_flag_set(QUEUE_FLAG_SECERASE, q);
 
 
 
 
1900
1901	if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_WC))) {
1902		wc = true;
1903		if (dm_table_supports_flush(t, (1UL << QUEUE_FLAG_FUA)))
1904			fua = true;
1905	}
1906	blk_queue_write_cache(q, wc, fua);
1907
1908	if (dm_table_supports_dax(t))
1909		blk_queue_flag_set(QUEUE_FLAG_DAX, q);
1910	if (dm_table_supports_dax_write_cache(t))
1911		dax_write_cache(t->md->dax_dev, true);
1912
1913	/* Ensure that all underlying devices are non-rotational. */
1914	if (dm_table_all_devices_attribute(t, device_is_nonrot))
1915		blk_queue_flag_set(QUEUE_FLAG_NONROT, q);
1916	else
1917		blk_queue_flag_clear(QUEUE_FLAG_NONROT, q);
1918
1919	if (!dm_table_supports_write_same(t))
1920		q->limits.max_write_same_sectors = 0;
1921	if (!dm_table_supports_write_zeroes(t))
1922		q->limits.max_write_zeroes_sectors = 0;
1923
1924	if (dm_table_all_devices_attribute(t, queue_supports_sg_merge))
1925		blk_queue_flag_clear(QUEUE_FLAG_NO_SG_MERGE, q);
1926	else
1927		blk_queue_flag_set(QUEUE_FLAG_NO_SG_MERGE, q);
1928
1929	dm_table_verify_integrity(t);
1930
1931	/*
1932	 * Determine whether or not this queue's I/O timings contribute
1933	 * to the entropy pool, Only request-based targets use this.
1934	 * Clear QUEUE_FLAG_ADD_RANDOM if any underlying device does not
1935	 * have it set.
1936	 */
1937	if (blk_queue_add_random(q) && dm_table_all_devices_attribute(t, device_is_not_random))
1938		blk_queue_flag_clear(QUEUE_FLAG_ADD_RANDOM, q);
 
 
 
 
 
 
 
 
 
 
 
 
 
1939}
1940
1941unsigned int dm_table_get_num_targets(struct dm_table *t)
1942{
1943	return t->num_targets;
1944}
1945
1946struct list_head *dm_table_get_devices(struct dm_table *t)
1947{
1948	return &t->devices;
1949}
1950
1951fmode_t dm_table_get_mode(struct dm_table *t)
1952{
1953	return t->mode;
1954}
1955EXPORT_SYMBOL(dm_table_get_mode);
1956
1957enum suspend_mode {
1958	PRESUSPEND,
1959	PRESUSPEND_UNDO,
1960	POSTSUSPEND,
1961};
1962
1963static void suspend_targets(struct dm_table *t, enum suspend_mode mode)
1964{
1965	int i = t->num_targets;
1966	struct dm_target *ti = t->targets;
1967
1968	lockdep_assert_held(&t->md->suspend_lock);
1969
1970	while (i--) {
1971		switch (mode) {
1972		case PRESUSPEND:
1973			if (ti->type->presuspend)
1974				ti->type->presuspend(ti);
1975			break;
1976		case PRESUSPEND_UNDO:
1977			if (ti->type->presuspend_undo)
1978				ti->type->presuspend_undo(ti);
1979			break;
1980		case POSTSUSPEND:
1981			if (ti->type->postsuspend)
1982				ti->type->postsuspend(ti);
1983			break;
1984		}
1985		ti++;
1986	}
1987}
1988
1989void dm_table_presuspend_targets(struct dm_table *t)
1990{
1991	if (!t)
1992		return;
1993
1994	suspend_targets(t, PRESUSPEND);
1995}
1996
1997void dm_table_presuspend_undo_targets(struct dm_table *t)
1998{
1999	if (!t)
2000		return;
2001
2002	suspend_targets(t, PRESUSPEND_UNDO);
2003}
2004
2005void dm_table_postsuspend_targets(struct dm_table *t)
2006{
2007	if (!t)
2008		return;
2009
2010	suspend_targets(t, POSTSUSPEND);
2011}
2012
2013int dm_table_resume_targets(struct dm_table *t)
2014{
2015	int i, r = 0;
2016
2017	lockdep_assert_held(&t->md->suspend_lock);
2018
2019	for (i = 0; i < t->num_targets; i++) {
2020		struct dm_target *ti = t->targets + i;
2021
2022		if (!ti->type->preresume)
2023			continue;
2024
2025		r = ti->type->preresume(ti);
2026		if (r) {
2027			DMERR("%s: %s: preresume failed, error = %d",
2028			      dm_device_name(t->md), ti->type->name, r);
2029			return r;
2030		}
2031	}
2032
2033	for (i = 0; i < t->num_targets; i++) {
2034		struct dm_target *ti = t->targets + i;
2035
2036		if (ti->type->resume)
2037			ti->type->resume(ti);
2038	}
2039
2040	return 0;
2041}
2042
2043void dm_table_add_target_callbacks(struct dm_table *t, struct dm_target_callbacks *cb)
2044{
2045	list_add(&cb->list, &t->target_callbacks);
2046}
2047EXPORT_SYMBOL_GPL(dm_table_add_target_callbacks);
2048
2049int dm_table_any_congested(struct dm_table *t, int bdi_bits)
2050{
2051	struct dm_dev_internal *dd;
2052	struct list_head *devices = dm_table_get_devices(t);
2053	struct dm_target_callbacks *cb;
2054	int r = 0;
2055
2056	list_for_each_entry(dd, devices, list) {
2057		struct request_queue *q = bdev_get_queue(dd->dm_dev->bdev);
2058		char b[BDEVNAME_SIZE];
2059
2060		if (likely(q))
2061			r |= bdi_congested(q->backing_dev_info, bdi_bits);
2062		else
2063			DMWARN_LIMIT("%s: any_congested: nonexistent device %s",
2064				     dm_device_name(t->md),
2065				     bdevname(dd->dm_dev->bdev, b));
2066	}
2067
2068	list_for_each_entry(cb, &t->target_callbacks, list)
2069		if (cb->congested_fn)
2070			r |= cb->congested_fn(cb, bdi_bits);
2071
2072	return r;
2073}
2074
2075struct mapped_device *dm_table_get_md(struct dm_table *t)
2076{
2077	return t->md;
2078}
2079EXPORT_SYMBOL(dm_table_get_md);
2080
2081void dm_table_run_md_queue_async(struct dm_table *t)
2082{
2083	struct mapped_device *md;
2084	struct request_queue *queue;
2085	unsigned long flags;
2086
2087	if (!dm_table_request_based(t))
2088		return;
2089
2090	md = dm_table_get_md(t);
2091	queue = dm_get_md_queue(md);
2092	if (queue) {
2093		if (queue->mq_ops)
2094			blk_mq_run_hw_queues(queue, true);
2095		else {
2096			spin_lock_irqsave(queue->queue_lock, flags);
2097			blk_run_queue_async(queue);
2098			spin_unlock_irqrestore(queue->queue_lock, flags);
2099		}
2100	}
2101}
2102EXPORT_SYMBOL(dm_table_run_md_queue_async);
2103