1/*
   2 * Copyright (C) 2001, 2002 Sistina Software (UK) Limited.
   3 * Copyright (C) 2004 - 2006 Red Hat, Inc. All rights reserved.
   4 *
   5 * This file is released under the GPL.
   6 */
   7
   8#include "dm-core.h"
   9#include "dm-ima.h"
  10#include <linux/module.h>
  11#include <linux/vmalloc.h>
  12#include <linux/miscdevice.h>
  13#include <linux/sched/mm.h>
  14#include <linux/init.h>
  15#include <linux/wait.h>
  16#include <linux/slab.h>
  17#include <linux/rbtree.h>
  18#include <linux/dm-ioctl.h>
  19#include <linux/hdreg.h>
  20#include <linux/compat.h>
  21#include <linux/nospec.h>
  22
  23#include <linux/uaccess.h>
  24#include <linux/ima.h>
  25
  26#define DM_MSG_PREFIX "ioctl"
  27#define DM_DRIVER_EMAIL "dm-devel@redhat.com"
  28
  29struct dm_file {
  30	/*
  31	 * poll will wait until the global event number is greater than
  32	 * this value.
  33	 */
  34	volatile unsigned global_event_nr;
  35};
  36
  37/*-----------------------------------------------------------------
  38 * The ioctl interface needs to be able to look up devices by
  39 * name or uuid.
  40 *---------------------------------------------------------------*/
  41struct hash_cell {
  42	struct rb_node name_node;
  43	struct rb_node uuid_node;
  44	bool name_set;
  45	bool uuid_set;
  46
  47	char *name;
  48	char *uuid;
  49	struct mapped_device *md;
  50	struct dm_table *new_map;
  51};
  52
  53struct vers_iter {
  54    size_t param_size;
  55    struct dm_target_versions *vers, *old_vers;
  56    char *end;
  57    uint32_t flags;
  58};
  59
  60
  61static struct rb_root name_rb_tree = RB_ROOT;
  62static struct rb_root uuid_rb_tree = RB_ROOT;
  63
  64static void dm_hash_remove_all(bool keep_open_devices, bool mark_deferred, bool only_deferred);
  65
  66/*
  67 * Guards access to both hash tables.
  68 */
  69static DECLARE_RWSEM(_hash_lock);
  70
  71/*
  72 * Protects use of mdptr to obtain hash cell name and uuid from mapped device.
  73 */
  74static DEFINE_MUTEX(dm_hash_cells_mutex);
  75
  76static void dm_hash_exit(void)
  77{
  78	dm_hash_remove_all(false, false, false);
  79}
  80
  81/*-----------------------------------------------------------------
  82 * Code for looking up a device by name
  83 *---------------------------------------------------------------*/
  84static struct hash_cell *__get_name_cell(const char *str)
  85{
  86	struct rb_node *n = name_rb_tree.rb_node;
  87
  88	while (n) {
  89		struct hash_cell *hc = container_of(n, struct hash_cell, name_node);
  90		int c = strcmp(hc->name, str);
  91		if (!c) {
  92			dm_get(hc->md);
  93			return hc;
  94		}
  95		n = c >= 0 ? n->rb_left : n->rb_right;
  96	}
  97
  98	return NULL;
  99}
 100
 101static struct hash_cell *__get_uuid_cell(const char *str)
 102{
 103	struct rb_node *n = uuid_rb_tree.rb_node;
 104
 105	while (n) {
 106		struct hash_cell *hc = container_of(n, struct hash_cell, uuid_node);
 107		int c = strcmp(hc->uuid, str);
 108		if (!c) {
 109			dm_get(hc->md);
 110			return hc;
 111		}
 112		n = c >= 0 ? n->rb_left : n->rb_right;
 113	}
 114
 115	return NULL;
 116}
 117
 118static void __unlink_name(struct hash_cell *hc)
 119{
 120	if (hc->name_set) {
 121		hc->name_set = false;
 122		rb_erase(&hc->name_node, &name_rb_tree);
 123	}
 124}
 125
 126static void __unlink_uuid(struct hash_cell *hc)
 127{
 128	if (hc->uuid_set) {
 129		hc->uuid_set = false;
 130		rb_erase(&hc->uuid_node, &uuid_rb_tree);
 131	}
 132}
 133
 134static void __link_name(struct hash_cell *new_hc)
 135{
 136	struct rb_node **n, *parent;
 137
 138	__unlink_name(new_hc);
 139
 140	new_hc->name_set = true;
 141
 142	n = &name_rb_tree.rb_node;
 143	parent = NULL;
 144
 145	while (*n) {
 146		struct hash_cell *hc = container_of(*n, struct hash_cell, name_node);
 147		int c = strcmp(hc->name, new_hc->name);
 148		BUG_ON(!c);
 149		parent = *n;
 150		n = c >= 0 ? &hc->name_node.rb_left : &hc->name_node.rb_right;
 151	}
 152
 153	rb_link_node(&new_hc->name_node, parent, n);
 154	rb_insert_color(&new_hc->name_node, &name_rb_tree);
 155}
 156
 157static void __link_uuid(struct hash_cell *new_hc)
 158{
 159	struct rb_node **n, *parent;
 160
 161	__unlink_uuid(new_hc);
 162
 163	new_hc->uuid_set = true;
 164
 165	n = &uuid_rb_tree.rb_node;
 166	parent = NULL;
 167
 168	while (*n) {
 169		struct hash_cell *hc = container_of(*n, struct hash_cell, uuid_node);
 170		int c = strcmp(hc->uuid, new_hc->uuid);
 171		BUG_ON(!c);
 172		parent = *n;
 173		n = c > 0 ? &hc->uuid_node.rb_left : &hc->uuid_node.rb_right;
 174	}
 175
 176	rb_link_node(&new_hc->uuid_node, parent, n);
 177	rb_insert_color(&new_hc->uuid_node, &uuid_rb_tree);
 178}
 179
 180static struct hash_cell *__get_dev_cell(uint64_t dev)
 181{
 182	struct mapped_device *md;
 183	struct hash_cell *hc;
 184
 185	md = dm_get_md(huge_decode_dev(dev));
 186	if (!md)
 187		return NULL;
 188
 189	hc = dm_get_mdptr(md);
 190	if (!hc) {
 191		dm_put(md);
 192		return NULL;
 193	}
 194
 195	return hc;
 196}
 197
 198/*-----------------------------------------------------------------
 199 * Inserting, removing and renaming a device.
 200 *---------------------------------------------------------------*/
 201static struct hash_cell *alloc_cell(const char *name, const char *uuid,
 202				    struct mapped_device *md)
 203{
 204	struct hash_cell *hc;
 205
 206	hc = kmalloc(sizeof(*hc), GFP_KERNEL);
 207	if (!hc)
 208		return NULL;
 209
 210	hc->name = kstrdup(name, GFP_KERNEL);
 211	if (!hc->name) {
 212		kfree(hc);
 213		return NULL;
 214	}
 215
 216	if (!uuid)
 217		hc->uuid = NULL;
 218
 219	else {
 220		hc->uuid = kstrdup(uuid, GFP_KERNEL);
 221		if (!hc->uuid) {
 222			kfree(hc->name);
 223			kfree(hc);
 224			return NULL;
 225		}
 226	}
 227
 228	hc->name_set = hc->uuid_set = false;
 229	hc->md = md;
 230	hc->new_map = NULL;
 231	return hc;
 232}
 233
 234static void free_cell(struct hash_cell *hc)
 235{
 236	if (hc) {
 237		kfree(hc->name);
 238		kfree(hc->uuid);
 239		kfree(hc);
 240	}
 241}
 242
 243/*
 244 * The kdev_t and uuid of a device can never change once it is
 245 * initially inserted.
 246 */
 247static int dm_hash_insert(const char *name, const char *uuid, struct mapped_device *md)
 248{
 249	struct hash_cell *cell, *hc;
 250
 251	/*
 252	 * Allocate the new cells.
 253	 */
 254	cell = alloc_cell(name, uuid, md);
 255	if (!cell)
 256		return -ENOMEM;
 257
 258	/*
 259	 * Insert the cell into both hash tables.
 260	 */
 261	down_write(&_hash_lock);
 262	hc = __get_name_cell(name);
 263	if (hc) {
 264		dm_put(hc->md);
 265		goto bad;
 266	}
 267
 268	__link_name(cell);
 269
 270	if (uuid) {
 271		hc = __get_uuid_cell(uuid);
 272		if (hc) {
 273			__unlink_name(cell);
 274			dm_put(hc->md);
 275			goto bad;
 276		}
 277		__link_uuid(cell);
 278	}
 279	dm_get(md);
 280	mutex_lock(&dm_hash_cells_mutex);
 281	dm_set_mdptr(md, cell);
 282	mutex_unlock(&dm_hash_cells_mutex);
 283	up_write(&_hash_lock);
 284
 285	return 0;
 286
 287 bad:
 288	up_write(&_hash_lock);
 289	free_cell(cell);
 290	return -EBUSY;
 291}
 292
 293static struct dm_table *__hash_remove(struct hash_cell *hc)
 294{
 295	struct dm_table *table;
 296	int srcu_idx;
 297
 298	/* remove from the dev trees */
 299	__unlink_name(hc);
 300	__unlink_uuid(hc);
 301	mutex_lock(&dm_hash_cells_mutex);
 302	dm_set_mdptr(hc->md, NULL);
 303	mutex_unlock(&dm_hash_cells_mutex);
 304
 305	table = dm_get_live_table(hc->md, &srcu_idx);
 306	if (table)
 307		dm_table_event(table);
 308	dm_put_live_table(hc->md, srcu_idx);
 309
 310	table = NULL;
 311	if (hc->new_map)
 312		table = hc->new_map;
 313	dm_put(hc->md);
 314	free_cell(hc);
 315
 316	return table;
 317}
 318
 319static void dm_hash_remove_all(bool keep_open_devices, bool mark_deferred, bool only_deferred)
 320{
 321	int dev_skipped;
 322	struct rb_node *n;
 323	struct hash_cell *hc;
 324	struct mapped_device *md;
 325	struct dm_table *t;
 326
 327retry:
 328	dev_skipped = 0;
 329
 330	down_write(&_hash_lock);
 331
 332	for (n = rb_first(&name_rb_tree); n; n = rb_next(n)) {
 333		hc = container_of(n, struct hash_cell, name_node);
 334		md = hc->md;
 335		dm_get(md);
 336
 337		if (keep_open_devices &&
 338		    dm_lock_for_deletion(md, mark_deferred, only_deferred)) {
 339			dm_put(md);
 340			dev_skipped++;
 341			continue;
 342		}
 343
 344		t = __hash_remove(hc);
 345
 346		up_write(&_hash_lock);
 347
 348		if (t) {
 349			dm_sync_table(md);
 350			dm_table_destroy(t);
 351		}
 352		dm_ima_measure_on_device_remove(md, true);
 353		dm_put(md);
 354		if (likely(keep_open_devices))
 355			dm_destroy(md);
 356		else
 357			dm_destroy_immediate(md);
 358
 359		/*
 360		 * Some mapped devices may be using other mapped
 361		 * devices, so repeat until we make no further
 362		 * progress.  If a new mapped device is created
 363		 * here it will also get removed.
 364		 */
 365		goto retry;
 366	}
 367
 368	up_write(&_hash_lock);
 369
 370	if (dev_skipped)
 371		DMWARN("remove_all left %d open device(s)", dev_skipped);
 372}
 373
 374/*
 375 * Set the uuid of a hash_cell that isn't already set.
 376 */
 377static void __set_cell_uuid(struct hash_cell *hc, char *new_uuid)
 378{
 379	mutex_lock(&dm_hash_cells_mutex);
 380	hc->uuid = new_uuid;
 381	mutex_unlock(&dm_hash_cells_mutex);
 382
 383	__link_uuid(hc);
 384}
 385
 386/*
 387 * Changes the name of a hash_cell and returns the old name for
 388 * the caller to free.
 389 */
 390static char *__change_cell_name(struct hash_cell *hc, char *new_name)
 391{
 392	char *old_name;
 393
 394	/*
 395	 * Rename and move the name cell.
 396	 */
 397	__unlink_name(hc);
 398	old_name = hc->name;
 399
 400	mutex_lock(&dm_hash_cells_mutex);
 401	hc->name = new_name;
 402	mutex_unlock(&dm_hash_cells_mutex);
 403
 404	__link_name(hc);
 405
 406	return old_name;
 407}
 408
 409static struct mapped_device *dm_hash_rename(struct dm_ioctl *param,
 410					    const char *new)
 411{
 412	char *new_data, *old_name = NULL;
 413	struct hash_cell *hc;
 414	struct dm_table *table;
 415	struct mapped_device *md;
 416	unsigned change_uuid = (param->flags & DM_UUID_FLAG) ? 1 : 0;
 417	int srcu_idx;
 418
 419	/*
 420	 * duplicate new.
 421	 */
 422	new_data = kstrdup(new, GFP_KERNEL);
 423	if (!new_data)
 424		return ERR_PTR(-ENOMEM);
 425
 426	down_write(&_hash_lock);
 427
 428	/*
 429	 * Is new free ?
 430	 */
 431	if (change_uuid)
 432		hc = __get_uuid_cell(new);
 433	else
 434		hc = __get_name_cell(new);
 435
 436	if (hc) {
 437		DMERR("Unable to change %s on mapped device %s to one that "
 438		      "already exists: %s",
 439		      change_uuid ? "uuid" : "name",
 440		      param->name, new);
 441		dm_put(hc->md);
 442		up_write(&_hash_lock);
 443		kfree(new_data);
 444		return ERR_PTR(-EBUSY);
 445	}
 446
 447	/*
 448	 * Is there such a device as 'old' ?
 449	 */
 450	hc = __get_name_cell(param->name);
 451	if (!hc) {
 452		DMERR("Unable to rename non-existent device, %s to %s%s",
 453		      param->name, change_uuid ? "uuid " : "", new);
 454		up_write(&_hash_lock);
 455		kfree(new_data);
 456		return ERR_PTR(-ENXIO);
 457	}
 458
 459	/*
 460	 * Does this device already have a uuid?
 461	 */
 462	if (change_uuid && hc->uuid) {
 463		DMERR("Unable to change uuid of mapped device %s to %s "
 464		      "because uuid is already set to %s",
 465		      param->name, new, hc->uuid);
 466		dm_put(hc->md);
 467		up_write(&_hash_lock);
 468		kfree(new_data);
 469		return ERR_PTR(-EINVAL);
 470	}
 471
 472	if (change_uuid)
 473		__set_cell_uuid(hc, new_data);
 474	else
 475		old_name = __change_cell_name(hc, new_data);
 476
 477	/*
 478	 * Wake up any dm event waiters.
 479	 */
 480	table = dm_get_live_table(hc->md, &srcu_idx);
 481	if (table)
 482		dm_table_event(table);
 483	dm_put_live_table(hc->md, srcu_idx);
 484
 485	if (!dm_kobject_uevent(hc->md, KOBJ_CHANGE, param->event_nr))
 486		param->flags |= DM_UEVENT_GENERATED_FLAG;
 487
 488	md = hc->md;
 489
 490	dm_ima_measure_on_device_rename(md);
 491
 492	up_write(&_hash_lock);
 493	kfree(old_name);
 494
 495	return md;
 496}
 497
 498void dm_deferred_remove(void)
 499{
 500	dm_hash_remove_all(true, false, true);
 501}
 502
 503/*-----------------------------------------------------------------
 504 * Implementation of the ioctl commands
 505 *---------------------------------------------------------------*/
 506/*
 507 * All the ioctl commands get dispatched to functions with this
 508 * prototype.
 509 */
 510typedef int (*ioctl_fn)(struct file *filp, struct dm_ioctl *param, size_t param_size);
 511
 512static int remove_all(struct file *filp, struct dm_ioctl *param, size_t param_size)
 513{
 514	dm_hash_remove_all(true, !!(param->flags & DM_DEFERRED_REMOVE), false);
 515	param->data_size = 0;
 516	return 0;
 517}
 518
 519/*
 520 * Round up the ptr to an 8-byte boundary.
 521 */
 522#define ALIGN_MASK 7
 523static inline size_t align_val(size_t val)
 524{
 525	return (val + ALIGN_MASK) & ~ALIGN_MASK;
 526}
 527static inline void *align_ptr(void *ptr)
 528{
 529	return (void *)align_val((size_t)ptr);
 530}
 531
 532/*
 533 * Retrieves the data payload buffer from an already allocated
 534 * struct dm_ioctl.
 535 */
 536static void *get_result_buffer(struct dm_ioctl *param, size_t param_size,
 537			       size_t *len)
 538{
 539	param->data_start = align_ptr(param + 1) - (void *) param;
 540
 541	if (param->data_start < param_size)
 542		*len = param_size - param->data_start;
 543	else
 544		*len = 0;
 545
 546	return ((void *) param) + param->data_start;
 547}
 548
 549static bool filter_device(struct hash_cell *hc, const char *pfx_name, const char *pfx_uuid)
 550{
 551	const char *val;
 552	size_t val_len, pfx_len;
 553
 554	val = hc->name;
 555	val_len = strlen(val);
 556	pfx_len = strnlen(pfx_name, DM_NAME_LEN);
 557	if (pfx_len > val_len)
 558		return false;
 559	if (memcmp(val, pfx_name, pfx_len))
 560		return false;
 561
 562	val = hc->uuid ? hc->uuid : "";
 563	val_len = strlen(val);
 564	pfx_len = strnlen(pfx_uuid, DM_UUID_LEN);
 565	if (pfx_len > val_len)
 566		return false;
 567	if (memcmp(val, pfx_uuid, pfx_len))
 568		return false;
 569
 570	return true;
 571}
 572
 573static int list_devices(struct file *filp, struct dm_ioctl *param, size_t param_size)
 574{
 575	struct rb_node *n;
 576	struct hash_cell *hc;
 577	size_t len, needed = 0;
 578	struct gendisk *disk;
 579	struct dm_name_list *orig_nl, *nl, *old_nl = NULL;
 580	uint32_t *event_nr;
 581
 582	down_write(&_hash_lock);
 583
 584	/*
 585	 * Loop through all the devices working out how much
 586	 * space we need.
 587	 */
 588	for (n = rb_first(&name_rb_tree); n; n = rb_next(n)) {
 589		hc = container_of(n, struct hash_cell, name_node);
 590		if (!filter_device(hc, param->name, param->uuid))
 591			continue;
 592		needed += align_val(offsetof(struct dm_name_list, name) + strlen(hc->name) + 1);
 593		needed += align_val(sizeof(uint32_t) * 2);
 594		if (param->flags & DM_UUID_FLAG && hc->uuid)
 595			needed += align_val(strlen(hc->uuid) + 1);
 596	}
 597
 598	/*
 599	 * Grab our output buffer.
 600	 */
 601	nl = orig_nl = get_result_buffer(param, param_size, &len);
 602	if (len < needed || len < sizeof(nl->dev)) {
 603		param->flags |= DM_BUFFER_FULL_FLAG;
 604		goto out;
 605	}
 606	param->data_size = param->data_start + needed;
 607
 608	nl->dev = 0;	/* Flags no data */
 609
 610	/*
 611	 * Now loop through filling out the names.
 612	 */
 613	for (n = rb_first(&name_rb_tree); n; n = rb_next(n)) {
 614		void *uuid_ptr;
 615		hc = container_of(n, struct hash_cell, name_node);
 616		if (!filter_device(hc, param->name, param->uuid))
 617			continue;
 618		if (old_nl)
 619			old_nl->next = (uint32_t) ((void *) nl -
 620						   (void *) old_nl);
 621		disk = dm_disk(hc->md);
 622		nl->dev = huge_encode_dev(disk_devt(disk));
 623		nl->next = 0;
 624		strcpy(nl->name, hc->name);
 625
 626		old_nl = nl;
 627		event_nr = align_ptr(nl->name + strlen(hc->name) + 1);
 628		event_nr[0] = dm_get_event_nr(hc->md);
 629		event_nr[1] = 0;
 630		uuid_ptr = align_ptr(event_nr + 2);
 631		if (param->flags & DM_UUID_FLAG) {
 632			if (hc->uuid) {
 633				event_nr[1] |= DM_NAME_LIST_FLAG_HAS_UUID;
 634				strcpy(uuid_ptr, hc->uuid);
 635				uuid_ptr = align_ptr(uuid_ptr + strlen(hc->uuid) + 1);
 636			} else {
 637				event_nr[1] |= DM_NAME_LIST_FLAG_DOESNT_HAVE_UUID;
 638			}
 639		}
 640		nl = uuid_ptr;
 641	}
 642	/*
 643	 * If mismatch happens, security may be compromised due to buffer
 644	 * overflow, so it's better to crash.
 645	 */
 646	BUG_ON((char *)nl - (char *)orig_nl != needed);
 647
 648 out:
 649	up_write(&_hash_lock);
 650	return 0;
 651}
 652
 653static void list_version_get_needed(struct target_type *tt, void *needed_param)
 654{
 655    size_t *needed = needed_param;
 656
 657    *needed += sizeof(struct dm_target_versions);
 658    *needed += strlen(tt->name) + 1;
 659    *needed += ALIGN_MASK;
 660}
 661
 662static void list_version_get_info(struct target_type *tt, void *param)
 663{
 664    struct vers_iter *info = param;
 665
 666    /* Check space - it might have changed since the first iteration */
 667    if ((char *)info->vers + sizeof(tt->version) + strlen(tt->name) + 1 >
 668	info->end) {
 669
 670	info->flags = DM_BUFFER_FULL_FLAG;
 671	return;
 672    }
 673
 674    if (info->old_vers)
 675	info->old_vers->next = (uint32_t) ((void *)info->vers -
 676					   (void *)info->old_vers);
 677    info->vers->version[0] = tt->version[0];
 678    info->vers->version[1] = tt->version[1];
 679    info->vers->version[2] = tt->version[2];
 680    info->vers->next = 0;
 681    strcpy(info->vers->name, tt->name);
 682
 683    info->old_vers = info->vers;
 684    info->vers = align_ptr((void *)(info->vers + 1) + strlen(tt->name) + 1);
 685}
 686
 687static int __list_versions(struct dm_ioctl *param, size_t param_size, const char *name)
 688{
 689	size_t len, needed = 0;
 690	struct dm_target_versions *vers;
 691	struct vers_iter iter_info;
 692	struct target_type *tt = NULL;
 693
 694	if (name) {
 695		tt = dm_get_target_type(name);
 696		if (!tt)
 697			return -EINVAL;
 698	}
 699
 700	/*
 701	 * Loop through all the devices working out how much
 702	 * space we need.
 703	 */
 704	if (!tt)
 705		dm_target_iterate(list_version_get_needed, &needed);
 706	else
 707		list_version_get_needed(tt, &needed);
 708
 709	/*
 710	 * Grab our output buffer.
 711	 */
 712	vers = get_result_buffer(param, param_size, &len);
 713	if (len < needed) {
 714		param->flags |= DM_BUFFER_FULL_FLAG;
 715		goto out;
 716	}
 717	param->data_size = param->data_start + needed;
 718
 719	iter_info.param_size = param_size;
 720	iter_info.old_vers = NULL;
 721	iter_info.vers = vers;
 722	iter_info.flags = 0;
 723	iter_info.end = (char *)vers + needed;
 724
 725	/*
 726	 * Now loop through filling out the names & versions.
 727	 */
 728	if (!tt)
 729		dm_target_iterate(list_version_get_info, &iter_info);
 730	else
 731		list_version_get_info(tt, &iter_info);
 732	param->flags |= iter_info.flags;
 733
 734 out:
 735	if (tt)
 736		dm_put_target_type(tt);
 737	return 0;
 738}
 739
 740static int list_versions(struct file *filp, struct dm_ioctl *param, size_t param_size)
 741{
 742	return __list_versions(param, param_size, NULL);
 743}
 744
 745static int get_target_version(struct file *filp, struct dm_ioctl *param, size_t param_size)
 746{
 747	return __list_versions(param, param_size, param->name);
 748}
 749
 750static int check_name(const char *name)
 751{
 752	if (strchr(name, '/')) {
 753		DMERR("invalid device name");
 754		return -EINVAL;
 755	}
 756
 757	return 0;
 758}
 759
 760/*
 761 * On successful return, the caller must not attempt to acquire
 762 * _hash_lock without first calling dm_put_live_table, because dm_table_destroy
 763 * waits for this dm_put_live_table and could be called under this lock.
 764 */
 765static struct dm_table *dm_get_inactive_table(struct mapped_device *md, int *srcu_idx)
 766{
 767	struct hash_cell *hc;
 768	struct dm_table *table = NULL;
 769
 770	/* increment rcu count, we don't care about the table pointer */
 771	dm_get_live_table(md, srcu_idx);
 772
 773	down_read(&_hash_lock);
 774	hc = dm_get_mdptr(md);
 775	if (!hc || hc->md != md) {
 776		DMERR("device has been removed from the dev hash table.");
 777		goto out;
 778	}
 779
 780	table = hc->new_map;
 781
 782out:
 783	up_read(&_hash_lock);
 784
 785	return table;
 786}
 787
 788static struct dm_table *dm_get_live_or_inactive_table(struct mapped_device *md,
 789						      struct dm_ioctl *param,
 790						      int *srcu_idx)
 791{
 792	return (param->flags & DM_QUERY_INACTIVE_TABLE_FLAG) ?
 793		dm_get_inactive_table(md, srcu_idx) : dm_get_live_table(md, srcu_idx);
 794}
 795
 796/*
 797 * Fills in a dm_ioctl structure, ready for sending back to
 798 * userland.
 799 */
 800static void __dev_status(struct mapped_device *md, struct dm_ioctl *param)
 801{
 802	struct gendisk *disk = dm_disk(md);
 803	struct dm_table *table;
 804	int srcu_idx;
 805
 806	param->flags &= ~(DM_SUSPEND_FLAG | DM_READONLY_FLAG |
 807			  DM_ACTIVE_PRESENT_FLAG | DM_INTERNAL_SUSPEND_FLAG);
 808
 809	if (dm_suspended_md(md))
 810		param->flags |= DM_SUSPEND_FLAG;
 811
 812	if (dm_suspended_internally_md(md))
 813		param->flags |= DM_INTERNAL_SUSPEND_FLAG;
 814
 815	if (dm_test_deferred_remove_flag(md))
 816		param->flags |= DM_DEFERRED_REMOVE;
 817
 818	param->dev = huge_encode_dev(disk_devt(disk));
 819
 820	/*
 821	 * Yes, this will be out of date by the time it gets back
 822	 * to userland, but it is still very useful for
 823	 * debugging.
 824	 */
 825	param->open_count = dm_open_count(md);
 826
 827	param->event_nr = dm_get_event_nr(md);
 828	param->target_count = 0;
 829
 830	table = dm_get_live_table(md, &srcu_idx);
 831	if (table) {
 832		if (!(param->flags & DM_QUERY_INACTIVE_TABLE_FLAG)) {
 833			if (get_disk_ro(disk))
 834				param->flags |= DM_READONLY_FLAG;
 835			param->target_count = table->num_targets;
 836		}
 837
 838		param->flags |= DM_ACTIVE_PRESENT_FLAG;
 839	}
 840	dm_put_live_table(md, srcu_idx);
 841
 842	if (param->flags & DM_QUERY_INACTIVE_TABLE_FLAG) {
 843		int srcu_idx;
 844		table = dm_get_inactive_table(md, &srcu_idx);
 845		if (table) {
 846			if (!(dm_table_get_mode(table) & FMODE_WRITE))
 847				param->flags |= DM_READONLY_FLAG;
 848			param->target_count = table->num_targets;
 849		}
 850		dm_put_live_table(md, srcu_idx);
 851	}
 852}
 853
 854static int dev_create(struct file *filp, struct dm_ioctl *param, size_t param_size)
 855{
 856	int r, m = DM_ANY_MINOR;
 857	struct mapped_device *md;
 858
 859	r = check_name(param->name);
 860	if (r)
 861		return r;
 862
 863	if (param->flags & DM_PERSISTENT_DEV_FLAG)
 864		m = MINOR(huge_decode_dev(param->dev));
 865
 866	r = dm_create(m, &md);
 867	if (r)
 868		return r;
 869
 870	r = dm_hash_insert(param->name, *param->uuid ? param->uuid : NULL, md);
 871	if (r) {
 872		dm_put(md);
 873		dm_destroy(md);
 874		return r;
 875	}
 876
 877	param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
 878
 879	__dev_status(md, param);
 880
 881	dm_put(md);
 882
 883	return 0;
 884}
 885
 886/*
 887 * Always use UUID for lookups if it's present, otherwise use name or dev.
 888 */
 889static struct hash_cell *__find_device_hash_cell(struct dm_ioctl *param)
 890{
 891	struct hash_cell *hc = NULL;
 892
 893	if (*param->uuid) {
 894		if (*param->name || param->dev) {
 895			DMERR("Invalid ioctl structure: uuid %s, name %s, dev %llx",
 896			      param->uuid, param->name, (unsigned long long)param->dev);
 897			return NULL;
 898		}
 899
 900		hc = __get_uuid_cell(param->uuid);
 901		if (!hc)
 902			return NULL;
 903	} else if (*param->name) {
 904		if (param->dev) {
 905			DMERR("Invalid ioctl structure: name %s, dev %llx",
 906			      param->name, (unsigned long long)param->dev);
 907			return NULL;
 908		}
 909
 910		hc = __get_name_cell(param->name);
 911		if (!hc)
 912			return NULL;
 913	} else if (param->dev) {
 914		hc = __get_dev_cell(param->dev);
 915		if (!hc)
 916			return NULL;
 917	} else
 918		return NULL;
 919
 920	/*
 921	 * Sneakily write in both the name and the uuid
 922	 * while we have the cell.
 923	 */
 924	strlcpy(param->name, hc->name, sizeof(param->name));
 925	if (hc->uuid)
 926		strlcpy(param->uuid, hc->uuid, sizeof(param->uuid));
 927	else
 928		param->uuid[0] = '\0';
 929
 930	if (hc->new_map)
 931		param->flags |= DM_INACTIVE_PRESENT_FLAG;
 932	else
 933		param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
 934
 935	return hc;
 936}
 937
 938static struct mapped_device *find_device(struct dm_ioctl *param)
 939{
 940	struct hash_cell *hc;
 941	struct mapped_device *md = NULL;
 942
 943	down_read(&_hash_lock);
 944	hc = __find_device_hash_cell(param);
 945	if (hc)
 946		md = hc->md;
 947	up_read(&_hash_lock);
 948
 949	return md;
 950}
 951
 952static int dev_remove(struct file *filp, struct dm_ioctl *param, size_t param_size)
 953{
 954	struct hash_cell *hc;
 955	struct mapped_device *md;
 956	int r;
 957	struct dm_table *t;
 958
 959	down_write(&_hash_lock);
 960	hc = __find_device_hash_cell(param);
 961
 962	if (!hc) {
 963		DMDEBUG_LIMIT("device doesn't appear to be in the dev hash table.");
 964		up_write(&_hash_lock);
 965		return -ENXIO;
 966	}
 967
 968	md = hc->md;
 969
 970	/*
 971	 * Ensure the device is not open and nothing further can open it.
 972	 */
 973	r = dm_lock_for_deletion(md, !!(param->flags & DM_DEFERRED_REMOVE), false);
 974	if (r) {
 975		if (r == -EBUSY && param->flags & DM_DEFERRED_REMOVE) {
 976			up_write(&_hash_lock);
 977			dm_put(md);
 978			return 0;
 979		}
 980		DMDEBUG_LIMIT("unable to remove open device %s", hc->name);
 981		up_write(&_hash_lock);
 982		dm_put(md);
 983		return r;
 984	}
 985
 986	t = __hash_remove(hc);
 987	up_write(&_hash_lock);
 988
 989	if (t) {
 990		dm_sync_table(md);
 991		dm_table_destroy(t);
 992	}
 993
 994	param->flags &= ~DM_DEFERRED_REMOVE;
 995
 996	dm_ima_measure_on_device_remove(md, false);
 997
 998	if (!dm_kobject_uevent(md, KOBJ_REMOVE, param->event_nr))
 999		param->flags |= DM_UEVENT_GENERATED_FLAG;
1000
1001	dm_put(md);
1002	dm_destroy(md);
1003	return 0;
1004}
1005
1006/*
1007 * Check a string doesn't overrun the chunk of
1008 * memory we copied from userland.
1009 */
1010static int invalid_str(char *str, void *end)
1011{
1012	while ((void *) str < end)
1013		if (!*str++)
1014			return 0;
1015
1016	return -EINVAL;
1017}
1018
1019static int dev_rename(struct file *filp, struct dm_ioctl *param, size_t param_size)
1020{
1021	int r;
1022	char *new_data = (char *) param + param->data_start;
1023	struct mapped_device *md;
1024	unsigned change_uuid = (param->flags & DM_UUID_FLAG) ? 1 : 0;
1025
1026	if (new_data < param->data ||
1027	    invalid_str(new_data, (void *) param + param_size) || !*new_data ||
1028	    strlen(new_data) > (change_uuid ? DM_UUID_LEN - 1 : DM_NAME_LEN - 1)) {
1029		DMERR("Invalid new mapped device name or uuid string supplied.");
1030		return -EINVAL;
1031	}
1032
1033	if (!change_uuid) {
1034		r = check_name(new_data);
1035		if (r)
1036			return r;
1037	}
1038
1039	md = dm_hash_rename(param, new_data);
1040	if (IS_ERR(md))
1041		return PTR_ERR(md);
1042
1043	__dev_status(md, param);
1044	dm_put(md);
1045
1046	return 0;
1047}
1048
1049static int dev_set_geometry(struct file *filp, struct dm_ioctl *param, size_t param_size)
1050{
1051	int r = -EINVAL, x;
1052	struct mapped_device *md;
1053	struct hd_geometry geometry;
1054	unsigned long indata[4];
1055	char *geostr = (char *) param + param->data_start;
1056	char dummy;
1057
1058	md = find_device(param);
1059	if (!md)
1060		return -ENXIO;
1061
1062	if (geostr < param->data ||
1063	    invalid_str(geostr, (void *) param + param_size)) {
1064		DMERR("Invalid geometry supplied.");
1065		goto out;
1066	}
1067
1068	x = sscanf(geostr, "%lu %lu %lu %lu%c", indata,
1069		   indata + 1, indata + 2, indata + 3, &dummy);
1070
1071	if (x != 4) {
1072		DMERR("Unable to interpret geometry settings.");
1073		goto out;
1074	}
1075
1076	if (indata[0] > 65535 || indata[1] > 255 ||
1077	    indata[2] > 255 || indata[3] > ULONG_MAX) {
1078		DMERR("Geometry exceeds range limits.");
1079		goto out;
1080	}
1081
1082	geometry.cylinders = indata[0];
1083	geometry.heads = indata[1];
1084	geometry.sectors = indata[2];
1085	geometry.start = indata[3];
1086
1087	r = dm_set_geometry(md, &geometry);
1088
1089	param->data_size = 0;
1090
1091out:
1092	dm_put(md);
1093	return r;
1094}
1095
1096static int do_suspend(struct dm_ioctl *param)
1097{
1098	int r = 0;
1099	unsigned suspend_flags = DM_SUSPEND_LOCKFS_FLAG;
1100	struct mapped_device *md;
1101
1102	md = find_device(param);
1103	if (!md)
1104		return -ENXIO;
1105
1106	if (param->flags & DM_SKIP_LOCKFS_FLAG)
1107		suspend_flags &= ~DM_SUSPEND_LOCKFS_FLAG;
1108	if (param->flags & DM_NOFLUSH_FLAG)
1109		suspend_flags |= DM_SUSPEND_NOFLUSH_FLAG;
1110
1111	if (!dm_suspended_md(md)) {
1112		r = dm_suspend(md, suspend_flags);
1113		if (r)
1114			goto out;
1115	}
1116
1117	__dev_status(md, param);
1118
1119out:
1120	dm_put(md);
1121
1122	return r;
1123}
1124
1125static int do_resume(struct dm_ioctl *param)
1126{
1127	int r = 0;
1128	unsigned suspend_flags = DM_SUSPEND_LOCKFS_FLAG;
1129	struct hash_cell *hc;
1130	struct mapped_device *md;
1131	struct dm_table *new_map, *old_map = NULL;
1132
1133	down_write(&_hash_lock);
1134
1135	hc = __find_device_hash_cell(param);
1136	if (!hc) {
1137		DMDEBUG_LIMIT("device doesn't appear to be in the dev hash table.");
1138		up_write(&_hash_lock);
1139		return -ENXIO;
1140	}
1141
1142	md = hc->md;
1143
1144	new_map = hc->new_map;
1145	hc->new_map = NULL;
1146	param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
1147
1148	up_write(&_hash_lock);
1149
1150	/* Do we need to load a new map ? */
1151	if (new_map) {
1152		/* Suspend if it isn't already suspended */
1153		if (param->flags & DM_SKIP_LOCKFS_FLAG)
1154			suspend_flags &= ~DM_SUSPEND_LOCKFS_FLAG;
1155		if (param->flags & DM_NOFLUSH_FLAG)
1156			suspend_flags |= DM_SUSPEND_NOFLUSH_FLAG;
1157		if (!dm_suspended_md(md))
1158			dm_suspend(md, suspend_flags);
1159
1160		old_map = dm_swap_table(md, new_map);
1161		if (IS_ERR(old_map)) {
1162			dm_sync_table(md);
1163			dm_table_destroy(new_map);
1164			dm_put(md);
1165			return PTR_ERR(old_map);
1166		}
1167
1168		if (dm_table_get_mode(new_map) & FMODE_WRITE)
1169			set_disk_ro(dm_disk(md), 0);
1170		else
1171			set_disk_ro(dm_disk(md), 1);
1172	}
1173
1174	if (dm_suspended_md(md)) {
1175		r = dm_resume(md);
1176		if (!r) {
1177			dm_ima_measure_on_device_resume(md, new_map ? true : false);
1178
1179			if (!dm_kobject_uevent(md, KOBJ_CHANGE, param->event_nr))
1180				param->flags |= DM_UEVENT_GENERATED_FLAG;
1181		}
1182	}
1183
1184	/*
1185	 * Since dm_swap_table synchronizes RCU, nobody should be in
1186	 * read-side critical section already.
1187	 */
1188	if (old_map)
1189		dm_table_destroy(old_map);
1190
1191	if (!r)
1192		__dev_status(md, param);
1193
1194	dm_put(md);
1195	return r;
1196}
1197
1198/*
1199 * Set or unset the suspension state of a device.
1200 * If the device already is in the requested state we just return its status.
1201 */
1202static int dev_suspend(struct file *filp, struct dm_ioctl *param, size_t param_size)
1203{
1204	if (param->flags & DM_SUSPEND_FLAG)
1205		return do_suspend(param);
1206
1207	return do_resume(param);
1208}
1209
1210/*
1211 * Copies device info back to user space, used by
1212 * the create and info ioctls.
1213 */
1214static int dev_status(struct file *filp, struct dm_ioctl *param, size_t param_size)
1215{
1216	struct mapped_device *md;
1217
1218	md = find_device(param);
1219	if (!md)
1220		return -ENXIO;
1221
1222	__dev_status(md, param);
1223	dm_put(md);
1224
1225	return 0;
1226}
1227
1228/*
1229 * Build up the status struct for each target
1230 */
1231static void retrieve_status(struct dm_table *table,
1232			    struct dm_ioctl *param, size_t param_size)
1233{
1234	unsigned int i, num_targets;
1235	struct dm_target_spec *spec;
1236	char *outbuf, *outptr;
1237	status_type_t type;
1238	size_t remaining, len, used = 0;
1239	unsigned status_flags = 0;
1240
1241	outptr = outbuf = get_result_buffer(param, param_size, &len);
1242
1243	if (param->flags & DM_STATUS_TABLE_FLAG)
1244		type = STATUSTYPE_TABLE;
1245	else if (param->flags & DM_IMA_MEASUREMENT_FLAG)
1246		type = STATUSTYPE_IMA;
1247	else
1248		type = STATUSTYPE_INFO;
1249
1250	/* Get all the target info */
1251	num_targets = table->num_targets;
1252	for (i = 0; i < num_targets; i++) {
1253		struct dm_target *ti = dm_table_get_target(table, i);
1254		size_t l;
1255
1256		remaining = len - (outptr - outbuf);
1257		if (remaining <= sizeof(struct dm_target_spec)) {
1258			param->flags |= DM_BUFFER_FULL_FLAG;
1259			break;
1260		}
1261
1262		spec = (struct dm_target_spec *) outptr;
1263
1264		spec->status = 0;
1265		spec->sector_start = ti->begin;
1266		spec->length = ti->len;
1267		strncpy(spec->target_type, ti->type->name,
1268			sizeof(spec->target_type) - 1);
1269
1270		outptr += sizeof(struct dm_target_spec);
1271		remaining = len - (outptr - outbuf);
1272		if (remaining <= 0) {
1273			param->flags |= DM_BUFFER_FULL_FLAG;
1274			break;
1275		}
1276
1277		/* Get the status/table string from the target driver */
1278		if (ti->type->status) {
1279			if (param->flags & DM_NOFLUSH_FLAG)
1280				status_flags |= DM_STATUS_NOFLUSH_FLAG;
1281			ti->type->status(ti, type, status_flags, outptr, remaining);
1282		} else
1283			outptr[0] = '\0';
1284
1285		l = strlen(outptr) + 1;
1286		if (l == remaining) {
1287			param->flags |= DM_BUFFER_FULL_FLAG;
1288			break;
1289		}
1290
1291		outptr += l;
1292		used = param->data_start + (outptr - outbuf);
1293
1294		outptr = align_ptr(outptr);
1295		spec->next = outptr - outbuf;
1296	}
1297
1298	if (used)
1299		param->data_size = used;
1300
1301	param->target_count = num_targets;
1302}
1303
1304/*
1305 * Wait for a device to report an event
1306 */
1307static int dev_wait(struct file *filp, struct dm_ioctl *param, size_t param_size)
1308{
1309	int r = 0;
1310	struct mapped_device *md;
1311	struct dm_table *table;
1312	int srcu_idx;
1313
1314	md = find_device(param);
1315	if (!md)
1316		return -ENXIO;
1317
1318	/*
1319	 * Wait for a notification event
1320	 */
1321	if (dm_wait_event(md, param->event_nr)) {
1322		r = -ERESTARTSYS;
1323		goto out;
1324	}
1325
1326	/*
1327	 * The userland program is going to want to know what
1328	 * changed to trigger the event, so we may as well tell
1329	 * him and save an ioctl.
1330	 */
1331	__dev_status(md, param);
1332
1333	table = dm_get_live_or_inactive_table(md, param, &srcu_idx);
1334	if (table)
1335		retrieve_status(table, param, param_size);
1336	dm_put_live_table(md, srcu_idx);
1337
1338out:
1339	dm_put(md);
1340
1341	return r;
1342}
1343
1344/*
1345 * Remember the global event number and make it possible to poll
1346 * for further events.
1347 */
1348static int dev_arm_poll(struct file *filp, struct dm_ioctl *param, size_t param_size)
1349{
1350	struct dm_file *priv = filp->private_data;
1351
1352	priv->global_event_nr = atomic_read(&dm_global_event_nr);
1353
1354	return 0;
1355}
1356
1357static inline fmode_t get_mode(struct dm_ioctl *param)
1358{
1359	fmode_t mode = FMODE_READ | FMODE_WRITE;
1360
1361	if (param->flags & DM_READONLY_FLAG)
1362		mode = FMODE_READ;
1363
1364	return mode;
1365}
1366
1367static int next_target(struct dm_target_spec *last, uint32_t next, void *end,
1368		       struct dm_target_spec **spec, char **target_params)
1369{
1370	*spec = (struct dm_target_spec *) ((unsigned char *) last + next);
1371	*target_params = (char *) (*spec + 1);
1372
1373	if (*spec < (last + 1))
1374		return -EINVAL;
1375
1376	return invalid_str(*target_params, end);
1377}
1378
1379static int populate_table(struct dm_table *table,
1380			  struct dm_ioctl *param, size_t param_size)
1381{
1382	int r;
1383	unsigned int i = 0;
1384	struct dm_target_spec *spec = (struct dm_target_spec *) param;
1385	uint32_t next = param->data_start;
1386	void *end = (void *) param + param_size;
1387	char *target_params;
1388
1389	if (!param->target_count) {
1390		DMERR("populate_table: no targets specified");
1391		return -EINVAL;
1392	}
1393
1394	for (i = 0; i < param->target_count; i++) {
1395
1396		r = next_target(spec, next, end, &spec, &target_params);
1397		if (r) {
1398			DMERR("unable to find target");
1399			return r;
1400		}
1401
1402		r = dm_table_add_target(table, spec->target_type,
1403					(sector_t) spec->sector_start,
1404					(sector_t) spec->length,
1405					target_params);
1406		if (r) {
1407			DMERR("error adding target to table");
1408			return r;
1409		}
1410
1411		next = spec->next;
1412	}
1413
1414	return dm_table_complete(table);
1415}
1416
1417static bool is_valid_type(enum dm_queue_mode cur, enum dm_queue_mode new)
1418{
1419	if (cur == new ||
1420	    (cur == DM_TYPE_BIO_BASED && new == DM_TYPE_DAX_BIO_BASED))
1421		return true;
1422
1423	return false;
1424}
1425
1426static int table_load(struct file *filp, struct dm_ioctl *param, size_t param_size)
1427{
1428	int r;
1429	struct hash_cell *hc;
1430	struct dm_table *t, *old_map = NULL;
1431	struct mapped_device *md;
1432	struct target_type *immutable_target_type;
1433
1434	md = find_device(param);
1435	if (!md)
1436		return -ENXIO;
1437
1438	r = dm_table_create(&t, get_mode(param), param->target_count, md);
1439	if (r)
1440		goto err;
1441
1442	/* Protect md->type and md->queue against concurrent table loads. */
1443	dm_lock_md_type(md);
1444	r = populate_table(t, param, param_size);
1445	if (r)
1446		goto err_unlock_md_type;
1447
1448	dm_ima_measure_on_table_load(t, STATUSTYPE_IMA);
1449
1450	immutable_target_type = dm_get_immutable_target_type(md);
1451	if (immutable_target_type &&
1452	    (immutable_target_type != dm_table_get_immutable_target_type(t)) &&
1453	    !dm_table_get_wildcard_target(t)) {
1454		DMERR("can't replace immutable target type %s",
1455		      immutable_target_type->name);
1456		r = -EINVAL;
1457		goto err_unlock_md_type;
1458	}
1459
1460	if (dm_get_md_type(md) == DM_TYPE_NONE) {
 
 
 
1461		/* setup md->queue to reflect md's type (may block) */
1462		r = dm_setup_md_queue(md, t);
1463		if (r) {
1464			DMERR("unable to set up device queue for new table.");
1465			goto err_unlock_md_type;
1466		}
1467	} else if (!is_valid_type(dm_get_md_type(md), dm_table_get_type(t))) {
1468		DMERR("can't change device type (old=%u vs new=%u) after initial table load.",
1469		      dm_get_md_type(md), dm_table_get_type(t));
1470		r = -EINVAL;
1471		goto err_unlock_md_type;
1472	}
1473
1474	dm_unlock_md_type(md);
1475
1476	/* stage inactive table */
1477	down_write(&_hash_lock);
1478	hc = dm_get_mdptr(md);
1479	if (!hc || hc->md != md) {
1480		DMERR("device has been removed from the dev hash table.");
1481		up_write(&_hash_lock);
1482		r = -ENXIO;
1483		goto err_destroy_table;
1484	}
1485
1486	if (hc->new_map)
1487		old_map = hc->new_map;
1488	hc->new_map = t;
1489	up_write(&_hash_lock);
1490
1491	param->flags |= DM_INACTIVE_PRESENT_FLAG;
1492	__dev_status(md, param);
1493
1494	if (old_map) {
1495		dm_sync_table(md);
1496		dm_table_destroy(old_map);
1497	}
1498
1499	dm_put(md);
1500
1501	return 0;
1502
1503err_unlock_md_type:
1504	dm_unlock_md_type(md);
1505err_destroy_table:
1506	dm_table_destroy(t);
1507err:
1508	dm_put(md);
1509
1510	return r;
1511}
1512
1513static int table_clear(struct file *filp, struct dm_ioctl *param, size_t param_size)
1514{
1515	struct hash_cell *hc;
1516	struct mapped_device *md;
1517	struct dm_table *old_map = NULL;
1518	bool has_new_map = false;
1519
1520	down_write(&_hash_lock);
1521
1522	hc = __find_device_hash_cell(param);
1523	if (!hc) {
1524		DMDEBUG_LIMIT("device doesn't appear to be in the dev hash table.");
1525		up_write(&_hash_lock);
1526		return -ENXIO;
1527	}
1528
1529	if (hc->new_map) {
1530		old_map = hc->new_map;
1531		hc->new_map = NULL;
1532		has_new_map = true;
1533	}
1534
1535	param->flags &= ~DM_INACTIVE_PRESENT_FLAG;
1536
1537	__dev_status(hc->md, param);
1538	md = hc->md;
1539	up_write(&_hash_lock);
1540	if (old_map) {
1541		dm_sync_table(md);
1542		dm_table_destroy(old_map);
1543	}
1544	dm_ima_measure_on_table_clear(md, has_new_map);
1545	dm_put(md);
1546
1547	return 0;
1548}
1549
1550/*
1551 * Retrieves a list of devices used by a particular dm device.
1552 */
1553static void retrieve_deps(struct dm_table *table,
1554			  struct dm_ioctl *param, size_t param_size)
1555{
1556	unsigned int count = 0;
1557	struct list_head *tmp;
1558	size_t len, needed;
1559	struct dm_dev_internal *dd;
1560	struct dm_target_deps *deps;
1561
1562	deps = get_result_buffer(param, param_size, &len);
1563
1564	/*
1565	 * Count the devices.
1566	 */
1567	list_for_each (tmp, dm_table_get_devices(table))
1568		count++;
1569
1570	/*
1571	 * Check we have enough space.
1572	 */
1573	needed = struct_size(deps, dev, count);
1574	if (len < needed) {
1575		param->flags |= DM_BUFFER_FULL_FLAG;
1576		return;
1577	}
1578
1579	/*
1580	 * Fill in the devices.
1581	 */
1582	deps->count = count;
1583	count = 0;
1584	list_for_each_entry (dd, dm_table_get_devices(table), list)
1585		deps->dev[count++] = huge_encode_dev(dd->dm_dev->bdev->bd_dev);
1586
1587	param->data_size = param->data_start + needed;
1588}
1589
1590static int table_deps(struct file *filp, struct dm_ioctl *param, size_t param_size)
1591{
1592	struct mapped_device *md;
1593	struct dm_table *table;
1594	int srcu_idx;
1595
1596	md = find_device(param);
1597	if (!md)
1598		return -ENXIO;
1599
1600	__dev_status(md, param);
1601
1602	table = dm_get_live_or_inactive_table(md, param, &srcu_idx);
1603	if (table)
1604		retrieve_deps(table, param, param_size);
1605	dm_put_live_table(md, srcu_idx);
1606
1607	dm_put(md);
1608
1609	return 0;
1610}
1611
1612/*
1613 * Return the status of a device as a text string for each
1614 * target.
1615 */
1616static int table_status(struct file *filp, struct dm_ioctl *param, size_t param_size)
1617{
1618	struct mapped_device *md;
1619	struct dm_table *table;
1620	int srcu_idx;
1621
1622	md = find_device(param);
1623	if (!md)
1624		return -ENXIO;
1625
1626	__dev_status(md, param);
1627
1628	table = dm_get_live_or_inactive_table(md, param, &srcu_idx);
1629	if (table)
1630		retrieve_status(table, param, param_size);
1631	dm_put_live_table(md, srcu_idx);
1632
1633	dm_put(md);
1634
1635	return 0;
1636}
1637
1638/*
1639 * Process device-mapper dependent messages.  Messages prefixed with '@'
1640 * are processed by the DM core.  All others are delivered to the target.
1641 * Returns a number <= 1 if message was processed by device mapper.
1642 * Returns 2 if message should be delivered to the target.
1643 */
1644static int message_for_md(struct mapped_device *md, unsigned argc, char **argv,
1645			  char *result, unsigned maxlen)
1646{
1647	int r;
1648
1649	if (**argv != '@')
1650		return 2; /* no '@' prefix, deliver to target */
1651
1652	if (!strcasecmp(argv[0], "@cancel_deferred_remove")) {
1653		if (argc != 1) {
1654			DMERR("Invalid arguments for @cancel_deferred_remove");
1655			return -EINVAL;
1656		}
1657		return dm_cancel_deferred_remove(md);
1658	}
1659
1660	r = dm_stats_message(md, argc, argv, result, maxlen);
1661	if (r < 2)
1662		return r;
1663
1664	DMERR("Unsupported message sent to DM core: %s", argv[0]);
1665	return -EINVAL;
1666}
1667
1668/*
1669 * Pass a message to the target that's at the supplied device offset.
1670 */
1671static int target_message(struct file *filp, struct dm_ioctl *param, size_t param_size)
1672{
1673	int r, argc;
1674	char **argv;
1675	struct mapped_device *md;
1676	struct dm_table *table;
1677	struct dm_target *ti;
1678	struct dm_target_msg *tmsg = (void *) param + param->data_start;
1679	size_t maxlen;
1680	char *result = get_result_buffer(param, param_size, &maxlen);
1681	int srcu_idx;
1682
1683	md = find_device(param);
1684	if (!md)
1685		return -ENXIO;
1686
1687	if (tmsg < (struct dm_target_msg *) param->data ||
1688	    invalid_str(tmsg->message, (void *) param + param_size)) {
1689		DMERR("Invalid target message parameters.");
1690		r = -EINVAL;
1691		goto out;
1692	}
1693
1694	r = dm_split_args(&argc, &argv, tmsg->message);
1695	if (r) {
1696		DMERR("Failed to split target message parameters");
1697		goto out;
1698	}
1699
1700	if (!argc) {
1701		DMERR("Empty message received.");
1702		r = -EINVAL;
1703		goto out_argv;
1704	}
1705
1706	r = message_for_md(md, argc, argv, result, maxlen);
1707	if (r <= 1)
1708		goto out_argv;
1709
1710	table = dm_get_live_table(md, &srcu_idx);
1711	if (!table)
1712		goto out_table;
1713
1714	if (dm_deleting_md(md)) {
1715		r = -ENXIO;
1716		goto out_table;
1717	}
1718
1719	ti = dm_table_find_target(table, tmsg->sector);
1720	if (!ti) {
1721		DMERR("Target message sector outside device.");
1722		r = -EINVAL;
1723	} else if (ti->type->message)
1724		r = ti->type->message(ti, argc, argv, result, maxlen);
1725	else {
1726		DMERR("Target type does not support messages");
1727		r = -EINVAL;
1728	}
1729
1730 out_table:
1731	dm_put_live_table(md, srcu_idx);
1732 out_argv:
1733	kfree(argv);
1734 out:
1735	if (r >= 0)
1736		__dev_status(md, param);
1737
1738	if (r == 1) {
1739		param->flags |= DM_DATA_OUT_FLAG;
1740		if (dm_message_test_buffer_overflow(result, maxlen))
1741			param->flags |= DM_BUFFER_FULL_FLAG;
1742		else
1743			param->data_size = param->data_start + strlen(result) + 1;
1744		r = 0;
1745	}
1746
1747	dm_put(md);
1748	return r;
1749}
1750
1751/*
1752 * The ioctl parameter block consists of two parts, a dm_ioctl struct
1753 * followed by a data buffer.  This flag is set if the second part,
1754 * which has a variable size, is not used by the function processing
1755 * the ioctl.
1756 */
1757#define IOCTL_FLAGS_NO_PARAMS		1
1758#define IOCTL_FLAGS_ISSUE_GLOBAL_EVENT	2
1759
1760/*-----------------------------------------------------------------
1761 * Implementation of open/close/ioctl on the special char
1762 * device.
1763 *---------------------------------------------------------------*/
1764static ioctl_fn lookup_ioctl(unsigned int cmd, int *ioctl_flags)
1765{
1766	static const struct {
1767		int cmd;
1768		int flags;
1769		ioctl_fn fn;
1770	} _ioctls[] = {
1771		{DM_VERSION_CMD, 0, NULL}, /* version is dealt with elsewhere */
1772		{DM_REMOVE_ALL_CMD, IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, remove_all},
1773		{DM_LIST_DEVICES_CMD, 0, list_devices},
1774
1775		{DM_DEV_CREATE_CMD, IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_create},
1776		{DM_DEV_REMOVE_CMD, IOCTL_FLAGS_NO_PARAMS | IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_remove},
1777		{DM_DEV_RENAME_CMD, IOCTL_FLAGS_ISSUE_GLOBAL_EVENT, dev_rename},
1778		{DM_DEV_SUSPEND_CMD, IOCTL_FLAGS_NO_PARAMS, dev_suspend},
1779		{DM_DEV_STATUS_CMD, IOCTL_FLAGS_NO_PARAMS, dev_status},
1780		{DM_DEV_WAIT_CMD, 0, dev_wait},
1781
1782		{DM_TABLE_LOAD_CMD, 0, table_load},
1783		{DM_TABLE_CLEAR_CMD, IOCTL_FLAGS_NO_PARAMS, table_clear},
1784		{DM_TABLE_DEPS_CMD, 0, table_deps},
1785		{DM_TABLE_STATUS_CMD, 0, table_status},
1786
1787		{DM_LIST_VERSIONS_CMD, 0, list_versions},
1788
1789		{DM_TARGET_MSG_CMD, 0, target_message},
1790		{DM_DEV_SET_GEOMETRY_CMD, 0, dev_set_geometry},
1791		{DM_DEV_ARM_POLL_CMD, IOCTL_FLAGS_NO_PARAMS, dev_arm_poll},
1792		{DM_GET_TARGET_VERSION_CMD, 0, get_target_version},
1793	};
1794
1795	if (unlikely(cmd >= ARRAY_SIZE(_ioctls)))
1796		return NULL;
1797
1798	cmd = array_index_nospec(cmd, ARRAY_SIZE(_ioctls));
1799	*ioctl_flags = _ioctls[cmd].flags;
1800	return _ioctls[cmd].fn;
1801}
1802
1803/*
1804 * As well as checking the version compatibility this always
1805 * copies the kernel interface version out.
1806 */
1807static int check_version(unsigned int cmd, struct dm_ioctl __user *user)
1808{
1809	uint32_t version[3];
1810	int r = 0;
1811
1812	if (copy_from_user(version, user->version, sizeof(version)))
1813		return -EFAULT;
1814
1815	if ((DM_VERSION_MAJOR != version[0]) ||
1816	    (DM_VERSION_MINOR < version[1])) {
1817		DMERR("ioctl interface mismatch: "
1818		      "kernel(%u.%u.%u), user(%u.%u.%u), cmd(%d)",
1819		      DM_VERSION_MAJOR, DM_VERSION_MINOR,
1820		      DM_VERSION_PATCHLEVEL,
1821		      version[0], version[1], version[2], cmd);
1822		r = -EINVAL;
1823	}
1824
1825	/*
1826	 * Fill in the kernel version.
1827	 */
1828	version[0] = DM_VERSION_MAJOR;
1829	version[1] = DM_VERSION_MINOR;
1830	version[2] = DM_VERSION_PATCHLEVEL;
1831	if (copy_to_user(user->version, version, sizeof(version)))
1832		return -EFAULT;
1833
1834	return r;
1835}
1836
1837#define DM_PARAMS_MALLOC	0x0001	/* Params allocated with kvmalloc() */
1838#define DM_WIPE_BUFFER		0x0010	/* Wipe input buffer before returning from ioctl */
1839
1840static void free_params(struct dm_ioctl *param, size_t param_size, int param_flags)
1841{
1842	if (param_flags & DM_WIPE_BUFFER)
1843		memset(param, 0, param_size);
1844
1845	if (param_flags & DM_PARAMS_MALLOC)
1846		kvfree(param);
1847}
1848
1849static int copy_params(struct dm_ioctl __user *user, struct dm_ioctl *param_kernel,
1850		       int ioctl_flags, struct dm_ioctl **param, int *param_flags)
1851{
1852	struct dm_ioctl *dmi;
1853	int secure_data;
1854	const size_t minimum_data_size = offsetof(struct dm_ioctl, data);
1855	unsigned noio_flag;
1856
1857	if (copy_from_user(param_kernel, user, minimum_data_size))
1858		return -EFAULT;
1859
1860	if (param_kernel->data_size < minimum_data_size) {
1861		DMERR("Invalid data size in the ioctl structure: %u",
1862		      param_kernel->data_size);
1863		return -EINVAL;
1864	}
1865
1866	secure_data = param_kernel->flags & DM_SECURE_DATA_FLAG;
1867
1868	*param_flags = secure_data ? DM_WIPE_BUFFER : 0;
1869
1870	if (ioctl_flags & IOCTL_FLAGS_NO_PARAMS) {
1871		dmi = param_kernel;
1872		dmi->data_size = minimum_data_size;
1873		goto data_copied;
1874	}
1875
1876	/*
1877	 * Use __GFP_HIGH to avoid low memory issues when a device is
1878	 * suspended and the ioctl is needed to resume it.
1879	 * Use kmalloc() rather than vmalloc() when we can.
1880	 */
1881	dmi = NULL;
1882	noio_flag = memalloc_noio_save();
1883	dmi = kvmalloc(param_kernel->data_size, GFP_KERNEL | __GFP_HIGH);
1884	memalloc_noio_restore(noio_flag);
1885
1886	if (!dmi) {
1887		if (secure_data && clear_user(user, param_kernel->data_size))
1888			return -EFAULT;
1889		return -ENOMEM;
1890	}
1891
1892	*param_flags |= DM_PARAMS_MALLOC;
1893
1894	/* Copy from param_kernel (which was already copied from user) */
1895	memcpy(dmi, param_kernel, minimum_data_size);
1896
1897	if (copy_from_user(&dmi->data, (char __user *)user + minimum_data_size,
1898			   param_kernel->data_size - minimum_data_size))
1899		goto bad;
1900data_copied:
1901	/* Wipe the user buffer so we do not return it to userspace */
1902	if (secure_data && clear_user(user, param_kernel->data_size))
1903		goto bad;
1904
1905	*param = dmi;
1906	return 0;
1907
1908bad:
1909	free_params(dmi, param_kernel->data_size, *param_flags);
1910
1911	return -EFAULT;
1912}
1913
1914static int validate_params(uint cmd, struct dm_ioctl *param)
1915{
1916	/* Always clear this flag */
1917	param->flags &= ~DM_BUFFER_FULL_FLAG;
1918	param->flags &= ~DM_UEVENT_GENERATED_FLAG;
1919	param->flags &= ~DM_SECURE_DATA_FLAG;
1920	param->flags &= ~DM_DATA_OUT_FLAG;
1921
1922	/* Ignores parameters */
1923	if (cmd == DM_REMOVE_ALL_CMD ||
1924	    cmd == DM_LIST_DEVICES_CMD ||
1925	    cmd == DM_LIST_VERSIONS_CMD)
1926		return 0;
1927
1928	if (cmd == DM_DEV_CREATE_CMD) {
1929		if (!*param->name) {
1930			DMERR("name not supplied when creating device");
1931			return -EINVAL;
1932		}
1933	} else if (*param->uuid && *param->name) {
1934		DMERR("only supply one of name or uuid, cmd(%u)", cmd);
1935		return -EINVAL;
1936	}
1937
1938	/* Ensure strings are terminated */
1939	param->name[DM_NAME_LEN - 1] = '\0';
1940	param->uuid[DM_UUID_LEN - 1] = '\0';
1941
1942	return 0;
1943}
1944
1945static int ctl_ioctl(struct file *file, uint command, struct dm_ioctl __user *user)
1946{
1947	int r = 0;
1948	int ioctl_flags;
1949	int param_flags;
1950	unsigned int cmd;
1951	struct dm_ioctl *param;
1952	ioctl_fn fn = NULL;
1953	size_t input_param_size;
1954	struct dm_ioctl param_kernel;
1955
1956	/* only root can play with this */
1957	if (!capable(CAP_SYS_ADMIN))
1958		return -EACCES;
1959
1960	if (_IOC_TYPE(command) != DM_IOCTL)
1961		return -ENOTTY;
1962
1963	cmd = _IOC_NR(command);
1964
1965	/*
1966	 * Check the interface version passed in.  This also
1967	 * writes out the kernel's interface version.
1968	 */
1969	r = check_version(cmd, user);
1970	if (r)
1971		return r;
1972
1973	/*
1974	 * Nothing more to do for the version command.
1975	 */
1976	if (cmd == DM_VERSION_CMD)
1977		return 0;
1978
1979	fn = lookup_ioctl(cmd, &ioctl_flags);
1980	if (!fn) {
1981		DMERR("dm_ctl_ioctl: unknown command 0x%x", command);
1982		return -ENOTTY;
1983	}
1984
1985	/*
1986	 * Copy the parameters into kernel space.
1987	 */
1988	r = copy_params(user, &param_kernel, ioctl_flags, &param, &param_flags);
1989
1990	if (r)
1991		return r;
1992
1993	input_param_size = param->data_size;
1994	r = validate_params(cmd, param);
1995	if (r)
1996		goto out;
1997
1998	param->data_size = offsetof(struct dm_ioctl, data);
1999	r = fn(file, param, input_param_size);
2000
2001	if (unlikely(param->flags & DM_BUFFER_FULL_FLAG) &&
2002	    unlikely(ioctl_flags & IOCTL_FLAGS_NO_PARAMS))
2003		DMERR("ioctl %d tried to output some data but has IOCTL_FLAGS_NO_PARAMS set", cmd);
2004
2005	if (!r && ioctl_flags & IOCTL_FLAGS_ISSUE_GLOBAL_EVENT)
2006		dm_issue_global_event();
2007
2008	/*
2009	 * Copy the results back to userland.
2010	 */
2011	if (!r && copy_to_user(user, param, param->data_size))
2012		r = -EFAULT;
2013
2014out:
2015	free_params(param, input_param_size, param_flags);
2016	return r;
2017}
2018
2019static long dm_ctl_ioctl(struct file *file, uint command, ulong u)
2020{
2021	return (long)ctl_ioctl(file, command, (struct dm_ioctl __user *)u);
2022}
2023
2024#ifdef CONFIG_COMPAT
2025static long dm_compat_ctl_ioctl(struct file *file, uint command, ulong u)
2026{
2027	return (long)dm_ctl_ioctl(file, command, (ulong) compat_ptr(u));
2028}
2029#else
2030#define dm_compat_ctl_ioctl NULL
2031#endif
2032
2033static int dm_open(struct inode *inode, struct file *filp)
2034{
2035	int r;
2036	struct dm_file *priv;
2037
2038	r = nonseekable_open(inode, filp);
2039	if (unlikely(r))
2040		return r;
2041
2042	priv = filp->private_data = kmalloc(sizeof(struct dm_file), GFP_KERNEL);
2043	if (!priv)
2044		return -ENOMEM;
2045
2046	priv->global_event_nr = atomic_read(&dm_global_event_nr);
2047
2048	return 0;
2049}
2050
2051static int dm_release(struct inode *inode, struct file *filp)
2052{
2053	kfree(filp->private_data);
2054	return 0;
2055}
2056
2057static __poll_t dm_poll(struct file *filp, poll_table *wait)
2058{
2059	struct dm_file *priv = filp->private_data;
2060	__poll_t mask = 0;
2061
2062	poll_wait(filp, &dm_global_eventq, wait);
2063
2064	if ((int)(atomic_read(&dm_global_event_nr) - priv->global_event_nr) > 0)
2065		mask |= EPOLLIN;
2066
2067	return mask;
2068}
2069
2070static const struct file_operations _ctl_fops = {
2071	.open    = dm_open,
2072	.release = dm_release,
2073	.poll    = dm_poll,
2074	.unlocked_ioctl	 = dm_ctl_ioctl,
2075	.compat_ioctl = dm_compat_ctl_ioctl,
2076	.owner	 = THIS_MODULE,
2077	.llseek  = noop_llseek,
2078};
2079
2080static struct miscdevice _dm_misc = {
2081	.minor		= MAPPER_CTRL_MINOR,
2082	.name  		= DM_NAME,
2083	.nodename	= DM_DIR "/" DM_CONTROL_NODE,
2084	.fops  		= &_ctl_fops
2085};
2086
2087MODULE_ALIAS_MISCDEV(MAPPER_CTRL_MINOR);
2088MODULE_ALIAS("devname:" DM_DIR "/" DM_CONTROL_NODE);
2089
2090/*
2091 * Create misc character device and link to DM_DIR/control.
2092 */
2093int __init dm_interface_init(void)
2094{
2095	int r;
2096
2097	r = misc_register(&_dm_misc);
2098	if (r) {
2099		DMERR("misc_register failed for control device");
2100		return r;
2101	}
2102
2103	DMINFO("%d.%d.%d%s initialised: %s", DM_VERSION_MAJOR,
2104	       DM_VERSION_MINOR, DM_VERSION_PATCHLEVEL, DM_VERSION_EXTRA,
2105	       DM_DRIVER_EMAIL);
2106	return 0;
2107}
2108
2109void dm_interface_exit(void)
2110{
2111	misc_deregister(&_dm_misc);
2112	dm_hash_exit();
2113}
2114
2115/**
2116 * dm_copy_name_and_uuid - Copy mapped device name & uuid into supplied buffers
2117 * @md: Pointer to mapped_device
2118 * @name: Buffer (size DM_NAME_LEN) for name
2119 * @uuid: Buffer (size DM_UUID_LEN) for uuid or empty string if uuid not defined
2120 */
2121int dm_copy_name_and_uuid(struct mapped_device *md, char *name, char *uuid)
2122{
2123	int r = 0;
2124	struct hash_cell *hc;
2125
2126	if (!md)
2127		return -ENXIO;
2128
2129	mutex_lock(&dm_hash_cells_mutex);
2130	hc = dm_get_mdptr(md);
2131	if (!hc || hc->md != md) {
2132		r = -ENXIO;
2133		goto out;
2134	}
2135
2136	if (name)
2137		strcpy(name, hc->name);
2138	if (uuid)
2139		strcpy(uuid, hc->uuid ? : "");
2140
2141out:
2142	mutex_unlock(&dm_hash_cells_mutex);
2143
2144	return r;
2145}
2146EXPORT_SYMBOL_GPL(dm_copy_name_and_uuid);
2147
2148/**
2149 * dm_early_create - create a mapped device in early boot.
2150 *
2151 * @dmi: Contains main information of the device mapping to be created.
2152 * @spec_array: array of pointers to struct dm_target_spec. Describes the
2153 * mapping table of the device.
2154 * @target_params_array: array of strings with the parameters to a specific
2155 * target.
2156 *
2157 * Instead of having the struct dm_target_spec and the parameters for every
2158 * target embedded at the end of struct dm_ioctl (as performed in a normal
2159 * ioctl), pass them as arguments, so the caller doesn't need to serialize them.
2160 * The size of the spec_array and target_params_array is given by
2161 * @dmi->target_count.
2162 * This function is supposed to be called in early boot, so locking mechanisms
2163 * to protect against concurrent loads are not required.
2164 */
2165int __init dm_early_create(struct dm_ioctl *dmi,
2166			   struct dm_target_spec **spec_array,
2167			   char **target_params_array)
2168{
2169	int r, m = DM_ANY_MINOR;
2170	struct dm_table *t, *old_map;
2171	struct mapped_device *md;
2172	unsigned int i;
2173
2174	if (!dmi->target_count)
2175		return -EINVAL;
2176
2177	r = check_name(dmi->name);
2178	if (r)
2179		return r;
2180
2181	if (dmi->flags & DM_PERSISTENT_DEV_FLAG)
2182		m = MINOR(huge_decode_dev(dmi->dev));
2183
2184	/* alloc dm device */
2185	r = dm_create(m, &md);
2186	if (r)
2187		return r;
2188
2189	/* hash insert */
2190	r = dm_hash_insert(dmi->name, *dmi->uuid ? dmi->uuid : NULL, md);
2191	if (r)
2192		goto err_destroy_dm;
2193
2194	/* alloc table */
2195	r = dm_table_create(&t, get_mode(dmi), dmi->target_count, md);
2196	if (r)
2197		goto err_hash_remove;
2198
2199	/* add targets */
2200	for (i = 0; i < dmi->target_count; i++) {
2201		r = dm_table_add_target(t, spec_array[i]->target_type,
2202					(sector_t) spec_array[i]->sector_start,
2203					(sector_t) spec_array[i]->length,
2204					target_params_array[i]);
2205		if (r) {
2206			DMERR("error adding target to table");
2207			goto err_destroy_table;
2208		}
2209	}
2210
2211	/* finish table */
2212	r = dm_table_complete(t);
2213	if (r)
2214		goto err_destroy_table;
2215
 
2216	/* setup md->queue to reflect md's type (may block) */
2217	r = dm_setup_md_queue(md, t);
2218	if (r) {
2219		DMERR("unable to set up device queue for new table.");
2220		goto err_destroy_table;
2221	}
2222
2223	/* Set new map */
2224	dm_suspend(md, 0);
2225	old_map = dm_swap_table(md, t);
2226	if (IS_ERR(old_map)) {
2227		r = PTR_ERR(old_map);
2228		goto err_destroy_table;
2229	}
2230	set_disk_ro(dm_disk(md), !!(dmi->flags & DM_READONLY_FLAG));
2231
2232	/* resume device */
2233	r = dm_resume(md);
2234	if (r)
2235		goto err_destroy_table;
2236
2237	DMINFO("%s (%s) is ready", md->disk->disk_name, dmi->name);
2238	dm_put(md);
2239	return 0;
2240
2241err_destroy_table:
2242	dm_table_destroy(t);
2243err_hash_remove:
2244	(void) __hash_remove(__get_name_cell(dmi->name));
2245	/* release reference from __get_name_cell */
2246	dm_put(md);
2247err_destroy_dm:
2248	dm_put(md);
2249	dm_destroy(md);
2250	return r;
2251}