1/*
   2 * Copyright (c) 2006, Intel Corporation.
   3 *
   4 * This program is free software; you can redistribute it and/or modify it
   5 * under the terms and conditions of the GNU General Public License,
   6 * version 2, as published by the Free Software Foundation.
   7 *
   8 * This program is distributed in the hope it will be useful, but WITHOUT
   9 * ANY WARRANTY; without even the implied warranty of MERCHANTABILITY or
  10 * FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License for
  11 * more details.
  12 *
  13 * You should have received a copy of the GNU General Public License along with
  14 * this program; if not, write to the Free Software Foundation, Inc., 59 Temple
  15 * Place - Suite 330, Boston, MA 02111-1307 USA.
  16 *
  17 * Copyright (C) 2006-2008 Intel Corporation
  18 * Author: Ashok Raj <ashok.raj@intel.com>
  19 * Author: Shaohua Li <shaohua.li@intel.com>
  20 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
  21 *
  22 * This file implements early detection/parsing of Remapping Devices
  23 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  24 * tables.
  25 *
  26 * These routines are used by both DMA-remapping and Interrupt-remapping
  27 */
  28
  29#define pr_fmt(fmt)     "DMAR: " fmt
  30
  31#include <linux/pci.h>
  32#include <linux/dmar.h>
  33#include <linux/iova.h>
  34#include <linux/intel-iommu.h>
  35#include <linux/timer.h>
  36#include <linux/irq.h>
  37#include <linux/interrupt.h>
  38#include <linux/tboot.h>
  39#include <linux/dmi.h>
  40#include <linux/slab.h>
  41#include <linux/iommu.h>
  42#include <asm/irq_remapping.h>
  43#include <asm/iommu_table.h>
  44
  45#include "irq_remapping.h"
  46
  47typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
  48struct dmar_res_callback {
  49	dmar_res_handler_t	cb[ACPI_DMAR_TYPE_RESERVED];
  50	void			*arg[ACPI_DMAR_TYPE_RESERVED];
  51	bool			ignore_unhandled;
  52	bool			print_entry;
  53};
  54
  55/*
  56 * Assumptions:
  57 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
  58 *    before IO devices managed by that unit.
  59 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
  60 *    after IO devices managed by that unit.
  61 * 3) Hotplug events are rare.
  62 *
  63 * Locking rules for DMA and interrupt remapping related global data structures:
  64 * 1) Use dmar_global_lock in process context
  65 * 2) Use RCU in interrupt context
  66 */
  67DECLARE_RWSEM(dmar_global_lock);
  68LIST_HEAD(dmar_drhd_units);
  69
  70struct acpi_table_header * __initdata dmar_tbl;
  71static acpi_size dmar_tbl_size;
  72static int dmar_dev_scope_status = 1;
  73static unsigned long dmar_seq_ids[BITS_TO_LONGS(DMAR_UNITS_SUPPORTED)];
  74
  75static int alloc_iommu(struct dmar_drhd_unit *drhd);
  76static void free_iommu(struct intel_iommu *iommu);
  77
  78static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  79{
  80	/*
  81	 * add INCLUDE_ALL at the tail, so scan the list will find it at
  82	 * the very end.
  83	 */
  84	if (drhd->include_all)
  85		list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
  86	else
  87		list_add_rcu(&drhd->list, &dmar_drhd_units);
  88}
  89
  90void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
  91{
  92	struct acpi_dmar_device_scope *scope;
  93
  94	*cnt = 0;
  95	while (start < end) {
  96		scope = start;
  97		if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
  98		    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
  99		    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 100			(*cnt)++;
 101		else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
 102			scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
 103			pr_warn("Unsupported device scope\n");
 104		}
 105		start += scope->length;
 106	}
 107	if (*cnt == 0)
 108		return NULL;
 109
 110	return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
 111}
 112
 113void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
 
 114{
 115	int i;
 116	struct device *tmp_dev;
 117
 118	if (*devices && *cnt) {
 119		for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
 120			put_device(tmp_dev);
 121		kfree(*devices);
 122	}
 123
 124	*devices = NULL;
 125	*cnt = 0;
 126}
 127
 128/* Optimize out kzalloc()/kfree() for normal cases */
 129static char dmar_pci_notify_info_buf[64];
 130
 131static struct dmar_pci_notify_info *
 132dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
 133{
 134	int level = 0;
 135	size_t size;
 136	struct pci_dev *tmp;
 137	struct dmar_pci_notify_info *info;
 138
 139	BUG_ON(dev->is_virtfn);
 140
 141	/* Only generate path[] for device addition event */
 142	if (event == BUS_NOTIFY_ADD_DEVICE)
 143		for (tmp = dev; tmp; tmp = tmp->bus->self)
 144			level++;
 145
 146	size = sizeof(*info) + level * sizeof(struct acpi_dmar_pci_path);
 147	if (size <= sizeof(dmar_pci_notify_info_buf)) {
 148		info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
 149	} else {
 150		info = kzalloc(size, GFP_KERNEL);
 151		if (!info) {
 152			pr_warn("Out of memory when allocating notify_info "
 153				"for %s.\n", pci_name(dev));
 154			if (dmar_dev_scope_status == 0)
 155				dmar_dev_scope_status = -ENOMEM;
 156			return NULL;
 157		}
 158	}
 159
 160	info->event = event;
 161	info->dev = dev;
 162	info->seg = pci_domain_nr(dev->bus);
 163	info->level = level;
 164	if (event == BUS_NOTIFY_ADD_DEVICE) {
 165		for (tmp = dev; tmp; tmp = tmp->bus->self) {
 166			level--;
 167			info->path[level].bus = tmp->bus->number;
 168			info->path[level].device = PCI_SLOT(tmp->devfn);
 169			info->path[level].function = PCI_FUNC(tmp->devfn);
 170			if (pci_is_root_bus(tmp->bus))
 171				info->bus = tmp->bus->number;
 172		}
 
 
 
 
 
 
 
 
 
 
 173	}
 174
 175	return info;
 176}
 177
 178static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
 179{
 180	if ((void *)info != dmar_pci_notify_info_buf)
 181		kfree(info);
 182}
 183
 184static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
 185				struct acpi_dmar_pci_path *path, int count)
 186{
 187	int i;
 188
 189	if (info->bus != bus)
 190		goto fallback;
 191	if (info->level != count)
 192		goto fallback;
 193
 194	for (i = 0; i < count; i++) {
 195		if (path[i].device != info->path[i].device ||
 196		    path[i].function != info->path[i].function)
 197			goto fallback;
 198	}
 199
 200	return true;
 201
 202fallback:
 203
 204	if (count != 1)
 205		return false;
 206
 207	i = info->level - 1;
 208	if (bus              == info->path[i].bus &&
 209	    path[0].device   == info->path[i].device &&
 210	    path[0].function == info->path[i].function) {
 211		pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
 212			bus, path[0].device, path[0].function);
 213		return true;
 214	}
 215
 216	return false;
 217}
 218
 219/* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
 220int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
 221			  void *start, void*end, u16 segment,
 222			  struct dmar_dev_scope *devices,
 223			  int devices_cnt)
 224{
 225	int i, level;
 226	struct device *tmp, *dev = &info->dev->dev;
 227	struct acpi_dmar_device_scope *scope;
 228	struct acpi_dmar_pci_path *path;
 229
 230	if (segment != info->seg)
 231		return 0;
 232
 233	for (; start < end; start += scope->length) {
 
 234		scope = start;
 235		if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 236		    scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 237			continue;
 238
 239		path = (struct acpi_dmar_pci_path *)(scope + 1);
 240		level = (scope->length - sizeof(*scope)) / sizeof(*path);
 241		if (!dmar_match_pci_path(info, scope->bus, path, level))
 242			continue;
 243
 244		if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT) ^
 245		    (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL)) {
 246			pr_warn("Device scope type does not match for %s\n",
 247				pci_name(info->dev));
 248			return -EINVAL;
 249		}
 250
 251		for_each_dev_scope(devices, devices_cnt, i, tmp)
 252			if (tmp == NULL) {
 253				devices[i].bus = info->dev->bus->number;
 254				devices[i].devfn = info->dev->devfn;
 255				rcu_assign_pointer(devices[i].dev,
 256						   get_device(dev));
 257				return 1;
 258			}
 259		BUG_ON(i >= devices_cnt);
 260	}
 261
 262	return 0;
 263}
 264
 265int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
 266			  struct dmar_dev_scope *devices, int count)
 267{
 268	int index;
 269	struct device *tmp;
 270
 271	if (info->seg != segment)
 272		return 0;
 273
 274	for_each_active_dev_scope(devices, count, index, tmp)
 275		if (tmp == &info->dev->dev) {
 276			RCU_INIT_POINTER(devices[index].dev, NULL);
 277			synchronize_rcu();
 278			put_device(tmp);
 279			return 1;
 280		}
 281
 282	return 0;
 283}
 284
 285static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
 286{
 287	int ret = 0;
 288	struct dmar_drhd_unit *dmaru;
 289	struct acpi_dmar_hardware_unit *drhd;
 290
 291	for_each_drhd_unit(dmaru) {
 292		if (dmaru->include_all)
 293			continue;
 294
 295		drhd = container_of(dmaru->hdr,
 296				    struct acpi_dmar_hardware_unit, header);
 297		ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
 298				((void *)drhd) + drhd->header.length,
 299				dmaru->segment,
 300				dmaru->devices, dmaru->devices_cnt);
 301		if (ret != 0)
 302			break;
 
 
 
 
 
 
 
 303	}
 304	if (ret >= 0)
 305		ret = dmar_iommu_notify_scope_dev(info);
 306	if (ret < 0 && dmar_dev_scope_status == 0)
 307		dmar_dev_scope_status = ret;
 308
 309	return ret;
 310}
 311
 312static void  dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
 313{
 314	struct dmar_drhd_unit *dmaru;
 315
 316	for_each_drhd_unit(dmaru)
 317		if (dmar_remove_dev_scope(info, dmaru->segment,
 318			dmaru->devices, dmaru->devices_cnt))
 319			break;
 320	dmar_iommu_notify_scope_dev(info);
 321}
 322
 323static int dmar_pci_bus_notifier(struct notifier_block *nb,
 324				 unsigned long action, void *data)
 325{
 326	struct pci_dev *pdev = to_pci_dev(data);
 327	struct dmar_pci_notify_info *info;
 328
 329	/* Only care about add/remove events for physical functions */
 330	if (pdev->is_virtfn)
 331		return NOTIFY_DONE;
 332	if (action != BUS_NOTIFY_ADD_DEVICE &&
 333	    action != BUS_NOTIFY_REMOVED_DEVICE)
 334		return NOTIFY_DONE;
 335
 336	info = dmar_alloc_pci_notify_info(pdev, action);
 337	if (!info)
 338		return NOTIFY_DONE;
 339
 340	down_write(&dmar_global_lock);
 341	if (action == BUS_NOTIFY_ADD_DEVICE)
 342		dmar_pci_bus_add_dev(info);
 343	else if (action == BUS_NOTIFY_REMOVED_DEVICE)
 344		dmar_pci_bus_del_dev(info);
 345	up_write(&dmar_global_lock);
 346
 347	dmar_free_pci_notify_info(info);
 348
 349	return NOTIFY_OK;
 350}
 351
 352static struct notifier_block dmar_pci_bus_nb = {
 353	.notifier_call = dmar_pci_bus_notifier,
 354	.priority = INT_MIN,
 355};
 356
 357static struct dmar_drhd_unit *
 358dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
 359{
 360	struct dmar_drhd_unit *dmaru;
 361
 362	list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list)
 363		if (dmaru->segment == drhd->segment &&
 364		    dmaru->reg_base_addr == drhd->address)
 365			return dmaru;
 366
 367	return NULL;
 368}
 369
 370/**
 371 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 372 * structure which uniquely represent one DMA remapping hardware unit
 373 * present in the platform
 374 */
 375static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
 
 376{
 377	struct acpi_dmar_hardware_unit *drhd;
 378	struct dmar_drhd_unit *dmaru;
 379	int ret = 0;
 380
 381	drhd = (struct acpi_dmar_hardware_unit *)header;
 382	dmaru = dmar_find_dmaru(drhd);
 383	if (dmaru)
 384		goto out;
 385
 386	dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
 387	if (!dmaru)
 388		return -ENOMEM;
 389
 390	/*
 391	 * If header is allocated from slab by ACPI _DSM method, we need to
 392	 * copy the content because the memory buffer will be freed on return.
 393	 */
 394	dmaru->hdr = (void *)(dmaru + 1);
 395	memcpy(dmaru->hdr, header, header->length);
 396	dmaru->reg_base_addr = drhd->address;
 397	dmaru->segment = drhd->segment;
 398	dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 399	dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
 400					      ((void *)drhd) + drhd->header.length,
 401					      &dmaru->devices_cnt);
 402	if (dmaru->devices_cnt && dmaru->devices == NULL) {
 403		kfree(dmaru);
 404		return -ENOMEM;
 405	}
 406
 407	ret = alloc_iommu(dmaru);
 408	if (ret) {
 409		dmar_free_dev_scope(&dmaru->devices,
 410				    &dmaru->devices_cnt);
 411		kfree(dmaru);
 412		return ret;
 413	}
 414	dmar_register_drhd_unit(dmaru);
 415
 416out:
 417	if (arg)
 418		(*(int *)arg)++;
 419
 420	return 0;
 421}
 422
 423static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
 424{
 425	if (dmaru->devices && dmaru->devices_cnt)
 426		dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
 427	if (dmaru->iommu)
 428		free_iommu(dmaru->iommu);
 429	kfree(dmaru);
 430}
 431
 432static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
 433				      void *arg)
 434{
 435	struct acpi_dmar_andd *andd = (void *)header;
 436
 437	/* Check for NUL termination within the designated length */
 438	if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
 439		WARN_TAINT(1, TAINT_FIRMWARE_WORKAROUND,
 440			   "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
 441			   "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 442			   dmi_get_system_info(DMI_BIOS_VENDOR),
 443			   dmi_get_system_info(DMI_BIOS_VERSION),
 444			   dmi_get_system_info(DMI_PRODUCT_VERSION));
 445		return -EINVAL;
 446	}
 447	pr_info("ANDD device: %x name: %s\n", andd->device_number,
 448		andd->device_name);
 449
 450	return 0;
 
 
 
 
 
 
 
 
 
 
 
 
 
 451}
 452
 453#ifdef CONFIG_ACPI_NUMA
 454static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
 
 455{
 456	struct acpi_dmar_rhsa *rhsa;
 457	struct dmar_drhd_unit *drhd;
 458
 459	rhsa = (struct acpi_dmar_rhsa *)header;
 460	for_each_drhd_unit(drhd) {
 461		if (drhd->reg_base_addr == rhsa->base_address) {
 462			int node = acpi_map_pxm_to_node(rhsa->proximity_domain);
 463
 464			if (!node_online(node))
 465				node = -1;
 466			drhd->iommu->node = node;
 467			return 0;
 468		}
 469	}
 470	WARN_TAINT(
 471		1, TAINT_FIRMWARE_WORKAROUND,
 472		"Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 473		"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 474		drhd->reg_base_addr,
 475		dmi_get_system_info(DMI_BIOS_VENDOR),
 476		dmi_get_system_info(DMI_BIOS_VERSION),
 477		dmi_get_system_info(DMI_PRODUCT_VERSION));
 478
 479	return 0;
 480}
 481#else
 482#define	dmar_parse_one_rhsa		dmar_res_noop
 483#endif
 484
 485static void __init
 486dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 487{
 488	struct acpi_dmar_hardware_unit *drhd;
 489	struct acpi_dmar_reserved_memory *rmrr;
 490	struct acpi_dmar_atsr *atsr;
 491	struct acpi_dmar_rhsa *rhsa;
 492
 493	switch (header->type) {
 494	case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 495		drhd = container_of(header, struct acpi_dmar_hardware_unit,
 496				    header);
 497		pr_info("DRHD base: %#016Lx flags: %#x\n",
 
 498			(unsigned long long)drhd->address, drhd->flags);
 499		break;
 500	case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 501		rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 502				    header);
 503		pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 
 504			(unsigned long long)rmrr->base_address,
 505			(unsigned long long)rmrr->end_address);
 506		break;
 507	case ACPI_DMAR_TYPE_ROOT_ATS:
 508		atsr = container_of(header, struct acpi_dmar_atsr, header);
 509		pr_info("ATSR flags: %#x\n", atsr->flags);
 510		break;
 511	case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
 512		rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 513		pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 514		       (unsigned long long)rhsa->base_address,
 515		       rhsa->proximity_domain);
 516		break;
 517	case ACPI_DMAR_TYPE_NAMESPACE:
 518		/* We don't print this here because we need to sanity-check
 519		   it first. So print it in dmar_parse_one_andd() instead. */
 520		break;
 521	}
 522}
 523
 524/**
 525 * dmar_table_detect - checks to see if the platform supports DMAR devices
 526 */
 527static int __init dmar_table_detect(void)
 528{
 529	acpi_status status = AE_OK;
 530
 531	/* if we could find DMAR table, then there are DMAR devices */
 532	status = acpi_get_table_with_size(ACPI_SIG_DMAR, 0,
 533				(struct acpi_table_header **)&dmar_tbl,
 534				&dmar_tbl_size);
 535
 536	if (ACPI_SUCCESS(status) && !dmar_tbl) {
 537		pr_warn("Unable to map DMAR\n");
 538		status = AE_NOT_FOUND;
 539	}
 540
 541	return (ACPI_SUCCESS(status) ? 1 : 0);
 542}
 543
 544static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
 545				       size_t len, struct dmar_res_callback *cb)
 546{
 547	int ret = 0;
 548	struct acpi_dmar_header *iter, *next;
 549	struct acpi_dmar_header *end = ((void *)start) + len;
 550
 551	for (iter = start; iter < end && ret == 0; iter = next) {
 552		next = (void *)iter + iter->length;
 553		if (iter->length == 0) {
 554			/* Avoid looping forever on bad ACPI tables */
 555			pr_debug(FW_BUG "Invalid 0-length structure\n");
 556			break;
 557		} else if (next > end) {
 558			/* Avoid passing table end */
 559			pr_warn(FW_BUG "Record passes table end\n");
 560			ret = -EINVAL;
 561			break;
 562		}
 563
 564		if (cb->print_entry)
 565			dmar_table_print_dmar_entry(iter);
 566
 567		if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
 568			/* continue for forward compatibility */
 569			pr_debug("Unknown DMAR structure type %d\n",
 570				 iter->type);
 571		} else if (cb->cb[iter->type]) {
 572			ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
 573		} else if (!cb->ignore_unhandled) {
 574			pr_warn("No handler for DMAR structure type %d\n",
 575				iter->type);
 576			ret = -EINVAL;
 577		}
 578	}
 579
 580	return ret;
 581}
 582
 583static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
 584				       struct dmar_res_callback *cb)
 585{
 586	return dmar_walk_remapping_entries((void *)(dmar + 1),
 587			dmar->header.length - sizeof(*dmar), cb);
 588}
 589
 590/**
 591 * parse_dmar_table - parses the DMA reporting table
 592 */
 593static int __init
 594parse_dmar_table(void)
 595{
 596	struct acpi_table_dmar *dmar;
 
 597	int ret = 0;
 598	int drhd_count = 0;
 599	struct dmar_res_callback cb = {
 600		.print_entry = true,
 601		.ignore_unhandled = true,
 602		.arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
 603		.cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
 604		.cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
 605		.cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
 606		.cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
 607		.cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
 608	};
 609
 610	/*
 611	 * Do it again, earlier dmar_tbl mapping could be mapped with
 612	 * fixed map.
 613	 */
 614	dmar_table_detect();
 615
 616	/*
 617	 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 618	 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 619	 */
 620	dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 621
 622	dmar = (struct acpi_table_dmar *)dmar_tbl;
 623	if (!dmar)
 624		return -ENODEV;
 625
 626	if (dmar->width < PAGE_SHIFT - 1) {
 627		pr_warn("Invalid DMAR haw\n");
 628		return -EINVAL;
 629	}
 630
 631	pr_info("Host address width %d\n", dmar->width + 1);
 632	ret = dmar_walk_dmar_table(dmar, &cb);
 633	if (ret == 0 && drhd_count == 0)
 634		pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 635
 
 
 636	return ret;
 637}
 638
 639static int dmar_pci_device_match(struct dmar_dev_scope devices[],
 640				 int cnt, struct pci_dev *dev)
 641{
 642	int index;
 643	struct device *tmp;
 644
 645	while (dev) {
 646		for_each_active_dev_scope(devices, cnt, index, tmp)
 647			if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
 648				return 1;
 649
 650		/* Check our parent */
 651		dev = dev->bus->self;
 652	}
 653
 654	return 0;
 655}
 656
 657struct dmar_drhd_unit *
 658dmar_find_matched_drhd_unit(struct pci_dev *dev)
 659{
 660	struct dmar_drhd_unit *dmaru;
 661	struct acpi_dmar_hardware_unit *drhd;
 662
 663	dev = pci_physfn(dev);
 664
 665	rcu_read_lock();
 666	for_each_drhd_unit(dmaru) {
 667		drhd = container_of(dmaru->hdr,
 668				    struct acpi_dmar_hardware_unit,
 669				    header);
 670
 671		if (dmaru->include_all &&
 672		    drhd->segment == pci_domain_nr(dev->bus))
 673			goto out;
 674
 675		if (dmar_pci_device_match(dmaru->devices,
 676					  dmaru->devices_cnt, dev))
 677			goto out;
 678	}
 679	dmaru = NULL;
 680out:
 681	rcu_read_unlock();
 682
 683	return dmaru;
 684}
 685
 686static void __init dmar_acpi_insert_dev_scope(u8 device_number,
 687					      struct acpi_device *adev)
 688{
 689	struct dmar_drhd_unit *dmaru;
 690	struct acpi_dmar_hardware_unit *drhd;
 691	struct acpi_dmar_device_scope *scope;
 692	struct device *tmp;
 693	int i;
 694	struct acpi_dmar_pci_path *path;
 695
 696	for_each_drhd_unit(dmaru) {
 697		drhd = container_of(dmaru->hdr,
 698				    struct acpi_dmar_hardware_unit,
 699				    header);
 700
 701		for (scope = (void *)(drhd + 1);
 702		     (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
 703		     scope = ((void *)scope) + scope->length) {
 704			if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
 705				continue;
 706			if (scope->enumeration_id != device_number)
 707				continue;
 708
 709			path = (void *)(scope + 1);
 710			pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
 711				dev_name(&adev->dev), dmaru->reg_base_addr,
 712				scope->bus, path->device, path->function);
 713			for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
 714				if (tmp == NULL) {
 715					dmaru->devices[i].bus = scope->bus;
 716					dmaru->devices[i].devfn = PCI_DEVFN(path->device,
 717									    path->function);
 718					rcu_assign_pointer(dmaru->devices[i].dev,
 719							   get_device(&adev->dev));
 720					return;
 721				}
 722			BUG_ON(i >= dmaru->devices_cnt);
 723		}
 724	}
 725	pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
 726		device_number, dev_name(&adev->dev));
 727}
 728
 729static int __init dmar_acpi_dev_scope_init(void)
 730{
 731	struct acpi_dmar_andd *andd;
 732
 733	if (dmar_tbl == NULL)
 734		return -ENODEV;
 735
 736	for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
 737	     ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
 738	     andd = ((void *)andd) + andd->header.length) {
 739		if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
 740			acpi_handle h;
 741			struct acpi_device *adev;
 742
 743			if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
 744							  andd->device_name,
 745							  &h))) {
 746				pr_err("Failed to find handle for ACPI object %s\n",
 747				       andd->device_name);
 748				continue;
 749			}
 750			if (acpi_bus_get_device(h, &adev)) {
 751				pr_err("Failed to get device for ACPI object %s\n",
 752				       andd->device_name);
 753				continue;
 754			}
 755			dmar_acpi_insert_dev_scope(andd->device_number, adev);
 756		}
 757	}
 758	return 0;
 759}
 760
 761int __init dmar_dev_scope_init(void)
 762{
 763	struct pci_dev *dev = NULL;
 764	struct dmar_pci_notify_info *info;
 
 765
 766	if (dmar_dev_scope_status != 1)
 767		return dmar_dev_scope_status;
 768
 769	if (list_empty(&dmar_drhd_units)) {
 770		dmar_dev_scope_status = -ENODEV;
 771	} else {
 772		dmar_dev_scope_status = 0;
 773
 774		dmar_acpi_dev_scope_init();
 
 
 
 
 775
 776		for_each_pci_dev(dev) {
 777			if (dev->is_virtfn)
 778				continue;
 779
 780			info = dmar_alloc_pci_notify_info(dev,
 781					BUS_NOTIFY_ADD_DEVICE);
 782			if (!info) {
 783				return dmar_dev_scope_status;
 784			} else {
 785				dmar_pci_bus_add_dev(info);
 786				dmar_free_pci_notify_info(info);
 787			}
 788		}
 789
 790		bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
 791	}
 792
 793	return dmar_dev_scope_status;
 
 
 794}
 795
 796
 797int __init dmar_table_init(void)
 798{
 799	static int dmar_table_initialized;
 800	int ret;
 801
 802	if (dmar_table_initialized == 0) {
 803		ret = parse_dmar_table();
 804		if (ret < 0) {
 805			if (ret != -ENODEV)
 806				pr_info("Parse DMAR table failure.\n");
 807		} else  if (list_empty(&dmar_drhd_units)) {
 808			pr_info("No DMAR devices found\n");
 809			ret = -ENODEV;
 810		}
 
 
 811
 812		if (ret < 0)
 813			dmar_table_initialized = ret;
 814		else
 815			dmar_table_initialized = 1;
 816	}
 817
 818	return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
 819}
 820
 821static void warn_invalid_dmar(u64 addr, const char *message)
 822{
 823	WARN_TAINT_ONCE(
 824		1, TAINT_FIRMWARE_WORKAROUND,
 825		"Your BIOS is broken; DMAR reported at address %llx%s!\n"
 826		"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 827		addr, message,
 828		dmi_get_system_info(DMI_BIOS_VENDOR),
 829		dmi_get_system_info(DMI_BIOS_VERSION),
 830		dmi_get_system_info(DMI_PRODUCT_VERSION));
 831}
 832
 833static int __ref
 834dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
 835{
 
 
 836	struct acpi_dmar_hardware_unit *drhd;
 837	void __iomem *addr;
 838	u64 cap, ecap;
 839
 840	drhd = (void *)entry;
 841	if (!drhd->address) {
 842		warn_invalid_dmar(0, "");
 843		return -EINVAL;
 844	}
 845
 846	if (arg)
 847		addr = ioremap(drhd->address, VTD_PAGE_SIZE);
 848	else
 849		addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 850	if (!addr) {
 851		pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
 852		return -EINVAL;
 853	}
 854
 855	cap = dmar_readq(addr + DMAR_CAP_REG);
 856	ecap = dmar_readq(addr + DMAR_ECAP_REG);
 
 
 
 
 
 
 
 857
 858	if (arg)
 859		iounmap(addr);
 860	else
 861		early_iounmap(addr, VTD_PAGE_SIZE);
 
 
 
 
 
 
 
 
 
 
 862
 863	if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 864		warn_invalid_dmar(drhd->address, " returns all ones");
 865		return -EINVAL;
 866	}
 
 867
 
 868	return 0;
 869}
 870
 871int __init detect_intel_iommu(void)
 872{
 873	int ret;
 874	struct dmar_res_callback validate_drhd_cb = {
 875		.cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
 876		.ignore_unhandled = true,
 877	};
 878
 879	down_write(&dmar_global_lock);
 880	ret = dmar_table_detect();
 881	if (ret)
 882		ret = !dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
 883					    &validate_drhd_cb);
 884	if (ret && !no_iommu && !iommu_detected && !dmar_disabled) {
 885		iommu_detected = 1;
 886		/* Make sure ACS will be enabled */
 887		pci_request_acs();
 888	}
 
 
 
 
 
 
 
 
 
 889
 890#ifdef CONFIG_X86
 891	if (ret)
 892		x86_init.iommu.iommu_init = intel_iommu_init;
 893#endif
 894
 895	early_acpi_os_unmap_memory((void __iomem *)dmar_tbl, dmar_tbl_size);
 896	dmar_tbl = NULL;
 897	up_write(&dmar_global_lock);
 898
 899	return ret ? 1 : -ENODEV;
 900}
 901
 902
 903static void unmap_iommu(struct intel_iommu *iommu)
 904{
 905	iounmap(iommu->reg);
 906	release_mem_region(iommu->reg_phys, iommu->reg_size);
 907}
 908
 909/**
 910 * map_iommu: map the iommu's registers
 911 * @iommu: the iommu to map
 912 * @phys_addr: the physical address of the base resgister
 913 *
 914 * Memory map the iommu's registers.  Start w/ a single page, and
 915 * possibly expand if that turns out to be insufficent.
 916 */
 917static int map_iommu(struct intel_iommu *iommu, u64 phys_addr)
 918{
 919	int map_size, err=0;
 920
 921	iommu->reg_phys = phys_addr;
 922	iommu->reg_size = VTD_PAGE_SIZE;
 923
 924	if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 925		pr_err("Can't reserve memory\n");
 926		err = -EBUSY;
 927		goto out;
 928	}
 929
 930	iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 931	if (!iommu->reg) {
 932		pr_err("Can't map the region\n");
 933		err = -ENOMEM;
 934		goto release;
 935	}
 936
 937	iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 938	iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 939
 940	if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 941		err = -EINVAL;
 942		warn_invalid_dmar(phys_addr, " returns all ones");
 943		goto unmap;
 944	}
 945
 946	/* the registers might be more than one page */
 947	map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 948			 cap_max_fault_reg_offset(iommu->cap));
 949	map_size = VTD_PAGE_ALIGN(map_size);
 950	if (map_size > iommu->reg_size) {
 951		iounmap(iommu->reg);
 952		release_mem_region(iommu->reg_phys, iommu->reg_size);
 953		iommu->reg_size = map_size;
 954		if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
 955					iommu->name)) {
 956			pr_err("Can't reserve memory\n");
 957			err = -EBUSY;
 958			goto out;
 959		}
 960		iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 961		if (!iommu->reg) {
 962			pr_err("Can't map the region\n");
 963			err = -ENOMEM;
 964			goto release;
 965		}
 966	}
 967	err = 0;
 968	goto out;
 969
 970unmap:
 971	iounmap(iommu->reg);
 972release:
 973	release_mem_region(iommu->reg_phys, iommu->reg_size);
 974out:
 975	return err;
 976}
 977
 978static int dmar_alloc_seq_id(struct intel_iommu *iommu)
 979{
 980	iommu->seq_id = find_first_zero_bit(dmar_seq_ids,
 981					    DMAR_UNITS_SUPPORTED);
 982	if (iommu->seq_id >= DMAR_UNITS_SUPPORTED) {
 983		iommu->seq_id = -1;
 984	} else {
 985		set_bit(iommu->seq_id, dmar_seq_ids);
 986		sprintf(iommu->name, "dmar%d", iommu->seq_id);
 987	}
 988
 989	return iommu->seq_id;
 990}
 991
 992static void dmar_free_seq_id(struct intel_iommu *iommu)
 993{
 994	if (iommu->seq_id >= 0) {
 995		clear_bit(iommu->seq_id, dmar_seq_ids);
 996		iommu->seq_id = -1;
 997	}
 998}
 999
1000static int alloc_iommu(struct dmar_drhd_unit *drhd)
1001{
1002	struct intel_iommu *iommu;
1003	u32 ver, sts;
 
 
1004	int agaw = 0;
1005	int msagaw = 0;
1006	int err;
1007
1008	if (!drhd->reg_base_addr) {
1009		warn_invalid_dmar(0, "");
1010		return -EINVAL;
1011	}
1012
1013	iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1014	if (!iommu)
1015		return -ENOMEM;
1016
1017	if (dmar_alloc_seq_id(iommu) < 0) {
1018		pr_err("Failed to allocate seq_id\n");
1019		err = -ENOSPC;
 
 
 
1020		goto error;
1021	}
 
 
1022
1023	err = map_iommu(iommu, drhd->reg_base_addr);
1024	if (err) {
1025		pr_err("Failed to map %s\n", iommu->name);
1026		goto error_free_seq_id;
1027	}
1028
1029	err = -EINVAL;
1030	agaw = iommu_calculate_agaw(iommu);
1031	if (agaw < 0) {
1032		pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1033			iommu->seq_id);
 
1034		goto err_unmap;
1035	}
1036	msagaw = iommu_calculate_max_sagaw(iommu);
1037	if (msagaw < 0) {
1038		pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
 
1039			iommu->seq_id);
1040		goto err_unmap;
1041	}
1042	iommu->agaw = agaw;
1043	iommu->msagaw = msagaw;
1044	iommu->segment = drhd->segment;
1045
1046	iommu->node = -1;
1047
 
 
 
 
 
 
 
 
 
 
 
 
 
1048	ver = readl(iommu->reg + DMAR_VER_REG);
1049	pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1050		iommu->name,
1051		(unsigned long long)drhd->reg_base_addr,
1052		DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1053		(unsigned long long)iommu->cap,
1054		(unsigned long long)iommu->ecap);
1055
1056	/* Reflect status in gcmd */
1057	sts = readl(iommu->reg + DMAR_GSTS_REG);
1058	if (sts & DMA_GSTS_IRES)
1059		iommu->gcmd |= DMA_GCMD_IRE;
1060	if (sts & DMA_GSTS_TES)
1061		iommu->gcmd |= DMA_GCMD_TE;
1062	if (sts & DMA_GSTS_QIES)
1063		iommu->gcmd |= DMA_GCMD_QIE;
1064
1065	raw_spin_lock_init(&iommu->register_lock);
1066
1067	if (intel_iommu_enabled) {
1068		iommu->iommu_dev = iommu_device_create(NULL, iommu,
1069						       intel_iommu_groups,
1070						       "%s", iommu->name);
1071
1072		if (IS_ERR(iommu->iommu_dev)) {
1073			err = PTR_ERR(iommu->iommu_dev);
1074			goto err_unmap;
1075		}
1076	}
1077
1078	drhd->iommu = iommu;
1079
1080	return 0;
1081
1082err_unmap:
1083	unmap_iommu(iommu);
1084error_free_seq_id:
1085	dmar_free_seq_id(iommu);
1086error:
1087	kfree(iommu);
1088	return err;
1089}
1090
1091static void free_iommu(struct intel_iommu *iommu)
1092{
1093	iommu_device_destroy(iommu->iommu_dev);
1094
1095	if (iommu->irq) {
1096		if (iommu->pr_irq) {
1097			free_irq(iommu->pr_irq, iommu);
1098			dmar_free_hwirq(iommu->pr_irq);
1099			iommu->pr_irq = 0;
1100		}
1101		free_irq(iommu->irq, iommu);
1102		dmar_free_hwirq(iommu->irq);
1103		iommu->irq = 0;
1104	}
1105
1106	if (iommu->qi) {
1107		free_page((unsigned long)iommu->qi->desc);
1108		kfree(iommu->qi->desc_status);
1109		kfree(iommu->qi);
1110	}
1111
1112	if (iommu->reg)
1113		unmap_iommu(iommu);
1114
1115	dmar_free_seq_id(iommu);
1116	kfree(iommu);
1117}
1118
1119/*
1120 * Reclaim all the submitted descriptors which have completed its work.
1121 */
1122static inline void reclaim_free_desc(struct q_inval *qi)
1123{
1124	while (qi->desc_status[qi->free_tail] == QI_DONE ||
1125	       qi->desc_status[qi->free_tail] == QI_ABORT) {
1126		qi->desc_status[qi->free_tail] = QI_FREE;
1127		qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1128		qi->free_cnt++;
1129	}
1130}
1131
1132static int qi_check_fault(struct intel_iommu *iommu, int index)
1133{
1134	u32 fault;
1135	int head, tail;
1136	struct q_inval *qi = iommu->qi;
1137	int wait_index = (index + 1) % QI_LENGTH;
1138
1139	if (qi->desc_status[wait_index] == QI_ABORT)
1140		return -EAGAIN;
1141
1142	fault = readl(iommu->reg + DMAR_FSTS_REG);
1143
1144	/*
1145	 * If IQE happens, the head points to the descriptor associated
1146	 * with the error. No new descriptors are fetched until the IQE
1147	 * is cleared.
1148	 */
1149	if (fault & DMA_FSTS_IQE) {
1150		head = readl(iommu->reg + DMAR_IQH_REG);
1151		if ((head >> DMAR_IQ_SHIFT) == index) {
1152			pr_err("VT-d detected invalid descriptor: "
1153				"low=%llx, high=%llx\n",
1154				(unsigned long long)qi->desc[index].low,
1155				(unsigned long long)qi->desc[index].high);
1156			memcpy(&qi->desc[index], &qi->desc[wait_index],
1157					sizeof(struct qi_desc));
1158			__iommu_flush_cache(iommu, &qi->desc[index],
1159					sizeof(struct qi_desc));
1160			writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1161			return -EINVAL;
1162		}
1163	}
1164
1165	/*
1166	 * If ITE happens, all pending wait_desc commands are aborted.
1167	 * No new descriptors are fetched until the ITE is cleared.
1168	 */
1169	if (fault & DMA_FSTS_ITE) {
1170		head = readl(iommu->reg + DMAR_IQH_REG);
1171		head = ((head >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1172		head |= 1;
1173		tail = readl(iommu->reg + DMAR_IQT_REG);
1174		tail = ((tail >> DMAR_IQ_SHIFT) - 1 + QI_LENGTH) % QI_LENGTH;
1175
1176		writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1177
1178		do {
1179			if (qi->desc_status[head] == QI_IN_USE)
1180				qi->desc_status[head] = QI_ABORT;
1181			head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1182		} while (head != tail);
1183
1184		if (qi->desc_status[wait_index] == QI_ABORT)
1185			return -EAGAIN;
1186	}
1187
1188	if (fault & DMA_FSTS_ICE)
1189		writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1190
1191	return 0;
1192}
1193
1194/*
1195 * Submit the queued invalidation descriptor to the remapping
1196 * hardware unit and wait for its completion.
1197 */
1198int qi_submit_sync(struct qi_desc *desc, struct intel_iommu *iommu)
1199{
1200	int rc;
1201	struct q_inval *qi = iommu->qi;
1202	struct qi_desc *hw, wait_desc;
1203	int wait_index, index;
1204	unsigned long flags;
1205
1206	if (!qi)
1207		return 0;
1208
1209	hw = qi->desc;
1210
1211restart:
1212	rc = 0;
1213
1214	raw_spin_lock_irqsave(&qi->q_lock, flags);
1215	while (qi->free_cnt < 3) {
1216		raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1217		cpu_relax();
1218		raw_spin_lock_irqsave(&qi->q_lock, flags);
1219	}
1220
1221	index = qi->free_head;
1222	wait_index = (index + 1) % QI_LENGTH;
1223
1224	qi->desc_status[index] = qi->desc_status[wait_index] = QI_IN_USE;
1225
1226	hw[index] = *desc;
1227
1228	wait_desc.low = QI_IWD_STATUS_DATA(QI_DONE) |
1229			QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1230	wait_desc.high = virt_to_phys(&qi->desc_status[wait_index]);
1231
1232	hw[wait_index] = wait_desc;
1233
1234	__iommu_flush_cache(iommu, &hw[index], sizeof(struct qi_desc));
1235	__iommu_flush_cache(iommu, &hw[wait_index], sizeof(struct qi_desc));
1236
1237	qi->free_head = (qi->free_head + 2) % QI_LENGTH;
1238	qi->free_cnt -= 2;
1239
1240	/*
1241	 * update the HW tail register indicating the presence of
1242	 * new descriptors.
1243	 */
1244	writel(qi->free_head << DMAR_IQ_SHIFT, iommu->reg + DMAR_IQT_REG);
1245
1246	while (qi->desc_status[wait_index] != QI_DONE) {
1247		/*
1248		 * We will leave the interrupts disabled, to prevent interrupt
1249		 * context to queue another cmd while a cmd is already submitted
1250		 * and waiting for completion on this cpu. This is to avoid
1251		 * a deadlock where the interrupt context can wait indefinitely
1252		 * for free slots in the queue.
1253		 */
1254		rc = qi_check_fault(iommu, index);
1255		if (rc)
1256			break;
1257
1258		raw_spin_unlock(&qi->q_lock);
1259		cpu_relax();
1260		raw_spin_lock(&qi->q_lock);
1261	}
1262
1263	qi->desc_status[index] = QI_DONE;
1264
1265	reclaim_free_desc(qi);
1266	raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1267
1268	if (rc == -EAGAIN)
1269		goto restart;
1270
1271	return rc;
1272}
1273
1274/*
1275 * Flush the global interrupt entry cache.
1276 */
1277void qi_global_iec(struct intel_iommu *iommu)
1278{
1279	struct qi_desc desc;
1280
1281	desc.low = QI_IEC_TYPE;
1282	desc.high = 0;
1283
1284	/* should never fail */
1285	qi_submit_sync(&desc, iommu);
1286}
1287
1288void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1289		      u64 type)
1290{
1291	struct qi_desc desc;
1292
1293	desc.low = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1294			| QI_CC_GRAN(type) | QI_CC_TYPE;
1295	desc.high = 0;
1296
1297	qi_submit_sync(&desc, iommu);
1298}
1299
1300void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1301		    unsigned int size_order, u64 type)
1302{
1303	u8 dw = 0, dr = 0;
1304
1305	struct qi_desc desc;
1306	int ih = 0;
1307
1308	if (cap_write_drain(iommu->cap))
1309		dw = 1;
1310
1311	if (cap_read_drain(iommu->cap))
1312		dr = 1;
1313
1314	desc.low = QI_IOTLB_DID(did) | QI_IOTLB_DR(dr) | QI_IOTLB_DW(dw)
1315		| QI_IOTLB_GRAN(type) | QI_IOTLB_TYPE;
1316	desc.high = QI_IOTLB_ADDR(addr) | QI_IOTLB_IH(ih)
1317		| QI_IOTLB_AM(size_order);
1318
1319	qi_submit_sync(&desc, iommu);
1320}
1321
1322void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 qdep,
1323			u64 addr, unsigned mask)
1324{
1325	struct qi_desc desc;
1326
1327	if (mask) {
1328		BUG_ON(addr & ((1 << (VTD_PAGE_SHIFT + mask)) - 1));
1329		addr |= (1 << (VTD_PAGE_SHIFT + mask - 1)) - 1;
1330		desc.high = QI_DEV_IOTLB_ADDR(addr) | QI_DEV_IOTLB_SIZE;
1331	} else
1332		desc.high = QI_DEV_IOTLB_ADDR(addr);
1333
1334	if (qdep >= QI_DEV_IOTLB_MAX_INVS)
1335		qdep = 0;
1336
1337	desc.low = QI_DEV_IOTLB_SID(sid) | QI_DEV_IOTLB_QDEP(qdep) |
1338		   QI_DIOTLB_TYPE;
1339
1340	qi_submit_sync(&desc, iommu);
1341}
1342
1343/*
1344 * Disable Queued Invalidation interface.
1345 */
1346void dmar_disable_qi(struct intel_iommu *iommu)
1347{
1348	unsigned long flags;
1349	u32 sts;
1350	cycles_t start_time = get_cycles();
1351
1352	if (!ecap_qis(iommu->ecap))
1353		return;
1354
1355	raw_spin_lock_irqsave(&iommu->register_lock, flags);
1356
1357	sts =  readl(iommu->reg + DMAR_GSTS_REG);
1358	if (!(sts & DMA_GSTS_QIES))
1359		goto end;
1360
1361	/*
1362	 * Give a chance to HW to complete the pending invalidation requests.
1363	 */
1364	while ((readl(iommu->reg + DMAR_IQT_REG) !=
1365		readl(iommu->reg + DMAR_IQH_REG)) &&
1366		(DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1367		cpu_relax();
1368
1369	iommu->gcmd &= ~DMA_GCMD_QIE;
1370	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1371
1372	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1373		      !(sts & DMA_GSTS_QIES), sts);
1374end:
1375	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1376}
1377
1378/*
1379 * Enable queued invalidation.
1380 */
1381static void __dmar_enable_qi(struct intel_iommu *iommu)
1382{
1383	u32 sts;
1384	unsigned long flags;
1385	struct q_inval *qi = iommu->qi;
1386
1387	qi->free_head = qi->free_tail = 0;
1388	qi->free_cnt = QI_LENGTH;
1389
1390	raw_spin_lock_irqsave(&iommu->register_lock, flags);
1391
1392	/* write zero to the tail reg */
1393	writel(0, iommu->reg + DMAR_IQT_REG);
1394
1395	dmar_writeq(iommu->reg + DMAR_IQA_REG, virt_to_phys(qi->desc));
1396
1397	iommu->gcmd |= DMA_GCMD_QIE;
1398	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1399
1400	/* Make sure hardware complete it */
1401	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1402
1403	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1404}
1405
1406/*
1407 * Enable Queued Invalidation interface. This is a must to support
1408 * interrupt-remapping. Also used by DMA-remapping, which replaces
1409 * register based IOTLB invalidation.
1410 */
1411int dmar_enable_qi(struct intel_iommu *iommu)
1412{
1413	struct q_inval *qi;
1414	struct page *desc_page;
1415
1416	if (!ecap_qis(iommu->ecap))
1417		return -ENOENT;
1418
1419	/*
1420	 * queued invalidation is already setup and enabled.
1421	 */
1422	if (iommu->qi)
1423		return 0;
1424
1425	iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1426	if (!iommu->qi)
1427		return -ENOMEM;
1428
1429	qi = iommu->qi;
1430
1431
1432	desc_page = alloc_pages_node(iommu->node, GFP_ATOMIC | __GFP_ZERO, 0);
1433	if (!desc_page) {
1434		kfree(qi);
1435		iommu->qi = NULL;
1436		return -ENOMEM;
1437	}
1438
1439	qi->desc = page_address(desc_page);
1440
1441	qi->desc_status = kzalloc(QI_LENGTH * sizeof(int), GFP_ATOMIC);
1442	if (!qi->desc_status) {
1443		free_page((unsigned long) qi->desc);
1444		kfree(qi);
1445		iommu->qi = NULL;
1446		return -ENOMEM;
1447	}
1448
 
 
 
1449	raw_spin_lock_init(&qi->q_lock);
1450
1451	__dmar_enable_qi(iommu);
1452
1453	return 0;
1454}
1455
1456/* iommu interrupt handling. Most stuff are MSI-like. */
1457
1458enum faulttype {
1459	DMA_REMAP,
1460	INTR_REMAP,
1461	UNKNOWN,
1462};
1463
1464static const char *dma_remap_fault_reasons[] =
1465{
1466	"Software",
1467	"Present bit in root entry is clear",
1468	"Present bit in context entry is clear",
1469	"Invalid context entry",
1470	"Access beyond MGAW",
1471	"PTE Write access is not set",
1472	"PTE Read access is not set",
1473	"Next page table ptr is invalid",
1474	"Root table address invalid",
1475	"Context table ptr is invalid",
1476	"non-zero reserved fields in RTP",
1477	"non-zero reserved fields in CTP",
1478	"non-zero reserved fields in PTE",
1479	"PCE for translation request specifies blocking",
1480};
1481
1482static const char *irq_remap_fault_reasons[] =
1483{
1484	"Detected reserved fields in the decoded interrupt-remapped request",
1485	"Interrupt index exceeded the interrupt-remapping table size",
1486	"Present field in the IRTE entry is clear",
1487	"Error accessing interrupt-remapping table pointed by IRTA_REG",
1488	"Detected reserved fields in the IRTE entry",
1489	"Blocked a compatibility format interrupt request",
1490	"Blocked an interrupt request due to source-id verification failure",
1491};
1492
1493static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
 
 
1494{
1495	if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1496					ARRAY_SIZE(irq_remap_fault_reasons))) {
1497		*fault_type = INTR_REMAP;
1498		return irq_remap_fault_reasons[fault_reason - 0x20];
1499	} else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1500		*fault_type = DMA_REMAP;
1501		return dma_remap_fault_reasons[fault_reason];
1502	} else {
1503		*fault_type = UNKNOWN;
1504		return "Unknown";
1505	}
1506}
1507
1508
1509static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1510{
1511	if (iommu->irq == irq)
1512		return DMAR_FECTL_REG;
1513	else if (iommu->pr_irq == irq)
1514		return DMAR_PECTL_REG;
1515	else
1516		BUG();
1517}
1518
1519void dmar_msi_unmask(struct irq_data *data)
1520{
1521	struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1522	int reg = dmar_msi_reg(iommu, data->irq);
1523	unsigned long flag;
1524
1525	/* unmask it */
1526	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1527	writel(0, iommu->reg + reg);
1528	/* Read a reg to force flush the post write */
1529	readl(iommu->reg + reg);
1530	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1531}
1532
1533void dmar_msi_mask(struct irq_data *data)
1534{
1535	struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1536	int reg = dmar_msi_reg(iommu, data->irq);
1537	unsigned long flag;
 
1538
1539	/* mask it */
1540	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1541	writel(DMA_FECTL_IM, iommu->reg + reg);
1542	/* Read a reg to force flush the post write */
1543	readl(iommu->reg + reg);
1544	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1545}
1546
1547void dmar_msi_write(int irq, struct msi_msg *msg)
1548{
1549	struct intel_iommu *iommu = irq_get_handler_data(irq);
1550	int reg = dmar_msi_reg(iommu, irq);
1551	unsigned long flag;
1552
1553	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1554	writel(msg->data, iommu->reg + reg + 4);
1555	writel(msg->address_lo, iommu->reg + reg + 8);
1556	writel(msg->address_hi, iommu->reg + reg + 12);
1557	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1558}
1559
1560void dmar_msi_read(int irq, struct msi_msg *msg)
1561{
1562	struct intel_iommu *iommu = irq_get_handler_data(irq);
1563	int reg = dmar_msi_reg(iommu, irq);
1564	unsigned long flag;
1565
1566	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1567	msg->data = readl(iommu->reg + reg + 4);
1568	msg->address_lo = readl(iommu->reg + reg + 8);
1569	msg->address_hi = readl(iommu->reg + reg + 12);
1570	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1571}
1572
1573static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1574		u8 fault_reason, u16 source_id, unsigned long long addr)
1575{
1576	const char *reason;
1577	int fault_type;
1578
1579	reason = dmar_get_fault_reason(fault_reason, &fault_type);
1580
1581	if (fault_type == INTR_REMAP)
1582		pr_err("INTR-REMAP: Request device [[%02x:%02x.%d] "
1583		       "fault index %llx\n"
1584			"INTR-REMAP:[fault reason %02d] %s\n",
1585			(source_id >> 8), PCI_SLOT(source_id & 0xFF),
1586			PCI_FUNC(source_id & 0xFF), addr >> 48,
1587			fault_reason, reason);
1588	else
1589		pr_err("DMAR:[%s] Request device [%02x:%02x.%d] "
 
1590		       "fault addr %llx \n"
1591		       "DMAR:[fault reason %02d] %s\n",
1592		       (type ? "DMA Read" : "DMA Write"),
1593		       (source_id >> 8), PCI_SLOT(source_id & 0xFF),
1594		       PCI_FUNC(source_id & 0xFF), addr, fault_reason, reason);
1595	return 0;
1596}
1597
1598#define PRIMARY_FAULT_REG_LEN (16)
1599irqreturn_t dmar_fault(int irq, void *dev_id)
1600{
1601	struct intel_iommu *iommu = dev_id;
1602	int reg, fault_index;
1603	u32 fault_status;
1604	unsigned long flag;
1605
1606	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1607	fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1608	if (fault_status)
1609		pr_err("DRHD: handling fault status reg %x\n", fault_status);
 
1610
1611	/* TBD: ignore advanced fault log currently */
1612	if (!(fault_status & DMA_FSTS_PPF))
1613		goto unlock_exit;
1614
1615	fault_index = dma_fsts_fault_record_index(fault_status);
1616	reg = cap_fault_reg_offset(iommu->cap);
1617	while (1) {
1618		u8 fault_reason;
1619		u16 source_id;
1620		u64 guest_addr;
1621		int type;
1622		u32 data;
1623
1624		/* highest 32 bits */
1625		data = readl(iommu->reg + reg +
1626				fault_index * PRIMARY_FAULT_REG_LEN + 12);
1627		if (!(data & DMA_FRCD_F))
1628			break;
1629
1630		fault_reason = dma_frcd_fault_reason(data);
1631		type = dma_frcd_type(data);
1632
1633		data = readl(iommu->reg + reg +
1634				fault_index * PRIMARY_FAULT_REG_LEN + 8);
1635		source_id = dma_frcd_source_id(data);
1636
1637		guest_addr = dmar_readq(iommu->reg + reg +
1638				fault_index * PRIMARY_FAULT_REG_LEN);
1639		guest_addr = dma_frcd_page_addr(guest_addr);
1640		/* clear the fault */
1641		writel(DMA_FRCD_F, iommu->reg + reg +
1642			fault_index * PRIMARY_FAULT_REG_LEN + 12);
1643
1644		raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1645
1646		dmar_fault_do_one(iommu, type, fault_reason,
1647				source_id, guest_addr);
1648
1649		fault_index++;
1650		if (fault_index >= cap_num_fault_regs(iommu->cap))
1651			fault_index = 0;
1652		raw_spin_lock_irqsave(&iommu->register_lock, flag);
1653	}
 
 
 
 
1654
1655	writel(DMA_FSTS_PFO | DMA_FSTS_PPF, iommu->reg + DMAR_FSTS_REG);
1656
1657unlock_exit:
1658	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1659	return IRQ_HANDLED;
1660}
1661
1662int dmar_set_interrupt(struct intel_iommu *iommu)
1663{
1664	int irq, ret;
1665
1666	/*
1667	 * Check if the fault interrupt is already initialized.
1668	 */
1669	if (iommu->irq)
1670		return 0;
1671
1672	irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
1673	if (irq > 0) {
1674		iommu->irq = irq;
1675	} else {
1676		pr_err("No free IRQ vectors\n");
1677		return -EINVAL;
1678	}
1679
 
 
 
 
 
 
 
 
 
 
 
1680	ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
1681	if (ret)
1682		pr_err("Can't request irq\n");
1683	return ret;
1684}
1685
1686int __init enable_drhd_fault_handling(void)
1687{
1688	struct dmar_drhd_unit *drhd;
1689	struct intel_iommu *iommu;
1690
1691	/*
1692	 * Enable fault control interrupt.
1693	 */
1694	for_each_iommu(iommu, drhd) {
1695		u32 fault_status;
1696		int ret = dmar_set_interrupt(iommu);
 
1697
1698		if (ret) {
1699			pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
 
1700			       (unsigned long long)drhd->reg_base_addr, ret);
1701			return -1;
1702		}
1703
1704		/*
1705		 * Clear any previous faults.
1706		 */
1707		dmar_fault(iommu->irq, iommu);
1708		fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1709		writel(fault_status, iommu->reg + DMAR_FSTS_REG);
1710	}
1711
1712	return 0;
1713}
1714
1715/*
1716 * Re-enable Queued Invalidation interface.
1717 */
1718int dmar_reenable_qi(struct intel_iommu *iommu)
1719{
1720	if (!ecap_qis(iommu->ecap))
1721		return -ENOENT;
1722
1723	if (!iommu->qi)
1724		return -ENOENT;
1725
1726	/*
1727	 * First disable queued invalidation.
1728	 */
1729	dmar_disable_qi(iommu);
1730	/*
1731	 * Then enable queued invalidation again. Since there is no pending
1732	 * invalidation requests now, it's safe to re-enable queued
1733	 * invalidation.
1734	 */
1735	__dmar_enable_qi(iommu);
1736
1737	return 0;
1738}
1739
1740/*
1741 * Check interrupt remapping support in DMAR table description.
1742 */
1743int __init dmar_ir_support(void)
1744{
1745	struct acpi_table_dmar *dmar;
1746	dmar = (struct acpi_table_dmar *)dmar_tbl;
1747	if (!dmar)
1748		return 0;
1749	return dmar->flags & 0x1;
1750}
1751
1752/* Check whether DMAR units are in use */
1753static inline bool dmar_in_use(void)
1754{
1755	return irq_remapping_enabled || intel_iommu_enabled;
1756}
1757
1758static int __init dmar_free_unused_resources(void)
1759{
1760	struct dmar_drhd_unit *dmaru, *dmaru_n;
1761
1762	if (dmar_in_use())
1763		return 0;
1764
1765	if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
1766		bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
1767
1768	down_write(&dmar_global_lock);
1769	list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
1770		list_del(&dmaru->list);
1771		dmar_free_drhd(dmaru);
1772	}
1773	up_write(&dmar_global_lock);
1774
1775	return 0;
1776}
1777
1778late_initcall(dmar_free_unused_resources);
1779IOMMU_INIT_POST(detect_intel_iommu);
1780
1781/*
1782 * DMAR Hotplug Support
1783 * For more details, please refer to Intel(R) Virtualization Technology
1784 * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
1785 * "Remapping Hardware Unit Hot Plug".
1786 */
1787static u8 dmar_hp_uuid[] = {
1788	/* 0000 */    0xA6, 0xA3, 0xC1, 0xD8, 0x9B, 0xBE, 0x9B, 0x4C,
1789	/* 0008 */    0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF
1790};
1791
1792/*
1793 * Currently there's only one revision and BIOS will not check the revision id,
1794 * so use 0 for safety.
1795 */
1796#define	DMAR_DSM_REV_ID			0
1797#define	DMAR_DSM_FUNC_DRHD		1
1798#define	DMAR_DSM_FUNC_ATSR		2
1799#define	DMAR_DSM_FUNC_RHSA		3
1800
1801static inline bool dmar_detect_dsm(acpi_handle handle, int func)
1802{
1803	return acpi_check_dsm(handle, dmar_hp_uuid, DMAR_DSM_REV_ID, 1 << func);
1804}
1805
1806static int dmar_walk_dsm_resource(acpi_handle handle, int func,
1807				  dmar_res_handler_t handler, void *arg)
1808{
1809	int ret = -ENODEV;
1810	union acpi_object *obj;
1811	struct acpi_dmar_header *start;
1812	struct dmar_res_callback callback;
1813	static int res_type[] = {
1814		[DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
1815		[DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
1816		[DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
1817	};
1818
1819	if (!dmar_detect_dsm(handle, func))
1820		return 0;
1821
1822	obj = acpi_evaluate_dsm_typed(handle, dmar_hp_uuid, DMAR_DSM_REV_ID,
1823				      func, NULL, ACPI_TYPE_BUFFER);
1824	if (!obj)
1825		return -ENODEV;
1826
1827	memset(&callback, 0, sizeof(callback));
1828	callback.cb[res_type[func]] = handler;
1829	callback.arg[res_type[func]] = arg;
1830	start = (struct acpi_dmar_header *)obj->buffer.pointer;
1831	ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
1832
1833	ACPI_FREE(obj);
1834
1835	return ret;
1836}
1837
1838static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
1839{
1840	int ret;
1841	struct dmar_drhd_unit *dmaru;
1842
1843	dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1844	if (!dmaru)
1845		return -ENODEV;
1846
1847	ret = dmar_ir_hotplug(dmaru, true);
1848	if (ret == 0)
1849		ret = dmar_iommu_hotplug(dmaru, true);
1850
1851	return ret;
1852}
1853
1854static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
1855{
1856	int i, ret;
1857	struct device *dev;
1858	struct dmar_drhd_unit *dmaru;
1859
1860	dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1861	if (!dmaru)
1862		return 0;
1863
1864	/*
1865	 * All PCI devices managed by this unit should have been destroyed.
1866	 */
1867	if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt)
1868		for_each_active_dev_scope(dmaru->devices,
1869					  dmaru->devices_cnt, i, dev)
1870			return -EBUSY;
1871
1872	ret = dmar_ir_hotplug(dmaru, false);
1873	if (ret == 0)
1874		ret = dmar_iommu_hotplug(dmaru, false);
1875
1876	return ret;
1877}
1878
1879static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
1880{
1881	struct dmar_drhd_unit *dmaru;
1882
1883	dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
1884	if (dmaru) {
1885		list_del_rcu(&dmaru->list);
1886		synchronize_rcu();
1887		dmar_free_drhd(dmaru);
1888	}
1889
1890	return 0;
1891}
1892
1893static int dmar_hotplug_insert(acpi_handle handle)
1894{
1895	int ret;
1896	int drhd_count = 0;
1897
1898	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1899				     &dmar_validate_one_drhd, (void *)1);
1900	if (ret)
1901		goto out;
1902
1903	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1904				     &dmar_parse_one_drhd, (void *)&drhd_count);
1905	if (ret == 0 && drhd_count == 0) {
1906		pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
1907		goto out;
1908	} else if (ret) {
1909		goto release_drhd;
1910	}
1911
1912	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
1913				     &dmar_parse_one_rhsa, NULL);
1914	if (ret)
1915		goto release_drhd;
1916
1917	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1918				     &dmar_parse_one_atsr, NULL);
1919	if (ret)
1920		goto release_atsr;
1921
1922	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1923				     &dmar_hp_add_drhd, NULL);
1924	if (!ret)
1925		return 0;
1926
1927	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1928			       &dmar_hp_remove_drhd, NULL);
1929release_atsr:
1930	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1931			       &dmar_release_one_atsr, NULL);
1932release_drhd:
1933	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1934			       &dmar_hp_release_drhd, NULL);
1935out:
1936	return ret;
1937}
1938
1939static int dmar_hotplug_remove(acpi_handle handle)
1940{
1941	int ret;
1942
1943	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1944				     &dmar_check_one_atsr, NULL);
1945	if (ret)
1946		return ret;
1947
1948	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1949				     &dmar_hp_remove_drhd, NULL);
1950	if (ret == 0) {
1951		WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
1952					       &dmar_release_one_atsr, NULL));
1953		WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1954					       &dmar_hp_release_drhd, NULL));
1955	} else {
1956		dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
1957				       &dmar_hp_add_drhd, NULL);
1958	}
1959
1960	return ret;
1961}
1962
1963static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
1964				       void *context, void **retval)
1965{
1966	acpi_handle *phdl = retval;
1967
1968	if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1969		*phdl = handle;
1970		return AE_CTRL_TERMINATE;
1971	}
1972
1973	return AE_OK;
1974}
1975
1976static int dmar_device_hotplug(acpi_handle handle, bool insert)
1977{
1978	int ret;
1979	acpi_handle tmp = NULL;
1980	acpi_status status;
1981
1982	if (!dmar_in_use())
1983		return 0;
1984
1985	if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
1986		tmp = handle;
1987	} else {
1988		status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
1989					     ACPI_UINT32_MAX,
1990					     dmar_get_dsm_handle,
1991					     NULL, NULL, &tmp);
1992		if (ACPI_FAILURE(status)) {
1993			pr_warn("Failed to locate _DSM method.\n");
1994			return -ENXIO;
1995		}
1996	}
1997	if (tmp == NULL)
1998		return 0;
1999
2000	down_write(&dmar_global_lock);
2001	if (insert)
2002		ret = dmar_hotplug_insert(tmp);
2003	else
2004		ret = dmar_hotplug_remove(tmp);
2005	up_write(&dmar_global_lock);
2006
2007	return ret;
2008}
2009
2010int dmar_device_add(acpi_handle handle)
2011{
2012	return dmar_device_hotplug(handle, true);
2013}
2014
2015int dmar_device_remove(acpi_handle handle)
2016{
2017	return dmar_device_hotplug(handle, false);
2018}