1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * Copyright (c) 2006, Intel Corporation.
   4 *
   5 * Copyright (C) 2006-2008 Intel Corporation
   6 * Author: Ashok Raj <ashok.raj@intel.com>
   7 * Author: Shaohua Li <shaohua.li@intel.com>
   8 * Author: Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>
   9 *
  10 * This file implements early detection/parsing of Remapping Devices
  11 * reported to OS through BIOS via DMA remapping reporting (DMAR) ACPI
  12 * tables.
  13 *
  14 * These routines are used by both DMA-remapping and Interrupt-remapping
  15 */
  16
  17#define pr_fmt(fmt)     "DMAR: " fmt
  18
  19#include <linux/pci.h>
  20#include <linux/dmar.h>
  21#include <linux/iova.h>
 
  22#include <linux/timer.h>
  23#include <linux/irq.h>
  24#include <linux/interrupt.h>
  25#include <linux/tboot.h>
  26#include <linux/dmi.h>
  27#include <linux/slab.h>
  28#include <linux/iommu.h>
  29#include <linux/numa.h>
  30#include <linux/limits.h>
  31#include <asm/irq_remapping.h>
 
  32
  33#include "iommu.h"
  34#include "../irq_remapping.h"
  35#include "../iommu-pages.h"
  36#include "perf.h"
  37#include "trace.h"
  38#include "perfmon.h"
  39
  40typedef int (*dmar_res_handler_t)(struct acpi_dmar_header *, void *);
  41struct dmar_res_callback {
  42	dmar_res_handler_t	cb[ACPI_DMAR_TYPE_RESERVED];
  43	void			*arg[ACPI_DMAR_TYPE_RESERVED];
  44	bool			ignore_unhandled;
  45	bool			print_entry;
  46};
  47
  48/*
  49 * Assumptions:
  50 * 1) The hotplug framework guarentees that DMAR unit will be hot-added
  51 *    before IO devices managed by that unit.
  52 * 2) The hotplug framework guarantees that DMAR unit will be hot-removed
  53 *    after IO devices managed by that unit.
  54 * 3) Hotplug events are rare.
  55 *
  56 * Locking rules for DMA and interrupt remapping related global data structures:
  57 * 1) Use dmar_global_lock in process context
  58 * 2) Use RCU in interrupt context
  59 */
  60DECLARE_RWSEM(dmar_global_lock);
  61LIST_HEAD(dmar_drhd_units);
  62
  63struct acpi_table_header * __initdata dmar_tbl;
  64static int dmar_dev_scope_status = 1;
  65static DEFINE_IDA(dmar_seq_ids);
  66
  67static int alloc_iommu(struct dmar_drhd_unit *drhd);
  68static void free_iommu(struct intel_iommu *iommu);
  69
 
 
  70static void dmar_register_drhd_unit(struct dmar_drhd_unit *drhd)
  71{
  72	/*
  73	 * add INCLUDE_ALL at the tail, so scan the list will find it at
  74	 * the very end.
  75	 */
  76	if (drhd->include_all)
  77		list_add_tail_rcu(&drhd->list, &dmar_drhd_units);
  78	else
  79		list_add_rcu(&drhd->list, &dmar_drhd_units);
  80}
  81
  82void *dmar_alloc_dev_scope(void *start, void *end, int *cnt)
  83{
  84	struct acpi_dmar_device_scope *scope;
  85
  86	*cnt = 0;
  87	while (start < end) {
  88		scope = start;
  89		if (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_NAMESPACE ||
  90		    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT ||
  91		    scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE)
  92			(*cnt)++;
  93		else if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_IOAPIC &&
  94			scope->entry_type != ACPI_DMAR_SCOPE_TYPE_HPET) {
  95			pr_warn("Unsupported device scope\n");
  96		}
  97		start += scope->length;
  98	}
  99	if (*cnt == 0)
 100		return NULL;
 101
 102	return kcalloc(*cnt, sizeof(struct dmar_dev_scope), GFP_KERNEL);
 103}
 104
 105void dmar_free_dev_scope(struct dmar_dev_scope **devices, int *cnt)
 106{
 107	int i;
 108	struct device *tmp_dev;
 109
 110	if (*devices && *cnt) {
 111		for_each_active_dev_scope(*devices, *cnt, i, tmp_dev)
 112			put_device(tmp_dev);
 113		kfree(*devices);
 114	}
 115
 116	*devices = NULL;
 117	*cnt = 0;
 118}
 119
 120/* Optimize out kzalloc()/kfree() for normal cases */
 121static char dmar_pci_notify_info_buf[64];
 122
 123static struct dmar_pci_notify_info *
 124dmar_alloc_pci_notify_info(struct pci_dev *dev, unsigned long event)
 125{
 126	int level = 0;
 127	size_t size;
 128	struct pci_dev *tmp;
 129	struct dmar_pci_notify_info *info;
 130
 
 
 131	/*
 132	 * Ignore devices that have a domain number higher than what can
 133	 * be looked up in DMAR, e.g. VMD subdevices with domain 0x10000
 134	 */
 135	if (pci_domain_nr(dev->bus) > U16_MAX)
 136		return NULL;
 137
 138	/* Only generate path[] for device addition event */
 139	if (event == BUS_NOTIFY_ADD_DEVICE)
 140		for (tmp = dev; tmp; tmp = tmp->bus->self)
 141			level++;
 142
 143	size = struct_size(info, path, level);
 144	if (size <= sizeof(dmar_pci_notify_info_buf)) {
 145		info = (struct dmar_pci_notify_info *)dmar_pci_notify_info_buf;
 146	} else {
 147		info = kzalloc(size, GFP_KERNEL);
 148		if (!info) {
 
 
 149			if (dmar_dev_scope_status == 0)
 150				dmar_dev_scope_status = -ENOMEM;
 151			return NULL;
 152		}
 153	}
 154
 155	info->event = event;
 156	info->dev = dev;
 157	info->seg = pci_domain_nr(dev->bus);
 158	info->level = level;
 159	if (event == BUS_NOTIFY_ADD_DEVICE) {
 160		for (tmp = dev; tmp; tmp = tmp->bus->self) {
 161			level--;
 162			info->path[level].bus = tmp->bus->number;
 163			info->path[level].device = PCI_SLOT(tmp->devfn);
 164			info->path[level].function = PCI_FUNC(tmp->devfn);
 165			if (pci_is_root_bus(tmp->bus))
 166				info->bus = tmp->bus->number;
 167		}
 168	}
 169
 170	return info;
 171}
 172
 173static inline void dmar_free_pci_notify_info(struct dmar_pci_notify_info *info)
 174{
 175	if ((void *)info != dmar_pci_notify_info_buf)
 176		kfree(info);
 177}
 178
 179static bool dmar_match_pci_path(struct dmar_pci_notify_info *info, int bus,
 180				struct acpi_dmar_pci_path *path, int count)
 181{
 182	int i;
 183
 184	if (info->bus != bus)
 185		goto fallback;
 186	if (info->level != count)
 187		goto fallback;
 188
 189	for (i = 0; i < count; i++) {
 190		if (path[i].device != info->path[i].device ||
 191		    path[i].function != info->path[i].function)
 192			goto fallback;
 193	}
 194
 195	return true;
 196
 197fallback:
 198
 199	if (count != 1)
 200		return false;
 201
 202	i = info->level - 1;
 203	if (bus              == info->path[i].bus &&
 204	    path[0].device   == info->path[i].device &&
 205	    path[0].function == info->path[i].function) {
 206		pr_info(FW_BUG "RMRR entry for device %02x:%02x.%x is broken - applying workaround\n",
 207			bus, path[0].device, path[0].function);
 208		return true;
 209	}
 210
 211	return false;
 212}
 213
 214/* Return: > 0 if match found, 0 if no match found, < 0 if error happens */
 215int dmar_insert_dev_scope(struct dmar_pci_notify_info *info,
 216			  void *start, void*end, u16 segment,
 217			  struct dmar_dev_scope *devices,
 218			  int devices_cnt)
 219{
 220	int i, level;
 221	struct device *tmp, *dev = &info->dev->dev;
 222	struct acpi_dmar_device_scope *scope;
 223	struct acpi_dmar_pci_path *path;
 224
 225	if (segment != info->seg)
 226		return 0;
 227
 228	for (; start < end; start += scope->length) {
 229		scope = start;
 230		if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 231		    scope->entry_type != ACPI_DMAR_SCOPE_TYPE_BRIDGE)
 232			continue;
 233
 234		path = (struct acpi_dmar_pci_path *)(scope + 1);
 235		level = (scope->length - sizeof(*scope)) / sizeof(*path);
 236		if (!dmar_match_pci_path(info, scope->bus, path, level))
 237			continue;
 238
 239		/*
 240		 * We expect devices with endpoint scope to have normal PCI
 241		 * headers, and devices with bridge scope to have bridge PCI
 242		 * headers.  However PCI NTB devices may be listed in the
 243		 * DMAR table with bridge scope, even though they have a
 244		 * normal PCI header.  NTB devices are identified by class
 245		 * "BRIDGE_OTHER" (0680h) - we don't declare a socpe mismatch
 246		 * for this special case.
 247		 */
 248		if ((scope->entry_type == ACPI_DMAR_SCOPE_TYPE_ENDPOINT &&
 249		     info->dev->hdr_type != PCI_HEADER_TYPE_NORMAL) ||
 250		    (scope->entry_type == ACPI_DMAR_SCOPE_TYPE_BRIDGE &&
 251		     (info->dev->hdr_type == PCI_HEADER_TYPE_NORMAL &&
 252		      info->dev->class >> 16 != PCI_BASE_CLASS_BRIDGE))) {
 253			pr_warn("Device scope type does not match for %s\n",
 254				pci_name(info->dev));
 255			return -EINVAL;
 256		}
 257
 258		for_each_dev_scope(devices, devices_cnt, i, tmp)
 259			if (tmp == NULL) {
 260				devices[i].bus = info->dev->bus->number;
 261				devices[i].devfn = info->dev->devfn;
 262				rcu_assign_pointer(devices[i].dev,
 263						   get_device(dev));
 264				return 1;
 265			}
 266		if (WARN_ON(i >= devices_cnt))
 267			return -EINVAL;
 268	}
 269
 270	return 0;
 271}
 272
 273int dmar_remove_dev_scope(struct dmar_pci_notify_info *info, u16 segment,
 274			  struct dmar_dev_scope *devices, int count)
 275{
 276	int index;
 277	struct device *tmp;
 278
 279	if (info->seg != segment)
 280		return 0;
 281
 282	for_each_active_dev_scope(devices, count, index, tmp)
 283		if (tmp == &info->dev->dev) {
 284			RCU_INIT_POINTER(devices[index].dev, NULL);
 285			synchronize_rcu();
 286			put_device(tmp);
 287			return 1;
 288		}
 289
 290	return 0;
 291}
 292
 293static int dmar_pci_bus_add_dev(struct dmar_pci_notify_info *info)
 294{
 295	int ret = 0;
 296	struct dmar_drhd_unit *dmaru;
 297	struct acpi_dmar_hardware_unit *drhd;
 298
 299	for_each_drhd_unit(dmaru) {
 300		if (dmaru->include_all)
 301			continue;
 302
 303		drhd = container_of(dmaru->hdr,
 304				    struct acpi_dmar_hardware_unit, header);
 305		ret = dmar_insert_dev_scope(info, (void *)(drhd + 1),
 306				((void *)drhd) + drhd->header.length,
 307				dmaru->segment,
 308				dmaru->devices, dmaru->devices_cnt);
 309		if (ret)
 310			break;
 311	}
 312	if (ret >= 0)
 313		ret = dmar_iommu_notify_scope_dev(info);
 314	if (ret < 0 && dmar_dev_scope_status == 0)
 315		dmar_dev_scope_status = ret;
 316
 317	if (ret >= 0)
 318		intel_irq_remap_add_device(info);
 319
 320	return ret;
 321}
 322
 323static void  dmar_pci_bus_del_dev(struct dmar_pci_notify_info *info)
 324{
 325	struct dmar_drhd_unit *dmaru;
 326
 327	for_each_drhd_unit(dmaru)
 328		if (dmar_remove_dev_scope(info, dmaru->segment,
 329			dmaru->devices, dmaru->devices_cnt))
 330			break;
 331	dmar_iommu_notify_scope_dev(info);
 332}
 333
 334static inline void vf_inherit_msi_domain(struct pci_dev *pdev)
 335{
 336	struct pci_dev *physfn = pci_physfn(pdev);
 337
 338	dev_set_msi_domain(&pdev->dev, dev_get_msi_domain(&physfn->dev));
 339}
 340
 341static int dmar_pci_bus_notifier(struct notifier_block *nb,
 342				 unsigned long action, void *data)
 343{
 344	struct pci_dev *pdev = to_pci_dev(data);
 345	struct dmar_pci_notify_info *info;
 346
 347	/* Only care about add/remove events for physical functions.
 348	 * For VFs we actually do the lookup based on the corresponding
 349	 * PF in device_to_iommu() anyway. */
 350	if (pdev->is_virtfn) {
 351		/*
 352		 * Ensure that the VF device inherits the irq domain of the
 353		 * PF device. Ideally the device would inherit the domain
 354		 * from the bus, but DMAR can have multiple units per bus
 355		 * which makes this impossible. The VF 'bus' could inherit
 356		 * from the PF device, but that's yet another x86'sism to
 357		 * inflict on everybody else.
 358		 */
 359		if (action == BUS_NOTIFY_ADD_DEVICE)
 360			vf_inherit_msi_domain(pdev);
 361		return NOTIFY_DONE;
 362	}
 363
 364	if (action != BUS_NOTIFY_ADD_DEVICE &&
 365	    action != BUS_NOTIFY_REMOVED_DEVICE)
 366		return NOTIFY_DONE;
 367
 368	info = dmar_alloc_pci_notify_info(pdev, action);
 369	if (!info)
 370		return NOTIFY_DONE;
 371
 372	down_write(&dmar_global_lock);
 373	if (action == BUS_NOTIFY_ADD_DEVICE)
 374		dmar_pci_bus_add_dev(info);
 375	else if (action == BUS_NOTIFY_REMOVED_DEVICE)
 376		dmar_pci_bus_del_dev(info);
 377	up_write(&dmar_global_lock);
 378
 379	dmar_free_pci_notify_info(info);
 380
 381	return NOTIFY_OK;
 382}
 383
 384static struct notifier_block dmar_pci_bus_nb = {
 385	.notifier_call = dmar_pci_bus_notifier,
 386	.priority = 1,
 387};
 388
 389static struct dmar_drhd_unit *
 390dmar_find_dmaru(struct acpi_dmar_hardware_unit *drhd)
 391{
 392	struct dmar_drhd_unit *dmaru;
 393
 394	list_for_each_entry_rcu(dmaru, &dmar_drhd_units, list,
 395				dmar_rcu_check())
 396		if (dmaru->segment == drhd->segment &&
 397		    dmaru->reg_base_addr == drhd->address)
 398			return dmaru;
 399
 400	return NULL;
 401}
 402
 403/*
 404 * dmar_parse_one_drhd - parses exactly one DMA remapping hardware definition
 405 * structure which uniquely represent one DMA remapping hardware unit
 406 * present in the platform
 407 */
 408static int dmar_parse_one_drhd(struct acpi_dmar_header *header, void *arg)
 409{
 410	struct acpi_dmar_hardware_unit *drhd;
 411	struct dmar_drhd_unit *dmaru;
 412	int ret;
 413
 414	drhd = (struct acpi_dmar_hardware_unit *)header;
 415	dmaru = dmar_find_dmaru(drhd);
 416	if (dmaru)
 417		goto out;
 418
 419	dmaru = kzalloc(sizeof(*dmaru) + header->length, GFP_KERNEL);
 420	if (!dmaru)
 421		return -ENOMEM;
 422
 423	/*
 424	 * If header is allocated from slab by ACPI _DSM method, we need to
 425	 * copy the content because the memory buffer will be freed on return.
 426	 */
 427	dmaru->hdr = (void *)(dmaru + 1);
 428	memcpy(dmaru->hdr, header, header->length);
 429	dmaru->reg_base_addr = drhd->address;
 430	dmaru->segment = drhd->segment;
 431	/* The size of the register set is 2 ^ N 4 KB pages. */
 432	dmaru->reg_size = 1UL << (drhd->size + 12);
 433	dmaru->include_all = drhd->flags & 0x1; /* BIT0: INCLUDE_ALL */
 434	dmaru->devices = dmar_alloc_dev_scope((void *)(drhd + 1),
 435					      ((void *)drhd) + drhd->header.length,
 436					      &dmaru->devices_cnt);
 437	if (dmaru->devices_cnt && dmaru->devices == NULL) {
 438		kfree(dmaru);
 439		return -ENOMEM;
 440	}
 441
 442	ret = alloc_iommu(dmaru);
 443	if (ret) {
 444		dmar_free_dev_scope(&dmaru->devices,
 445				    &dmaru->devices_cnt);
 446		kfree(dmaru);
 447		return ret;
 448	}
 449	dmar_register_drhd_unit(dmaru);
 450
 451out:
 452	if (arg)
 453		(*(int *)arg)++;
 454
 455	return 0;
 456}
 457
 458static void dmar_free_drhd(struct dmar_drhd_unit *dmaru)
 459{
 460	if (dmaru->devices && dmaru->devices_cnt)
 461		dmar_free_dev_scope(&dmaru->devices, &dmaru->devices_cnt);
 462	if (dmaru->iommu)
 463		free_iommu(dmaru->iommu);
 464	kfree(dmaru);
 465}
 466
 467static int __init dmar_parse_one_andd(struct acpi_dmar_header *header,
 468				      void *arg)
 469{
 470	struct acpi_dmar_andd *andd = (void *)header;
 471
 472	/* Check for NUL termination within the designated length */
 473	if (strnlen(andd->device_name, header->length - 8) == header->length - 8) {
 474		pr_warn(FW_BUG
 475			   "Your BIOS is broken; ANDD object name is not NUL-terminated\n"
 476			   "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 477			   dmi_get_system_info(DMI_BIOS_VENDOR),
 478			   dmi_get_system_info(DMI_BIOS_VERSION),
 479			   dmi_get_system_info(DMI_PRODUCT_VERSION));
 480		add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 481		return -EINVAL;
 482	}
 483	pr_info("ANDD device: %x name: %s\n", andd->device_number,
 484		andd->device_name);
 485
 486	return 0;
 487}
 488
 489#ifdef CONFIG_ACPI_NUMA
 490static int dmar_parse_one_rhsa(struct acpi_dmar_header *header, void *arg)
 491{
 492	struct acpi_dmar_rhsa *rhsa;
 493	struct dmar_drhd_unit *drhd;
 494
 495	rhsa = (struct acpi_dmar_rhsa *)header;
 496	for_each_drhd_unit(drhd) {
 497		if (drhd->reg_base_addr == rhsa->base_address) {
 498			int node = pxm_to_node(rhsa->proximity_domain);
 499
 500			if (node != NUMA_NO_NODE && !node_online(node))
 501				node = NUMA_NO_NODE;
 502			drhd->iommu->node = node;
 503			return 0;
 504		}
 505	}
 506	pr_warn(FW_BUG
 507		"Your BIOS is broken; RHSA refers to non-existent DMAR unit at %llx\n"
 508		"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 509		rhsa->base_address,
 510		dmi_get_system_info(DMI_BIOS_VENDOR),
 511		dmi_get_system_info(DMI_BIOS_VERSION),
 512		dmi_get_system_info(DMI_PRODUCT_VERSION));
 513	add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 514
 515	return 0;
 516}
 517#else
 518#define	dmar_parse_one_rhsa		dmar_res_noop
 519#endif
 520
 521static void
 522dmar_table_print_dmar_entry(struct acpi_dmar_header *header)
 523{
 524	struct acpi_dmar_hardware_unit *drhd;
 525	struct acpi_dmar_reserved_memory *rmrr;
 526	struct acpi_dmar_atsr *atsr;
 527	struct acpi_dmar_rhsa *rhsa;
 528	struct acpi_dmar_satc *satc;
 529
 530	switch (header->type) {
 531	case ACPI_DMAR_TYPE_HARDWARE_UNIT:
 532		drhd = container_of(header, struct acpi_dmar_hardware_unit,
 533				    header);
 534		pr_info("DRHD base: %#016Lx flags: %#x\n",
 535			(unsigned long long)drhd->address, drhd->flags);
 536		break;
 537	case ACPI_DMAR_TYPE_RESERVED_MEMORY:
 538		rmrr = container_of(header, struct acpi_dmar_reserved_memory,
 539				    header);
 540		pr_info("RMRR base: %#016Lx end: %#016Lx\n",
 541			(unsigned long long)rmrr->base_address,
 542			(unsigned long long)rmrr->end_address);
 543		break;
 544	case ACPI_DMAR_TYPE_ROOT_ATS:
 545		atsr = container_of(header, struct acpi_dmar_atsr, header);
 546		pr_info("ATSR flags: %#x\n", atsr->flags);
 547		break;
 548	case ACPI_DMAR_TYPE_HARDWARE_AFFINITY:
 549		rhsa = container_of(header, struct acpi_dmar_rhsa, header);
 550		pr_info("RHSA base: %#016Lx proximity domain: %#x\n",
 551		       (unsigned long long)rhsa->base_address,
 552		       rhsa->proximity_domain);
 553		break;
 554	case ACPI_DMAR_TYPE_NAMESPACE:
 555		/* We don't print this here because we need to sanity-check
 556		   it first. So print it in dmar_parse_one_andd() instead. */
 557		break;
 558	case ACPI_DMAR_TYPE_SATC:
 559		satc = container_of(header, struct acpi_dmar_satc, header);
 560		pr_info("SATC flags: 0x%x\n", satc->flags);
 561		break;
 562	}
 563}
 564
 565/**
 566 * dmar_table_detect - checks to see if the platform supports DMAR devices
 567 */
 568static int __init dmar_table_detect(void)
 569{
 570	acpi_status status = AE_OK;
 571
 572	/* if we could find DMAR table, then there are DMAR devices */
 573	status = acpi_get_table(ACPI_SIG_DMAR, 0, &dmar_tbl);
 574
 575	if (ACPI_SUCCESS(status) && !dmar_tbl) {
 576		pr_warn("Unable to map DMAR\n");
 577		status = AE_NOT_FOUND;
 578	}
 579
 580	return ACPI_SUCCESS(status) ? 0 : -ENOENT;
 581}
 582
 583static int dmar_walk_remapping_entries(struct acpi_dmar_header *start,
 584				       size_t len, struct dmar_res_callback *cb)
 585{
 586	struct acpi_dmar_header *iter, *next;
 587	struct acpi_dmar_header *end = ((void *)start) + len;
 588
 589	for (iter = start; iter < end; iter = next) {
 590		next = (void *)iter + iter->length;
 591		if (iter->length == 0) {
 592			/* Avoid looping forever on bad ACPI tables */
 593			pr_debug(FW_BUG "Invalid 0-length structure\n");
 594			break;
 595		} else if (next > end) {
 596			/* Avoid passing table end */
 597			pr_warn(FW_BUG "Record passes table end\n");
 598			return -EINVAL;
 599		}
 600
 601		if (cb->print_entry)
 602			dmar_table_print_dmar_entry(iter);
 603
 604		if (iter->type >= ACPI_DMAR_TYPE_RESERVED) {
 605			/* continue for forward compatibility */
 606			pr_debug("Unknown DMAR structure type %d\n",
 607				 iter->type);
 608		} else if (cb->cb[iter->type]) {
 609			int ret;
 610
 611			ret = cb->cb[iter->type](iter, cb->arg[iter->type]);
 612			if (ret)
 613				return ret;
 614		} else if (!cb->ignore_unhandled) {
 615			pr_warn("No handler for DMAR structure type %d\n",
 616				iter->type);
 617			return -EINVAL;
 618		}
 619	}
 620
 621	return 0;
 622}
 623
 624static inline int dmar_walk_dmar_table(struct acpi_table_dmar *dmar,
 625				       struct dmar_res_callback *cb)
 626{
 627	return dmar_walk_remapping_entries((void *)(dmar + 1),
 628			dmar->header.length - sizeof(*dmar), cb);
 629}
 630
 631/**
 632 * parse_dmar_table - parses the DMA reporting table
 633 */
 634static int __init
 635parse_dmar_table(void)
 636{
 637	struct acpi_table_dmar *dmar;
 638	int drhd_count = 0;
 639	int ret;
 640	struct dmar_res_callback cb = {
 641		.print_entry = true,
 642		.ignore_unhandled = true,
 643		.arg[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &drhd_count,
 644		.cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_parse_one_drhd,
 645		.cb[ACPI_DMAR_TYPE_RESERVED_MEMORY] = &dmar_parse_one_rmrr,
 646		.cb[ACPI_DMAR_TYPE_ROOT_ATS] = &dmar_parse_one_atsr,
 647		.cb[ACPI_DMAR_TYPE_HARDWARE_AFFINITY] = &dmar_parse_one_rhsa,
 648		.cb[ACPI_DMAR_TYPE_NAMESPACE] = &dmar_parse_one_andd,
 649		.cb[ACPI_DMAR_TYPE_SATC] = &dmar_parse_one_satc,
 650	};
 651
 652	/*
 653	 * Do it again, earlier dmar_tbl mapping could be mapped with
 654	 * fixed map.
 655	 */
 656	dmar_table_detect();
 657
 658	/*
 659	 * ACPI tables may not be DMA protected by tboot, so use DMAR copy
 660	 * SINIT saved in SinitMleData in TXT heap (which is DMA protected)
 661	 */
 662	dmar_tbl = tboot_get_dmar_table(dmar_tbl);
 663
 664	dmar = (struct acpi_table_dmar *)dmar_tbl;
 665	if (!dmar)
 666		return -ENODEV;
 667
 668	if (dmar->width < PAGE_SHIFT - 1) {
 669		pr_warn("Invalid DMAR haw\n");
 670		return -EINVAL;
 671	}
 672
 673	pr_info("Host address width %d\n", dmar->width + 1);
 674	ret = dmar_walk_dmar_table(dmar, &cb);
 675	if (ret == 0 && drhd_count == 0)
 676		pr_warn(FW_BUG "No DRHD structure found in DMAR table\n");
 677
 678	return ret;
 679}
 680
 681static int dmar_pci_device_match(struct dmar_dev_scope devices[],
 682				 int cnt, struct pci_dev *dev)
 683{
 684	int index;
 685	struct device *tmp;
 686
 687	while (dev) {
 688		for_each_active_dev_scope(devices, cnt, index, tmp)
 689			if (dev_is_pci(tmp) && dev == to_pci_dev(tmp))
 690				return 1;
 691
 692		/* Check our parent */
 693		dev = dev->bus->self;
 694	}
 695
 696	return 0;
 697}
 698
 699struct dmar_drhd_unit *
 700dmar_find_matched_drhd_unit(struct pci_dev *dev)
 701{
 702	struct dmar_drhd_unit *dmaru;
 703	struct acpi_dmar_hardware_unit *drhd;
 704
 705	dev = pci_physfn(dev);
 706
 707	rcu_read_lock();
 708	for_each_drhd_unit(dmaru) {
 709		drhd = container_of(dmaru->hdr,
 710				    struct acpi_dmar_hardware_unit,
 711				    header);
 712
 713		if (dmaru->include_all &&
 714		    drhd->segment == pci_domain_nr(dev->bus))
 715			goto out;
 716
 717		if (dmar_pci_device_match(dmaru->devices,
 718					  dmaru->devices_cnt, dev))
 719			goto out;
 720	}
 721	dmaru = NULL;
 722out:
 723	rcu_read_unlock();
 724
 725	return dmaru;
 726}
 727
 728static void __init dmar_acpi_insert_dev_scope(u8 device_number,
 729					      struct acpi_device *adev)
 730{
 731	struct dmar_drhd_unit *dmaru;
 732	struct acpi_dmar_hardware_unit *drhd;
 733	struct acpi_dmar_device_scope *scope;
 734	struct device *tmp;
 735	int i;
 736	struct acpi_dmar_pci_path *path;
 737
 738	for_each_drhd_unit(dmaru) {
 739		drhd = container_of(dmaru->hdr,
 740				    struct acpi_dmar_hardware_unit,
 741				    header);
 742
 743		for (scope = (void *)(drhd + 1);
 744		     (unsigned long)scope < ((unsigned long)drhd) + drhd->header.length;
 745		     scope = ((void *)scope) + scope->length) {
 746			if (scope->entry_type != ACPI_DMAR_SCOPE_TYPE_NAMESPACE)
 747				continue;
 748			if (scope->enumeration_id != device_number)
 749				continue;
 750
 751			path = (void *)(scope + 1);
 752			pr_info("ACPI device \"%s\" under DMAR at %llx as %02x:%02x.%d\n",
 753				dev_name(&adev->dev), dmaru->reg_base_addr,
 754				scope->bus, path->device, path->function);
 755			for_each_dev_scope(dmaru->devices, dmaru->devices_cnt, i, tmp)
 756				if (tmp == NULL) {
 757					dmaru->devices[i].bus = scope->bus;
 758					dmaru->devices[i].devfn = PCI_DEVFN(path->device,
 759									    path->function);
 760					rcu_assign_pointer(dmaru->devices[i].dev,
 761							   get_device(&adev->dev));
 762					return;
 763				}
 764			BUG_ON(i >= dmaru->devices_cnt);
 765		}
 766	}
 767	pr_warn("No IOMMU scope found for ANDD enumeration ID %d (%s)\n",
 768		device_number, dev_name(&adev->dev));
 769}
 770
 771static int __init dmar_acpi_dev_scope_init(void)
 772{
 773	struct acpi_dmar_andd *andd;
 774
 775	if (dmar_tbl == NULL)
 776		return -ENODEV;
 777
 778	for (andd = (void *)dmar_tbl + sizeof(struct acpi_table_dmar);
 779	     ((unsigned long)andd) < ((unsigned long)dmar_tbl) + dmar_tbl->length;
 780	     andd = ((void *)andd) + andd->header.length) {
 781		if (andd->header.type == ACPI_DMAR_TYPE_NAMESPACE) {
 782			acpi_handle h;
 783			struct acpi_device *adev;
 784
 785			if (!ACPI_SUCCESS(acpi_get_handle(ACPI_ROOT_OBJECT,
 786							  andd->device_name,
 787							  &h))) {
 788				pr_err("Failed to find handle for ACPI object %s\n",
 789				       andd->device_name);
 790				continue;
 791			}
 792			adev = acpi_fetch_acpi_dev(h);
 793			if (!adev) {
 794				pr_err("Failed to get device for ACPI object %s\n",
 795				       andd->device_name);
 796				continue;
 797			}
 798			dmar_acpi_insert_dev_scope(andd->device_number, adev);
 799		}
 800	}
 801	return 0;
 802}
 803
 804int __init dmar_dev_scope_init(void)
 805{
 806	struct pci_dev *dev = NULL;
 807	struct dmar_pci_notify_info *info;
 808
 809	if (dmar_dev_scope_status != 1)
 810		return dmar_dev_scope_status;
 811
 812	if (list_empty(&dmar_drhd_units)) {
 813		dmar_dev_scope_status = -ENODEV;
 814	} else {
 815		dmar_dev_scope_status = 0;
 816
 817		dmar_acpi_dev_scope_init();
 818
 819		for_each_pci_dev(dev) {
 820			if (dev->is_virtfn)
 821				continue;
 822
 823			info = dmar_alloc_pci_notify_info(dev,
 824					BUS_NOTIFY_ADD_DEVICE);
 825			if (!info) {
 826				pci_dev_put(dev);
 827				return dmar_dev_scope_status;
 828			} else {
 829				dmar_pci_bus_add_dev(info);
 830				dmar_free_pci_notify_info(info);
 831			}
 832		}
 833	}
 834
 835	return dmar_dev_scope_status;
 836}
 837
 838void __init dmar_register_bus_notifier(void)
 839{
 840	bus_register_notifier(&pci_bus_type, &dmar_pci_bus_nb);
 841}
 842
 843
 844int __init dmar_table_init(void)
 845{
 846	static int dmar_table_initialized;
 847	int ret;
 848
 849	if (dmar_table_initialized == 0) {
 850		ret = parse_dmar_table();
 851		if (ret < 0) {
 852			if (ret != -ENODEV)
 853				pr_info("Parse DMAR table failure.\n");
 854		} else  if (list_empty(&dmar_drhd_units)) {
 855			pr_info("No DMAR devices found\n");
 856			ret = -ENODEV;
 857		}
 858
 859		if (ret < 0)
 860			dmar_table_initialized = ret;
 861		else
 862			dmar_table_initialized = 1;
 863	}
 864
 865	return dmar_table_initialized < 0 ? dmar_table_initialized : 0;
 866}
 867
 868static void warn_invalid_dmar(u64 addr, const char *message)
 869{
 870	pr_warn_once(FW_BUG
 871		"Your BIOS is broken; DMAR reported at address %llx%s!\n"
 872		"BIOS vendor: %s; Ver: %s; Product Version: %s\n",
 873		addr, message,
 874		dmi_get_system_info(DMI_BIOS_VENDOR),
 875		dmi_get_system_info(DMI_BIOS_VERSION),
 876		dmi_get_system_info(DMI_PRODUCT_VERSION));
 877	add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
 878}
 879
 880static int __ref
 881dmar_validate_one_drhd(struct acpi_dmar_header *entry, void *arg)
 882{
 883	struct acpi_dmar_hardware_unit *drhd;
 884	void __iomem *addr;
 885	u64 cap, ecap;
 886
 887	drhd = (void *)entry;
 888	if (!drhd->address) {
 889		warn_invalid_dmar(0, "");
 890		return -EINVAL;
 891	}
 892
 893	if (arg)
 894		addr = ioremap(drhd->address, VTD_PAGE_SIZE);
 895	else
 896		addr = early_ioremap(drhd->address, VTD_PAGE_SIZE);
 897	if (!addr) {
 898		pr_warn("Can't validate DRHD address: %llx\n", drhd->address);
 899		return -EINVAL;
 900	}
 901
 902	cap = dmar_readq(addr + DMAR_CAP_REG);
 903	ecap = dmar_readq(addr + DMAR_ECAP_REG);
 904
 905	if (arg)
 906		iounmap(addr);
 907	else
 908		early_iounmap(addr, VTD_PAGE_SIZE);
 909
 910	if (cap == (uint64_t)-1 && ecap == (uint64_t)-1) {
 911		warn_invalid_dmar(drhd->address, " returns all ones");
 912		return -EINVAL;
 913	}
 914
 915	return 0;
 916}
 917
 918void __init detect_intel_iommu(void)
 919{
 920	int ret;
 921	struct dmar_res_callback validate_drhd_cb = {
 922		.cb[ACPI_DMAR_TYPE_HARDWARE_UNIT] = &dmar_validate_one_drhd,
 923		.ignore_unhandled = true,
 924	};
 925
 926	down_write(&dmar_global_lock);
 927	ret = dmar_table_detect();
 928	if (!ret)
 929		ret = dmar_walk_dmar_table((struct acpi_table_dmar *)dmar_tbl,
 930					   &validate_drhd_cb);
 931	if (!ret && !no_iommu && !iommu_detected &&
 932	    (!dmar_disabled || dmar_platform_optin())) {
 933		iommu_detected = 1;
 934		/* Make sure ACS will be enabled */
 935		pci_request_acs();
 936	}
 937
 938#ifdef CONFIG_X86
 939	if (!ret) {
 940		x86_init.iommu.iommu_init = intel_iommu_init;
 941		x86_platform.iommu_shutdown = intel_iommu_shutdown;
 942	}
 943
 944#endif
 945
 946	if (dmar_tbl) {
 947		acpi_put_table(dmar_tbl);
 948		dmar_tbl = NULL;
 949	}
 950	up_write(&dmar_global_lock);
 
 
 951}
 952
 953static void unmap_iommu(struct intel_iommu *iommu)
 954{
 955	iounmap(iommu->reg);
 956	release_mem_region(iommu->reg_phys, iommu->reg_size);
 957}
 958
 959/**
 960 * map_iommu: map the iommu's registers
 961 * @iommu: the iommu to map
 962 * @drhd: DMA remapping hardware definition structure
 963 *
 964 * Memory map the iommu's registers.  Start w/ a single page, and
 965 * possibly expand if that turns out to be insufficent.
 966 */
 967static int map_iommu(struct intel_iommu *iommu, struct dmar_drhd_unit *drhd)
 968{
 969	u64 phys_addr = drhd->reg_base_addr;
 970	int map_size, err=0;
 971
 972	iommu->reg_phys = phys_addr;
 973	iommu->reg_size = drhd->reg_size;
 974
 975	if (!request_mem_region(iommu->reg_phys, iommu->reg_size, iommu->name)) {
 976		pr_err("Can't reserve memory\n");
 977		err = -EBUSY;
 978		goto out;
 979	}
 980
 981	iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
 982	if (!iommu->reg) {
 983		pr_err("Can't map the region\n");
 984		err = -ENOMEM;
 985		goto release;
 986	}
 987
 988	iommu->cap = dmar_readq(iommu->reg + DMAR_CAP_REG);
 989	iommu->ecap = dmar_readq(iommu->reg + DMAR_ECAP_REG);
 990
 991	if (iommu->cap == (uint64_t)-1 && iommu->ecap == (uint64_t)-1) {
 992		err = -EINVAL;
 993		warn_invalid_dmar(phys_addr, " returns all ones");
 994		goto unmap;
 995	}
 
 996
 997	/* the registers might be more than one page */
 998	map_size = max_t(int, ecap_max_iotlb_offset(iommu->ecap),
 999			 cap_max_fault_reg_offset(iommu->cap));
1000	map_size = VTD_PAGE_ALIGN(map_size);
1001	if (map_size > iommu->reg_size) {
1002		iounmap(iommu->reg);
1003		release_mem_region(iommu->reg_phys, iommu->reg_size);
1004		iommu->reg_size = map_size;
1005		if (!request_mem_region(iommu->reg_phys, iommu->reg_size,
1006					iommu->name)) {
1007			pr_err("Can't reserve memory\n");
1008			err = -EBUSY;
1009			goto out;
1010		}
1011		iommu->reg = ioremap(iommu->reg_phys, iommu->reg_size);
1012		if (!iommu->reg) {
1013			pr_err("Can't map the region\n");
1014			err = -ENOMEM;
1015			goto release;
1016		}
1017	}
1018
1019	if (cap_ecmds(iommu->cap)) {
1020		int i;
1021
1022		for (i = 0; i < DMA_MAX_NUM_ECMDCAP; i++) {
1023			iommu->ecmdcap[i] = dmar_readq(iommu->reg + DMAR_ECCAP_REG +
1024						       i * DMA_ECMD_REG_STEP);
1025		}
1026	}
1027
1028	err = 0;
1029	goto out;
1030
1031unmap:
1032	iounmap(iommu->reg);
1033release:
1034	release_mem_region(iommu->reg_phys, iommu->reg_size);
1035out:
1036	return err;
1037}
1038
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1039static int alloc_iommu(struct dmar_drhd_unit *drhd)
1040{
1041	struct intel_iommu *iommu;
1042	u32 ver, sts;
1043	int agaw = -1;
1044	int msagaw = -1;
1045	int err;
1046
1047	if (!drhd->reg_base_addr) {
1048		warn_invalid_dmar(0, "");
1049		return -EINVAL;
1050	}
1051
1052	iommu = kzalloc(sizeof(*iommu), GFP_KERNEL);
1053	if (!iommu)
1054		return -ENOMEM;
1055
1056	iommu->seq_id = ida_alloc_range(&dmar_seq_ids, 0,
1057					DMAR_UNITS_SUPPORTED - 1, GFP_KERNEL);
1058	if (iommu->seq_id < 0) {
1059		pr_err("Failed to allocate seq_id\n");
1060		err = iommu->seq_id;
1061		goto error;
1062	}
1063	snprintf(iommu->name, sizeof(iommu->name), "dmar%d", iommu->seq_id);
1064
1065	err = map_iommu(iommu, drhd);
1066	if (err) {
1067		pr_err("Failed to map %s\n", iommu->name);
1068		goto error_free_seq_id;
1069	}
1070
1071	if (!cap_sagaw(iommu->cap) &&
1072	    (!ecap_smts(iommu->ecap) || ecap_slts(iommu->ecap))) {
1073		pr_info("%s: No supported address widths. Not attempting DMA translation.\n",
1074			iommu->name);
1075		drhd->ignored = 1;
1076	}
1077
1078	if (!drhd->ignored) {
1079		agaw = iommu_calculate_agaw(iommu);
1080		if (agaw < 0) {
1081			pr_err("Cannot get a valid agaw for iommu (seq_id = %d)\n",
1082			       iommu->seq_id);
1083			drhd->ignored = 1;
1084		}
1085	}
1086	if (!drhd->ignored) {
1087		msagaw = iommu_calculate_max_sagaw(iommu);
1088		if (msagaw < 0) {
1089			pr_err("Cannot get a valid max agaw for iommu (seq_id = %d)\n",
1090			       iommu->seq_id);
1091			drhd->ignored = 1;
1092			agaw = -1;
1093		}
1094	}
1095	iommu->agaw = agaw;
1096	iommu->msagaw = msagaw;
1097	iommu->segment = drhd->segment;
1098	iommu->device_rbtree = RB_ROOT;
1099	spin_lock_init(&iommu->device_rbtree_lock);
1100	mutex_init(&iommu->iopf_lock);
1101	iommu->node = NUMA_NO_NODE;
1102
1103	ver = readl(iommu->reg + DMAR_VER_REG);
1104	pr_info("%s: reg_base_addr %llx ver %d:%d cap %llx ecap %llx\n",
1105		iommu->name,
1106		(unsigned long long)drhd->reg_base_addr,
1107		DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver),
1108		(unsigned long long)iommu->cap,
1109		(unsigned long long)iommu->ecap);
1110
1111	/* Reflect status in gcmd */
1112	sts = readl(iommu->reg + DMAR_GSTS_REG);
1113	if (sts & DMA_GSTS_IRES)
1114		iommu->gcmd |= DMA_GCMD_IRE;
1115	if (sts & DMA_GSTS_TES)
1116		iommu->gcmd |= DMA_GCMD_TE;
1117	if (sts & DMA_GSTS_QIES)
1118		iommu->gcmd |= DMA_GCMD_QIE;
1119
1120	if (alloc_iommu_pmu(iommu))
1121		pr_debug("Cannot alloc PMU for iommu (seq_id = %d)\n", iommu->seq_id);
1122
1123	raw_spin_lock_init(&iommu->register_lock);
1124
1125	/*
1126	 * A value of N in PSS field of eCap register indicates hardware
1127	 * supports PASID field of N+1 bits.
1128	 */
1129	if (pasid_supported(iommu))
1130		iommu->iommu.max_pasids = 2UL << ecap_pss(iommu->ecap);
1131
1132	/*
1133	 * This is only for hotplug; at boot time intel_iommu_enabled won't
1134	 * be set yet. When intel_iommu_init() runs, it registers the units
1135	 * present at boot time, then sets intel_iommu_enabled.
1136	 */
1137	if (intel_iommu_enabled && !drhd->ignored) {
1138		err = iommu_device_sysfs_add(&iommu->iommu, NULL,
1139					     intel_iommu_groups,
1140					     "%s", iommu->name);
1141		if (err)
1142			goto err_unmap;
1143
1144		err = iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);
1145		if (err)
1146			goto err_sysfs;
1147
1148		iommu_pmu_register(iommu);
 
 
1149	}
1150
1151	drhd->iommu = iommu;
1152	iommu->drhd = drhd;
1153
1154	return 0;
1155
1156err_sysfs:
1157	iommu_device_sysfs_remove(&iommu->iommu);
1158err_unmap:
1159	free_iommu_pmu(iommu);
1160	unmap_iommu(iommu);
1161error_free_seq_id:
1162	ida_free(&dmar_seq_ids, iommu->seq_id);
1163error:
1164	kfree(iommu);
1165	return err;
1166}
1167
1168static void free_iommu(struct intel_iommu *iommu)
1169{
1170	if (intel_iommu_enabled && !iommu->drhd->ignored) {
1171		iommu_pmu_unregister(iommu);
1172		iommu_device_unregister(&iommu->iommu);
1173		iommu_device_sysfs_remove(&iommu->iommu);
1174	}
1175
1176	free_iommu_pmu(iommu);
1177
1178	if (iommu->irq) {
1179		if (iommu->pr_irq) {
1180			free_irq(iommu->pr_irq, iommu);
1181			dmar_free_hwirq(iommu->pr_irq);
1182			iommu->pr_irq = 0;
1183		}
1184		free_irq(iommu->irq, iommu);
1185		dmar_free_hwirq(iommu->irq);
1186		iommu->irq = 0;
1187	}
1188
1189	if (iommu->qi) {
1190		iommu_free_page(iommu->qi->desc);
1191		kfree(iommu->qi->desc_status);
1192		kfree(iommu->qi);
1193	}
1194
1195	if (iommu->reg)
1196		unmap_iommu(iommu);
1197
1198	ida_free(&dmar_seq_ids, iommu->seq_id);
1199	kfree(iommu);
1200}
1201
1202/*
1203 * Reclaim all the submitted descriptors which have completed its work.
1204 */
1205static inline void reclaim_free_desc(struct q_inval *qi)
1206{
1207	while (qi->desc_status[qi->free_tail] == QI_FREE && qi->free_tail != qi->free_head) {
 
 
1208		qi->free_tail = (qi->free_tail + 1) % QI_LENGTH;
1209		qi->free_cnt++;
1210	}
1211}
1212
1213static const char *qi_type_string(u8 type)
1214{
1215	switch (type) {
1216	case QI_CC_TYPE:
1217		return "Context-cache Invalidation";
1218	case QI_IOTLB_TYPE:
1219		return "IOTLB Invalidation";
1220	case QI_DIOTLB_TYPE:
1221		return "Device-TLB Invalidation";
1222	case QI_IEC_TYPE:
1223		return "Interrupt Entry Cache Invalidation";
1224	case QI_IWD_TYPE:
1225		return "Invalidation Wait";
1226	case QI_EIOTLB_TYPE:
1227		return "PASID-based IOTLB Invalidation";
1228	case QI_PC_TYPE:
1229		return "PASID-cache Invalidation";
1230	case QI_DEIOTLB_TYPE:
1231		return "PASID-based Device-TLB Invalidation";
1232	case QI_PGRP_RESP_TYPE:
1233		return "Page Group Response";
1234	default:
1235		return "UNKNOWN";
1236	}
1237}
1238
1239static void qi_dump_fault(struct intel_iommu *iommu, u32 fault)
1240{
1241	unsigned int head = dmar_readl(iommu->reg + DMAR_IQH_REG);
1242	u64 iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG);
1243	struct qi_desc *desc = iommu->qi->desc + head;
1244
1245	if (fault & DMA_FSTS_IQE)
1246		pr_err("VT-d detected Invalidation Queue Error: Reason %llx",
1247		       DMAR_IQER_REG_IQEI(iqe_err));
1248	if (fault & DMA_FSTS_ITE)
1249		pr_err("VT-d detected Invalidation Time-out Error: SID %llx",
1250		       DMAR_IQER_REG_ITESID(iqe_err));
1251	if (fault & DMA_FSTS_ICE)
1252		pr_err("VT-d detected Invalidation Completion Error: SID %llx",
1253		       DMAR_IQER_REG_ICESID(iqe_err));
1254
1255	pr_err("QI HEAD: %s qw0 = 0x%llx, qw1 = 0x%llx\n",
1256	       qi_type_string(desc->qw0 & 0xf),
1257	       (unsigned long long)desc->qw0,
1258	       (unsigned long long)desc->qw1);
1259
1260	head = ((head >> qi_shift(iommu)) + QI_LENGTH - 1) % QI_LENGTH;
1261	head <<= qi_shift(iommu);
1262	desc = iommu->qi->desc + head;
1263
1264	pr_err("QI PRIOR: %s qw0 = 0x%llx, qw1 = 0x%llx\n",
1265	       qi_type_string(desc->qw0 & 0xf),
1266	       (unsigned long long)desc->qw0,
1267	       (unsigned long long)desc->qw1);
1268}
1269
1270static int qi_check_fault(struct intel_iommu *iommu, int index, int wait_index)
1271{
1272	u32 fault;
1273	int head, tail;
1274	struct device *dev;
1275	u64 iqe_err, ite_sid;
1276	struct q_inval *qi = iommu->qi;
1277	int shift = qi_shift(iommu);
1278
1279	if (qi->desc_status[wait_index] == QI_ABORT)
1280		return -EAGAIN;
1281
1282	fault = readl(iommu->reg + DMAR_FSTS_REG);
1283	if (fault & (DMA_FSTS_IQE | DMA_FSTS_ITE | DMA_FSTS_ICE))
1284		qi_dump_fault(iommu, fault);
1285
1286	/*
1287	 * If IQE happens, the head points to the descriptor associated
1288	 * with the error. No new descriptors are fetched until the IQE
1289	 * is cleared.
1290	 */
1291	if (fault & DMA_FSTS_IQE) {
1292		head = readl(iommu->reg + DMAR_IQH_REG);
1293		if ((head >> shift) == index) {
1294			struct qi_desc *desc = qi->desc + head;
1295
1296			/*
1297			 * desc->qw2 and desc->qw3 are either reserved or
1298			 * used by software as private data. We won't print
1299			 * out these two qw's for security consideration.
1300			 */
 
 
 
1301			memcpy(desc, qi->desc + (wait_index << shift),
1302			       1 << shift);
1303			writel(DMA_FSTS_IQE, iommu->reg + DMAR_FSTS_REG);
1304			pr_info("Invalidation Queue Error (IQE) cleared\n");
1305			return -EINVAL;
1306		}
1307	}
1308
1309	/*
1310	 * If ITE happens, all pending wait_desc commands are aborted.
1311	 * No new descriptors are fetched until the ITE is cleared.
1312	 */
1313	if (fault & DMA_FSTS_ITE) {
1314		head = readl(iommu->reg + DMAR_IQH_REG);
1315		head = ((head >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1316		head |= 1;
1317		tail = readl(iommu->reg + DMAR_IQT_REG);
1318		tail = ((tail >> shift) - 1 + QI_LENGTH) % QI_LENGTH;
1319
1320		/*
1321		 * SID field is valid only when the ITE field is Set in FSTS_REG
1322		 * see Intel VT-d spec r4.1, section 11.4.9.9
1323		 */
1324		iqe_err = dmar_readq(iommu->reg + DMAR_IQER_REG);
1325		ite_sid = DMAR_IQER_REG_ITESID(iqe_err);
1326
1327		writel(DMA_FSTS_ITE, iommu->reg + DMAR_FSTS_REG);
1328		pr_info("Invalidation Time-out Error (ITE) cleared\n");
1329
1330		do {
1331			if (qi->desc_status[head] == QI_IN_USE)
1332				qi->desc_status[head] = QI_ABORT;
1333			head = (head - 2 + QI_LENGTH) % QI_LENGTH;
1334		} while (head != tail);
1335
1336		/*
1337		 * If device was released or isn't present, no need to retry
1338		 * the ATS invalidate request anymore.
1339		 *
1340		 * 0 value of ite_sid means old VT-d device, no ite_sid value.
1341		 * see Intel VT-d spec r4.1, section 11.4.9.9
1342		 */
1343		if (ite_sid) {
1344			dev = device_rbtree_find(iommu, ite_sid);
1345			if (!dev || !dev_is_pci(dev) ||
1346			    !pci_device_is_present(to_pci_dev(dev)))
1347				return -ETIMEDOUT;
1348		}
1349		if (qi->desc_status[wait_index] == QI_ABORT)
1350			return -EAGAIN;
1351	}
1352
1353	if (fault & DMA_FSTS_ICE) {
1354		writel(DMA_FSTS_ICE, iommu->reg + DMAR_FSTS_REG);
1355		pr_info("Invalidation Completion Error (ICE) cleared\n");
1356	}
1357
1358	return 0;
1359}
1360
1361/*
1362 * Function to submit invalidation descriptors of all types to the queued
1363 * invalidation interface(QI). Multiple descriptors can be submitted at a
1364 * time, a wait descriptor will be appended to each submission to ensure
1365 * hardware has completed the invalidation before return. Wait descriptors
1366 * can be part of the submission but it will not be polled for completion.
1367 */
1368int qi_submit_sync(struct intel_iommu *iommu, struct qi_desc *desc,
1369		   unsigned int count, unsigned long options)
1370{
1371	struct q_inval *qi = iommu->qi;
1372	s64 devtlb_start_ktime = 0;
1373	s64 iotlb_start_ktime = 0;
1374	s64 iec_start_ktime = 0;
1375	struct qi_desc wait_desc;
1376	int wait_index, index;
1377	unsigned long flags;
1378	int offset, shift;
1379	int rc, i;
1380	u64 type;
1381
1382	if (!qi)
1383		return 0;
1384
1385	type = desc->qw0 & GENMASK_ULL(3, 0);
1386
1387	if ((type == QI_IOTLB_TYPE || type == QI_EIOTLB_TYPE) &&
1388	    dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IOTLB))
1389		iotlb_start_ktime = ktime_to_ns(ktime_get());
1390
1391	if ((type == QI_DIOTLB_TYPE || type == QI_DEIOTLB_TYPE) &&
1392	    dmar_latency_enabled(iommu, DMAR_LATENCY_INV_DEVTLB))
1393		devtlb_start_ktime = ktime_to_ns(ktime_get());
1394
1395	if (type == QI_IEC_TYPE &&
1396	    dmar_latency_enabled(iommu, DMAR_LATENCY_INV_IEC))
1397		iec_start_ktime = ktime_to_ns(ktime_get());
1398
1399restart:
1400	rc = 0;
1401
1402	raw_spin_lock_irqsave(&qi->q_lock, flags);
1403	/*
1404	 * Check if we have enough empty slots in the queue to submit,
1405	 * the calculation is based on:
1406	 * # of desc + 1 wait desc + 1 space between head and tail
1407	 */
1408	while (qi->free_cnt < count + 2) {
1409		raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1410		cpu_relax();
1411		raw_spin_lock_irqsave(&qi->q_lock, flags);
1412	}
1413
1414	index = qi->free_head;
1415	wait_index = (index + count) % QI_LENGTH;
1416	shift = qi_shift(iommu);
1417
1418	for (i = 0; i < count; i++) {
1419		offset = ((index + i) % QI_LENGTH) << shift;
1420		memcpy(qi->desc + offset, &desc[i], 1 << shift);
1421		qi->desc_status[(index + i) % QI_LENGTH] = QI_IN_USE;
1422		trace_qi_submit(iommu, desc[i].qw0, desc[i].qw1,
1423				desc[i].qw2, desc[i].qw3);
1424	}
1425	qi->desc_status[wait_index] = QI_IN_USE;
1426
1427	wait_desc.qw0 = QI_IWD_STATUS_DATA(QI_DONE) |
1428			QI_IWD_STATUS_WRITE | QI_IWD_TYPE;
1429	if (options & QI_OPT_WAIT_DRAIN)
1430		wait_desc.qw0 |= QI_IWD_PRQ_DRAIN;
1431	wait_desc.qw1 = virt_to_phys(&qi->desc_status[wait_index]);
1432	wait_desc.qw2 = 0;
1433	wait_desc.qw3 = 0;
1434
1435	offset = wait_index << shift;
1436	memcpy(qi->desc + offset, &wait_desc, 1 << shift);
1437
1438	qi->free_head = (qi->free_head + count + 1) % QI_LENGTH;
1439	qi->free_cnt -= count + 1;
1440
1441	/*
1442	 * update the HW tail register indicating the presence of
1443	 * new descriptors.
1444	 */
1445	writel(qi->free_head << shift, iommu->reg + DMAR_IQT_REG);
1446
1447	while (READ_ONCE(qi->desc_status[wait_index]) != QI_DONE) {
1448		/*
1449		 * We will leave the interrupts disabled, to prevent interrupt
1450		 * context to queue another cmd while a cmd is already submitted
1451		 * and waiting for completion on this cpu. This is to avoid
1452		 * a deadlock where the interrupt context can wait indefinitely
1453		 * for free slots in the queue.
1454		 */
1455		rc = qi_check_fault(iommu, index, wait_index);
1456		if (rc)
1457			break;
1458
1459		raw_spin_unlock(&qi->q_lock);
1460		cpu_relax();
1461		raw_spin_lock(&qi->q_lock);
1462	}
1463
1464	/*
1465	 * The reclaim code can free descriptors from multiple submissions
1466	 * starting from the tail of the queue. When count == 0, the
1467	 * status of the standalone wait descriptor at the tail of the queue
1468	 * must be set to QI_FREE to allow the reclaim code to proceed.
1469	 * It is also possible that descriptors from one of the previous
1470	 * submissions has to be reclaimed by a subsequent submission.
1471	 */
1472	for (i = 0; i <= count; i++)
1473		qi->desc_status[(index + i) % QI_LENGTH] = QI_FREE;
1474
1475	reclaim_free_desc(qi);
1476	raw_spin_unlock_irqrestore(&qi->q_lock, flags);
1477
1478	if (rc == -EAGAIN)
1479		goto restart;
1480
1481	if (iotlb_start_ktime)
1482		dmar_latency_update(iommu, DMAR_LATENCY_INV_IOTLB,
1483				ktime_to_ns(ktime_get()) - iotlb_start_ktime);
1484
1485	if (devtlb_start_ktime)
1486		dmar_latency_update(iommu, DMAR_LATENCY_INV_DEVTLB,
1487				ktime_to_ns(ktime_get()) - devtlb_start_ktime);
1488
1489	if (iec_start_ktime)
1490		dmar_latency_update(iommu, DMAR_LATENCY_INV_IEC,
1491				ktime_to_ns(ktime_get()) - iec_start_ktime);
1492
1493	return rc;
1494}
1495
1496/*
1497 * Flush the global interrupt entry cache.
1498 */
1499void qi_global_iec(struct intel_iommu *iommu)
1500{
1501	struct qi_desc desc;
1502
1503	desc.qw0 = QI_IEC_TYPE;
1504	desc.qw1 = 0;
1505	desc.qw2 = 0;
1506	desc.qw3 = 0;
1507
1508	/* should never fail */
1509	qi_submit_sync(iommu, &desc, 1, 0);
1510}
1511
1512void qi_flush_context(struct intel_iommu *iommu, u16 did, u16 sid, u8 fm,
1513		      u64 type)
1514{
1515	struct qi_desc desc;
1516
1517	desc.qw0 = QI_CC_FM(fm) | QI_CC_SID(sid) | QI_CC_DID(did)
1518			| QI_CC_GRAN(type) | QI_CC_TYPE;
1519	desc.qw1 = 0;
1520	desc.qw2 = 0;
1521	desc.qw3 = 0;
1522
1523	qi_submit_sync(iommu, &desc, 1, 0);
1524}
1525
1526void qi_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1527		    unsigned int size_order, u64 type)
1528{
 
 
1529	struct qi_desc desc;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1530
1531	qi_desc_iotlb(iommu, did, addr, size_order, type, &desc);
1532	qi_submit_sync(iommu, &desc, 1, 0);
1533}
1534
1535void qi_flush_dev_iotlb(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1536			u16 qdep, u64 addr, unsigned mask)
1537{
1538	struct qi_desc desc;
1539
1540	/*
1541	 * VT-d spec, section 4.3:
1542	 *
1543	 * Software is recommended to not submit any Device-TLB invalidation
1544	 * requests while address remapping hardware is disabled.
1545	 */
1546	if (!(iommu->gcmd & DMA_GCMD_TE))
1547		return;
 
 
 
 
 
1548
1549	qi_desc_dev_iotlb(sid, pfsid, qdep, addr, mask, &desc);
1550	qi_submit_sync(iommu, &desc, 1, 0);
1551}
1552
1553/* PASID-based IOTLB invalidation */
1554void qi_flush_piotlb(struct intel_iommu *iommu, u16 did, u32 pasid, u64 addr,
1555		     unsigned long npages, bool ih)
1556{
1557	struct qi_desc desc = {.qw2 = 0, .qw3 = 0};
1558
1559	/*
1560	 * npages == -1 means a PASID-selective invalidation, otherwise,
1561	 * a positive value for Page-selective-within-PASID invalidation.
1562	 * 0 is not a valid input.
1563	 */
1564	if (WARN_ON(!npages)) {
1565		pr_err("Invalid input npages = %ld\n", npages);
1566		return;
1567	}
1568
1569	qi_desc_piotlb(did, pasid, addr, npages, ih, &desc);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1570	qi_submit_sync(iommu, &desc, 1, 0);
1571}
1572
1573/* PASID-based device IOTLB Invalidate */
1574void qi_flush_dev_iotlb_pasid(struct intel_iommu *iommu, u16 sid, u16 pfsid,
1575			      u32 pasid,  u16 qdep, u64 addr, unsigned int size_order)
1576{
 
1577	struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1578
1579	/*
1580	 * VT-d spec, section 4.3:
1581	 *
1582	 * Software is recommended to not submit any Device-TLB invalidation
1583	 * requests while address remapping hardware is disabled.
1584	 */
1585	if (!(iommu->gcmd & DMA_GCMD_TE))
1586		return;
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1587
1588	qi_desc_dev_iotlb_pasid(sid, pfsid, pasid,
1589				qdep, addr, size_order,
1590				&desc);
1591	qi_submit_sync(iommu, &desc, 1, 0);
1592}
1593
1594void qi_flush_pasid_cache(struct intel_iommu *iommu, u16 did,
1595			  u64 granu, u32 pasid)
1596{
1597	struct qi_desc desc = {.qw1 = 0, .qw2 = 0, .qw3 = 0};
1598
1599	desc.qw0 = QI_PC_PASID(pasid) | QI_PC_DID(did) |
1600			QI_PC_GRAN(granu) | QI_PC_TYPE;
1601	qi_submit_sync(iommu, &desc, 1, 0);
1602}
1603
1604/*
1605 * Disable Queued Invalidation interface.
1606 */
1607void dmar_disable_qi(struct intel_iommu *iommu)
1608{
1609	unsigned long flags;
1610	u32 sts;
1611	cycles_t start_time = get_cycles();
1612
1613	if (!ecap_qis(iommu->ecap))
1614		return;
1615
1616	raw_spin_lock_irqsave(&iommu->register_lock, flags);
1617
1618	sts =  readl(iommu->reg + DMAR_GSTS_REG);
1619	if (!(sts & DMA_GSTS_QIES))
1620		goto end;
1621
1622	/*
1623	 * Give a chance to HW to complete the pending invalidation requests.
1624	 */
1625	while ((readl(iommu->reg + DMAR_IQT_REG) !=
1626		readl(iommu->reg + DMAR_IQH_REG)) &&
1627		(DMAR_OPERATION_TIMEOUT > (get_cycles() - start_time)))
1628		cpu_relax();
1629
1630	iommu->gcmd &= ~DMA_GCMD_QIE;
1631	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1632
1633	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl,
1634		      !(sts & DMA_GSTS_QIES), sts);
1635end:
1636	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1637}
1638
1639/*
1640 * Enable queued invalidation.
1641 */
1642static void __dmar_enable_qi(struct intel_iommu *iommu)
1643{
1644	u32 sts;
1645	unsigned long flags;
1646	struct q_inval *qi = iommu->qi;
1647	u64 val = virt_to_phys(qi->desc);
1648
1649	qi->free_head = qi->free_tail = 0;
1650	qi->free_cnt = QI_LENGTH;
1651
1652	/*
1653	 * Set DW=1 and QS=1 in IQA_REG when Scalable Mode capability
1654	 * is present.
1655	 */
1656	if (ecap_smts(iommu->ecap))
1657		val |= BIT_ULL(11) | BIT_ULL(0);
1658
1659	raw_spin_lock_irqsave(&iommu->register_lock, flags);
1660
1661	/* write zero to the tail reg */
1662	writel(0, iommu->reg + DMAR_IQT_REG);
1663
1664	dmar_writeq(iommu->reg + DMAR_IQA_REG, val);
1665
1666	iommu->gcmd |= DMA_GCMD_QIE;
1667	writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1668
1669	/* Make sure hardware complete it */
1670	IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG, readl, (sts & DMA_GSTS_QIES), sts);
1671
1672	raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1673}
1674
1675/*
1676 * Enable Queued Invalidation interface. This is a must to support
1677 * interrupt-remapping. Also used by DMA-remapping, which replaces
1678 * register based IOTLB invalidation.
1679 */
1680int dmar_enable_qi(struct intel_iommu *iommu)
1681{
1682	struct q_inval *qi;
1683	void *desc;
1684	int order;
1685
1686	if (!ecap_qis(iommu->ecap))
1687		return -ENOENT;
1688
1689	/*
1690	 * queued invalidation is already setup and enabled.
1691	 */
1692	if (iommu->qi)
1693		return 0;
1694
1695	iommu->qi = kmalloc(sizeof(*qi), GFP_ATOMIC);
1696	if (!iommu->qi)
1697		return -ENOMEM;
1698
1699	qi = iommu->qi;
1700
1701	/*
1702	 * Need two pages to accommodate 256 descriptors of 256 bits each
1703	 * if the remapping hardware supports scalable mode translation.
1704	 */
1705	order = ecap_smts(iommu->ecap) ? 1 : 0;
1706	desc = iommu_alloc_pages_node(iommu->node, GFP_ATOMIC, order);
1707	if (!desc) {
1708		kfree(qi);
1709		iommu->qi = NULL;
1710		return -ENOMEM;
1711	}
1712
1713	qi->desc = desc;
1714
1715	qi->desc_status = kcalloc(QI_LENGTH, sizeof(int), GFP_ATOMIC);
1716	if (!qi->desc_status) {
1717		iommu_free_page(qi->desc);
1718		kfree(qi);
1719		iommu->qi = NULL;
1720		return -ENOMEM;
1721	}
1722
1723	raw_spin_lock_init(&qi->q_lock);
1724
1725	__dmar_enable_qi(iommu);
1726
1727	return 0;
1728}
1729
1730/* iommu interrupt handling. Most stuff are MSI-like. */
1731
1732enum faulttype {
1733	DMA_REMAP,
1734	INTR_REMAP,
1735	UNKNOWN,
1736};
1737
1738static const char *dma_remap_fault_reasons[] =
1739{
1740	"Software",
1741	"Present bit in root entry is clear",
1742	"Present bit in context entry is clear",
1743	"Invalid context entry",
1744	"Access beyond MGAW",
1745	"PTE Write access is not set",
1746	"PTE Read access is not set",
1747	"Next page table ptr is invalid",
1748	"Root table address invalid",
1749	"Context table ptr is invalid",
1750	"non-zero reserved fields in RTP",
1751	"non-zero reserved fields in CTP",
1752	"non-zero reserved fields in PTE",
1753	"PCE for translation request specifies blocking",
1754};
1755
1756static const char * const dma_remap_sm_fault_reasons[] = {
1757	"SM: Invalid Root Table Address",
1758	"SM: TTM 0 for request with PASID",
1759	"SM: TTM 0 for page group request",
1760	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x33-0x37 */
1761	"SM: Error attempting to access Root Entry",
1762	"SM: Present bit in Root Entry is clear",
1763	"SM: Non-zero reserved field set in Root Entry",
1764	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x3B-0x3F */
1765	"SM: Error attempting to access Context Entry",
1766	"SM: Present bit in Context Entry is clear",
1767	"SM: Non-zero reserved field set in the Context Entry",
1768	"SM: Invalid Context Entry",
1769	"SM: DTE field in Context Entry is clear",
1770	"SM: PASID Enable field in Context Entry is clear",
1771	"SM: PASID is larger than the max in Context Entry",
1772	"SM: PRE field in Context-Entry is clear",
1773	"SM: RID_PASID field error in Context-Entry",
1774	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x49-0x4F */
1775	"SM: Error attempting to access the PASID Directory Entry",
1776	"SM: Present bit in Directory Entry is clear",
1777	"SM: Non-zero reserved field set in PASID Directory Entry",
1778	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x53-0x57 */
1779	"SM: Error attempting to access PASID Table Entry",
1780	"SM: Present bit in PASID Table Entry is clear",
1781	"SM: Non-zero reserved field set in PASID Table Entry",
1782	"SM: Invalid Scalable-Mode PASID Table Entry",
1783	"SM: ERE field is clear in PASID Table Entry",
1784	"SM: SRE field is clear in PASID Table Entry",
1785	"Unknown", "Unknown",/* 0x5E-0x5F */
1786	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x60-0x67 */
1787	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x68-0x6F */
1788	"SM: Error attempting to access first-level paging entry",
1789	"SM: Present bit in first-level paging entry is clear",
1790	"SM: Non-zero reserved field set in first-level paging entry",
1791	"SM: Error attempting to access FL-PML4 entry",
1792	"SM: First-level entry address beyond MGAW in Nested translation",
1793	"SM: Read permission error in FL-PML4 entry in Nested translation",
1794	"SM: Read permission error in first-level paging entry in Nested translation",
1795	"SM: Write permission error in first-level paging entry in Nested translation",
1796	"SM: Error attempting to access second-level paging entry",
1797	"SM: Read/Write permission error in second-level paging entry",
1798	"SM: Non-zero reserved field set in second-level paging entry",
1799	"SM: Invalid second-level page table pointer",
1800	"SM: A/D bit update needed in second-level entry when set up in no snoop",
1801	"Unknown", "Unknown", "Unknown", /* 0x7D-0x7F */
1802	"SM: Address in first-level translation is not canonical",
1803	"SM: U/S set 0 for first-level translation with user privilege",
1804	"SM: No execute permission for request with PASID and ER=1",
1805	"SM: Address beyond the DMA hardware max",
1806	"SM: Second-level entry address beyond the max",
1807	"SM: No write permission for Write/AtomicOp request",
1808	"SM: No read permission for Read/AtomicOp request",
1809	"SM: Invalid address-interrupt address",
1810	"Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", "Unknown", /* 0x88-0x8F */
1811	"SM: A/D bit update needed in first-level entry when set up in no snoop",
1812};
1813
1814static const char *irq_remap_fault_reasons[] =
1815{
1816	"Detected reserved fields in the decoded interrupt-remapped request",
1817	"Interrupt index exceeded the interrupt-remapping table size",
1818	"Present field in the IRTE entry is clear",
1819	"Error accessing interrupt-remapping table pointed by IRTA_REG",
1820	"Detected reserved fields in the IRTE entry",
1821	"Blocked a compatibility format interrupt request",
1822	"Blocked an interrupt request due to source-id verification failure",
1823};
1824
1825static const char *dmar_get_fault_reason(u8 fault_reason, int *fault_type)
1826{
1827	if (fault_reason >= 0x20 && (fault_reason - 0x20 <
1828					ARRAY_SIZE(irq_remap_fault_reasons))) {
1829		*fault_type = INTR_REMAP;
1830		return irq_remap_fault_reasons[fault_reason - 0x20];
1831	} else if (fault_reason >= 0x30 && (fault_reason - 0x30 <
1832			ARRAY_SIZE(dma_remap_sm_fault_reasons))) {
1833		*fault_type = DMA_REMAP;
1834		return dma_remap_sm_fault_reasons[fault_reason - 0x30];
1835	} else if (fault_reason < ARRAY_SIZE(dma_remap_fault_reasons)) {
1836		*fault_type = DMA_REMAP;
1837		return dma_remap_fault_reasons[fault_reason];
1838	} else {
1839		*fault_type = UNKNOWN;
1840		return "Unknown";
1841	}
1842}
1843
1844
1845static inline int dmar_msi_reg(struct intel_iommu *iommu, int irq)
1846{
1847	if (iommu->irq == irq)
1848		return DMAR_FECTL_REG;
1849	else if (iommu->pr_irq == irq)
1850		return DMAR_PECTL_REG;
1851	else if (iommu->perf_irq == irq)
1852		return DMAR_PERFINTRCTL_REG;
1853	else
1854		BUG();
1855}
1856
1857void dmar_msi_unmask(struct irq_data *data)
1858{
1859	struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1860	int reg = dmar_msi_reg(iommu, data->irq);
1861	unsigned long flag;
1862
1863	/* unmask it */
1864	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1865	writel(0, iommu->reg + reg);
1866	/* Read a reg to force flush the post write */
1867	readl(iommu->reg + reg);
1868	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1869}
1870
1871void dmar_msi_mask(struct irq_data *data)
1872{
1873	struct intel_iommu *iommu = irq_data_get_irq_handler_data(data);
1874	int reg = dmar_msi_reg(iommu, data->irq);
1875	unsigned long flag;
1876
1877	/* mask it */
1878	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1879	writel(DMA_FECTL_IM, iommu->reg + reg);
1880	/* Read a reg to force flush the post write */
1881	readl(iommu->reg + reg);
1882	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1883}
1884
1885void dmar_msi_write(int irq, struct msi_msg *msg)
1886{
1887	struct intel_iommu *iommu = irq_get_handler_data(irq);
1888	int reg = dmar_msi_reg(iommu, irq);
1889	unsigned long flag;
1890
1891	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1892	writel(msg->data, iommu->reg + reg + 4);
1893	writel(msg->address_lo, iommu->reg + reg + 8);
1894	writel(msg->address_hi, iommu->reg + reg + 12);
1895	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1896}
1897
 
 
 
 
 
 
 
 
 
 
 
 
 
1898static int dmar_fault_do_one(struct intel_iommu *iommu, int type,
1899		u8 fault_reason, u32 pasid, u16 source_id,
1900		unsigned long long addr)
1901{
1902	const char *reason;
1903	int fault_type;
1904
1905	reason = dmar_get_fault_reason(fault_reason, &fault_type);
1906
1907	if (fault_type == INTR_REMAP) {
1908		pr_err("[INTR-REMAP] Request device [%02x:%02x.%d] fault index 0x%llx [fault reason 0x%02x] %s\n",
1909		       source_id >> 8, PCI_SLOT(source_id & 0xFF),
1910		       PCI_FUNC(source_id & 0xFF), addr >> 48,
1911		       fault_reason, reason);
1912
1913		return 0;
1914	}
1915
1916	if (pasid == IOMMU_PASID_INVALID)
1917		pr_err("[%s NO_PASID] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
1918		       type ? "DMA Read" : "DMA Write",
1919		       source_id >> 8, PCI_SLOT(source_id & 0xFF),
1920		       PCI_FUNC(source_id & 0xFF), addr,
1921		       fault_reason, reason);
1922	else
1923		pr_err("[%s PASID 0x%x] Request device [%02x:%02x.%d] fault addr 0x%llx [fault reason 0x%02x] %s\n",
1924		       type ? "DMA Read" : "DMA Write", pasid,
1925		       source_id >> 8, PCI_SLOT(source_id & 0xFF),
1926		       PCI_FUNC(source_id & 0xFF), addr,
1927		       fault_reason, reason);
1928
1929	dmar_fault_dump_ptes(iommu, source_id, addr, pasid);
1930
1931	return 0;
1932}
1933
1934#define PRIMARY_FAULT_REG_LEN (16)
1935irqreturn_t dmar_fault(int irq, void *dev_id)
1936{
1937	struct intel_iommu *iommu = dev_id;
1938	int reg, fault_index;
1939	u32 fault_status;
1940	unsigned long flag;
1941	static DEFINE_RATELIMIT_STATE(rs,
1942				      DEFAULT_RATELIMIT_INTERVAL,
1943				      DEFAULT_RATELIMIT_BURST);
1944
1945	raw_spin_lock_irqsave(&iommu->register_lock, flag);
1946	fault_status = readl(iommu->reg + DMAR_FSTS_REG);
1947	if (fault_status && __ratelimit(&rs))
1948		pr_err("DRHD: handling fault status reg %x\n", fault_status);
1949
1950	/* TBD: ignore advanced fault log currently */
1951	if (!(fault_status & DMA_FSTS_PPF))
1952		goto unlock_exit;
1953
1954	fault_index = dma_fsts_fault_record_index(fault_status);
1955	reg = cap_fault_reg_offset(iommu->cap);
1956	while (1) {
1957		/* Disable printing, simply clear the fault when ratelimited */
1958		bool ratelimited = !__ratelimit(&rs);
1959		u8 fault_reason;
1960		u16 source_id;
1961		u64 guest_addr;
1962		u32 pasid;
1963		int type;
1964		u32 data;
1965		bool pasid_present;
1966
1967		/* highest 32 bits */
1968		data = readl(iommu->reg + reg +
1969				fault_index * PRIMARY_FAULT_REG_LEN + 12);
1970		if (!(data & DMA_FRCD_F))
1971			break;
1972
1973		if (!ratelimited) {
1974			fault_reason = dma_frcd_fault_reason(data);
1975			type = dma_frcd_type(data);
1976
1977			pasid = dma_frcd_pasid_value(data);
1978			data = readl(iommu->reg + reg +
1979				     fault_index * PRIMARY_FAULT_REG_LEN + 8);
1980			source_id = dma_frcd_source_id(data);
1981
1982			pasid_present = dma_frcd_pasid_present(data);
1983			guest_addr = dmar_readq(iommu->reg + reg +
1984					fault_index * PRIMARY_FAULT_REG_LEN);
1985			guest_addr = dma_frcd_page_addr(guest_addr);
1986		}
1987
1988		/* clear the fault */
1989		writel(DMA_FRCD_F, iommu->reg + reg +
1990			fault_index * PRIMARY_FAULT_REG_LEN + 12);
1991
1992		raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1993
1994		if (!ratelimited)
1995			/* Using pasid -1 if pasid is not present */
1996			dmar_fault_do_one(iommu, type, fault_reason,
1997					  pasid_present ? pasid : IOMMU_PASID_INVALID,
1998					  source_id, guest_addr);
1999
2000		fault_index++;
2001		if (fault_index >= cap_num_fault_regs(iommu->cap))
2002			fault_index = 0;
2003		raw_spin_lock_irqsave(&iommu->register_lock, flag);
2004	}
2005
2006	writel(DMA_FSTS_PFO | DMA_FSTS_PPF | DMA_FSTS_PRO,
2007	       iommu->reg + DMAR_FSTS_REG);
2008
2009unlock_exit:
2010	raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2011	return IRQ_HANDLED;
2012}
2013
2014int dmar_set_interrupt(struct intel_iommu *iommu)
2015{
2016	int irq, ret;
2017
2018	/*
2019	 * Check if the fault interrupt is already initialized.
2020	 */
2021	if (iommu->irq)
2022		return 0;
2023
2024	irq = dmar_alloc_hwirq(iommu->seq_id, iommu->node, iommu);
2025	if (irq > 0) {
2026		iommu->irq = irq;
2027	} else {
2028		pr_err("No free IRQ vectors\n");
2029		return -EINVAL;
2030	}
2031
2032	ret = request_irq(irq, dmar_fault, IRQF_NO_THREAD, iommu->name, iommu);
2033	if (ret)
2034		pr_err("Can't request irq\n");
2035	return ret;
2036}
2037
2038int enable_drhd_fault_handling(unsigned int cpu)
2039{
2040	struct dmar_drhd_unit *drhd;
2041	struct intel_iommu *iommu;
2042
2043	/*
2044	 * Enable fault control interrupt.
2045	 */
2046	guard(rwsem_read)(&dmar_global_lock);
2047	for_each_iommu(iommu, drhd) {
2048		u32 fault_status;
2049		int ret;
2050
2051		if (iommu->irq || iommu->node != cpu_to_node(cpu))
2052			continue;
2053
2054		ret = dmar_set_interrupt(iommu);
2055
2056		if (ret) {
2057			pr_err("DRHD %Lx: failed to enable fault, interrupt, ret %d\n",
2058			       (unsigned long long)drhd->reg_base_addr, ret);
2059			return -1;
2060		}
2061
2062		/*
2063		 * Clear any previous faults.
2064		 */
2065		dmar_fault(iommu->irq, iommu);
2066		fault_status = readl(iommu->reg + DMAR_FSTS_REG);
2067		writel(fault_status, iommu->reg + DMAR_FSTS_REG);
2068	}
2069
2070	return 0;
2071}
2072
2073/*
2074 * Re-enable Queued Invalidation interface.
2075 */
2076int dmar_reenable_qi(struct intel_iommu *iommu)
2077{
2078	if (!ecap_qis(iommu->ecap))
2079		return -ENOENT;
2080
2081	if (!iommu->qi)
2082		return -ENOENT;
2083
2084	/*
2085	 * First disable queued invalidation.
2086	 */
2087	dmar_disable_qi(iommu);
2088	/*
2089	 * Then enable queued invalidation again. Since there is no pending
2090	 * invalidation requests now, it's safe to re-enable queued
2091	 * invalidation.
2092	 */
2093	__dmar_enable_qi(iommu);
2094
2095	return 0;
2096}
2097
2098/*
2099 * Check interrupt remapping support in DMAR table description.
2100 */
2101int __init dmar_ir_support(void)
2102{
2103	struct acpi_table_dmar *dmar;
2104	dmar = (struct acpi_table_dmar *)dmar_tbl;
2105	if (!dmar)
2106		return 0;
2107	return dmar->flags & 0x1;
2108}
2109
2110/* Check whether DMAR units are in use */
2111static inline bool dmar_in_use(void)
2112{
2113	return irq_remapping_enabled || intel_iommu_enabled;
2114}
2115
2116static int __init dmar_free_unused_resources(void)
2117{
2118	struct dmar_drhd_unit *dmaru, *dmaru_n;
2119
2120	if (dmar_in_use())
2121		return 0;
2122
2123	if (dmar_dev_scope_status != 1 && !list_empty(&dmar_drhd_units))
2124		bus_unregister_notifier(&pci_bus_type, &dmar_pci_bus_nb);
2125
2126	down_write(&dmar_global_lock);
2127	list_for_each_entry_safe(dmaru, dmaru_n, &dmar_drhd_units, list) {
2128		list_del(&dmaru->list);
2129		dmar_free_drhd(dmaru);
2130	}
2131	up_write(&dmar_global_lock);
2132
2133	return 0;
2134}
2135
2136late_initcall(dmar_free_unused_resources);
 
2137
2138/*
2139 * DMAR Hotplug Support
2140 * For more details, please refer to Intel(R) Virtualization Technology
2141 * for Directed-IO Architecture Specifiction, Rev 2.2, Section 8.8
2142 * "Remapping Hardware Unit Hot Plug".
2143 */
2144static guid_t dmar_hp_guid =
2145	GUID_INIT(0xD8C1A3A6, 0xBE9B, 0x4C9B,
2146		  0x91, 0xBF, 0xC3, 0xCB, 0x81, 0xFC, 0x5D, 0xAF);
2147
2148/*
2149 * Currently there's only one revision and BIOS will not check the revision id,
2150 * so use 0 for safety.
2151 */
2152#define	DMAR_DSM_REV_ID			0
2153#define	DMAR_DSM_FUNC_DRHD		1
2154#define	DMAR_DSM_FUNC_ATSR		2
2155#define	DMAR_DSM_FUNC_RHSA		3
2156#define	DMAR_DSM_FUNC_SATC		4
2157
2158static inline bool dmar_detect_dsm(acpi_handle handle, int func)
2159{
2160	return acpi_check_dsm(handle, &dmar_hp_guid, DMAR_DSM_REV_ID, 1 << func);
2161}
2162
2163static int dmar_walk_dsm_resource(acpi_handle handle, int func,
2164				  dmar_res_handler_t handler, void *arg)
2165{
2166	int ret = -ENODEV;
2167	union acpi_object *obj;
2168	struct acpi_dmar_header *start;
2169	struct dmar_res_callback callback;
2170	static int res_type[] = {
2171		[DMAR_DSM_FUNC_DRHD] = ACPI_DMAR_TYPE_HARDWARE_UNIT,
2172		[DMAR_DSM_FUNC_ATSR] = ACPI_DMAR_TYPE_ROOT_ATS,
2173		[DMAR_DSM_FUNC_RHSA] = ACPI_DMAR_TYPE_HARDWARE_AFFINITY,
2174		[DMAR_DSM_FUNC_SATC] = ACPI_DMAR_TYPE_SATC,
2175	};
2176
2177	if (!dmar_detect_dsm(handle, func))
2178		return 0;
2179
2180	obj = acpi_evaluate_dsm_typed(handle, &dmar_hp_guid, DMAR_DSM_REV_ID,
2181				      func, NULL, ACPI_TYPE_BUFFER);
2182	if (!obj)
2183		return -ENODEV;
2184
2185	memset(&callback, 0, sizeof(callback));
2186	callback.cb[res_type[func]] = handler;
2187	callback.arg[res_type[func]] = arg;
2188	start = (struct acpi_dmar_header *)obj->buffer.pointer;
2189	ret = dmar_walk_remapping_entries(start, obj->buffer.length, &callback);
2190
2191	ACPI_FREE(obj);
2192
2193	return ret;
2194}
2195
2196static int dmar_hp_add_drhd(struct acpi_dmar_header *header, void *arg)
2197{
2198	int ret;
2199	struct dmar_drhd_unit *dmaru;
2200
2201	dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2202	if (!dmaru)
2203		return -ENODEV;
2204
2205	ret = dmar_ir_hotplug(dmaru, true);
2206	if (ret == 0)
2207		ret = dmar_iommu_hotplug(dmaru, true);
2208
2209	return ret;
2210}
2211
2212static int dmar_hp_remove_drhd(struct acpi_dmar_header *header, void *arg)
2213{
2214	int i, ret;
2215	struct device *dev;
2216	struct dmar_drhd_unit *dmaru;
2217
2218	dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2219	if (!dmaru)
2220		return 0;
2221
2222	/*
2223	 * All PCI devices managed by this unit should have been destroyed.
2224	 */
2225	if (!dmaru->include_all && dmaru->devices && dmaru->devices_cnt) {
2226		for_each_active_dev_scope(dmaru->devices,
2227					  dmaru->devices_cnt, i, dev)
2228			return -EBUSY;
2229	}
2230
2231	ret = dmar_ir_hotplug(dmaru, false);
2232	if (ret == 0)
2233		ret = dmar_iommu_hotplug(dmaru, false);
2234
2235	return ret;
2236}
2237
2238static int dmar_hp_release_drhd(struct acpi_dmar_header *header, void *arg)
2239{
2240	struct dmar_drhd_unit *dmaru;
2241
2242	dmaru = dmar_find_dmaru((struct acpi_dmar_hardware_unit *)header);
2243	if (dmaru) {
2244		list_del_rcu(&dmaru->list);
2245		synchronize_rcu();
2246		dmar_free_drhd(dmaru);
2247	}
2248
2249	return 0;
2250}
2251
2252static int dmar_hotplug_insert(acpi_handle handle)
2253{
2254	int ret;
2255	int drhd_count = 0;
2256
2257	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2258				     &dmar_validate_one_drhd, (void *)1);
2259	if (ret)
2260		goto out;
2261
2262	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2263				     &dmar_parse_one_drhd, (void *)&drhd_count);
2264	if (ret == 0 && drhd_count == 0) {
2265		pr_warn(FW_BUG "No DRHD structures in buffer returned by _DSM method\n");
2266		goto out;
2267	} else if (ret) {
2268		goto release_drhd;
2269	}
2270
2271	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_RHSA,
2272				     &dmar_parse_one_rhsa, NULL);
2273	if (ret)
2274		goto release_drhd;
2275
2276	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2277				     &dmar_parse_one_atsr, NULL);
2278	if (ret)
2279		goto release_atsr;
2280
2281	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2282				     &dmar_hp_add_drhd, NULL);
2283	if (!ret)
2284		return 0;
2285
2286	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2287			       &dmar_hp_remove_drhd, NULL);
2288release_atsr:
2289	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2290			       &dmar_release_one_atsr, NULL);
2291release_drhd:
2292	dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2293			       &dmar_hp_release_drhd, NULL);
2294out:
2295	return ret;
2296}
2297
2298static int dmar_hotplug_remove(acpi_handle handle)
2299{
2300	int ret;
2301
2302	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2303				     &dmar_check_one_atsr, NULL);
2304	if (ret)
2305		return ret;
2306
2307	ret = dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2308				     &dmar_hp_remove_drhd, NULL);
2309	if (ret == 0) {
2310		WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_ATSR,
2311					       &dmar_release_one_atsr, NULL));
2312		WARN_ON(dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2313					       &dmar_hp_release_drhd, NULL));
2314	} else {
2315		dmar_walk_dsm_resource(handle, DMAR_DSM_FUNC_DRHD,
2316				       &dmar_hp_add_drhd, NULL);
2317	}
2318
2319	return ret;
2320}
2321
2322static acpi_status dmar_get_dsm_handle(acpi_handle handle, u32 lvl,
2323				       void *context, void **retval)
2324{
2325	acpi_handle *phdl = retval;
2326
2327	if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2328		*phdl = handle;
2329		return AE_CTRL_TERMINATE;
2330	}
2331
2332	return AE_OK;
2333}
2334
2335static int dmar_device_hotplug(acpi_handle handle, bool insert)
2336{
2337	int ret;
2338	acpi_handle tmp = NULL;
2339	acpi_status status;
2340
2341	if (!dmar_in_use())
2342		return 0;
2343
2344	if (dmar_detect_dsm(handle, DMAR_DSM_FUNC_DRHD)) {
2345		tmp = handle;
2346	} else {
2347		status = acpi_walk_namespace(ACPI_TYPE_DEVICE, handle,
2348					     ACPI_UINT32_MAX,
2349					     dmar_get_dsm_handle,
2350					     NULL, NULL, &tmp);
2351		if (ACPI_FAILURE(status)) {
2352			pr_warn("Failed to locate _DSM method.\n");
2353			return -ENXIO;
2354		}
2355	}
2356	if (tmp == NULL)
2357		return 0;
2358
2359	down_write(&dmar_global_lock);
2360	if (insert)
2361		ret = dmar_hotplug_insert(tmp);
2362	else
2363		ret = dmar_hotplug_remove(tmp);
2364	up_write(&dmar_global_lock);
2365
2366	return ret;
2367}
2368
2369int dmar_device_add(acpi_handle handle)
2370{
2371	return dmar_device_hotplug(handle, true);
2372}
2373
2374int dmar_device_remove(acpi_handle handle)
2375{
2376	return dmar_device_hotplug(handle, false);
2377}
2378
2379/*
2380 * dmar_platform_optin - Is %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in DMAR table
2381 *
2382 * Returns true if the platform has %DMA_CTRL_PLATFORM_OPT_IN_FLAG set in
2383 * the ACPI DMAR table. This means that the platform boot firmware has made
2384 * sure no device can issue DMA outside of RMRR regions.
2385 */
2386bool dmar_platform_optin(void)
2387{
2388	struct acpi_table_dmar *dmar;
2389	acpi_status status;
2390	bool ret;
2391
2392	status = acpi_get_table(ACPI_SIG_DMAR, 0,
2393				(struct acpi_table_header **)&dmar);
2394	if (ACPI_FAILURE(status))
2395		return false;
2396
2397	ret = !!(dmar->flags & DMAR_PLATFORM_OPT_IN);
2398	acpi_put_table((struct acpi_table_header *)dmar);
2399
2400	return ret;
2401}
2402EXPORT_SYMBOL_GPL(dmar_platform_optin);