Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright © 2006-2014 Intel Corporation.
4 *
5 * Authors: David Woodhouse <dwmw2@infradead.org>,
6 * Ashok Raj <ashok.raj@intel.com>,
7 * Shaohua Li <shaohua.li@intel.com>,
8 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
9 * Fenghua Yu <fenghua.yu@intel.com>
10 * Joerg Roedel <jroedel@suse.de>
11 */
12
13#define pr_fmt(fmt) "DMAR: " fmt
14#define dev_fmt(fmt) pr_fmt(fmt)
15
16#include <linux/crash_dump.h>
17#include <linux/dma-direct.h>
18#include <linux/dmi.h>
19#include <linux/memory.h>
20#include <linux/pci.h>
21#include <linux/pci-ats.h>
22#include <linux/spinlock.h>
23#include <linux/syscore_ops.h>
24#include <linux/tboot.h>
25#include <uapi/linux/iommufd.h>
26
27#include "iommu.h"
28#include "../dma-iommu.h"
29#include "../irq_remapping.h"
30#include "../iommu-pages.h"
31#include "pasid.h"
32#include "cap_audit.h"
33#include "perfmon.h"
34
35#define ROOT_SIZE VTD_PAGE_SIZE
36#define CONTEXT_SIZE VTD_PAGE_SIZE
37
38#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
39#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
40#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
41#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
42
43#define IOAPIC_RANGE_START (0xfee00000)
44#define IOAPIC_RANGE_END (0xfeefffff)
45#define IOVA_START_ADDR (0x1000)
46
47#define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
48
49#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << ((gaw) - VTD_PAGE_SHIFT)) - 1)
50#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << (gaw)) - 1)
51
52/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
53 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
54#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
55 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
56#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
57
58static void __init check_tylersburg_isoch(void);
59static int rwbf_quirk;
60
61/*
62 * set to 1 to panic kernel if can't successfully enable VT-d
63 * (used when kernel is launched w/ TXT)
64 */
65static int force_on = 0;
66static int intel_iommu_tboot_noforce;
67static int no_platform_optin;
68
69#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
70
71/*
72 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
73 * if marked present.
74 */
75static phys_addr_t root_entry_lctp(struct root_entry *re)
76{
77 if (!(re->lo & 1))
78 return 0;
79
80 return re->lo & VTD_PAGE_MASK;
81}
82
83/*
84 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
85 * if marked present.
86 */
87static phys_addr_t root_entry_uctp(struct root_entry *re)
88{
89 if (!(re->hi & 1))
90 return 0;
91
92 return re->hi & VTD_PAGE_MASK;
93}
94
95static int device_rid_cmp_key(const void *key, const struct rb_node *node)
96{
97 struct device_domain_info *info =
98 rb_entry(node, struct device_domain_info, node);
99 const u16 *rid_lhs = key;
100
101 if (*rid_lhs < PCI_DEVID(info->bus, info->devfn))
102 return -1;
103
104 if (*rid_lhs > PCI_DEVID(info->bus, info->devfn))
105 return 1;
106
107 return 0;
108}
109
110static int device_rid_cmp(struct rb_node *lhs, const struct rb_node *rhs)
111{
112 struct device_domain_info *info =
113 rb_entry(lhs, struct device_domain_info, node);
114 u16 key = PCI_DEVID(info->bus, info->devfn);
115
116 return device_rid_cmp_key(&key, rhs);
117}
118
119/*
120 * Looks up an IOMMU-probed device using its source ID.
121 *
122 * Returns the pointer to the device if there is a match. Otherwise,
123 * returns NULL.
124 *
125 * Note that this helper doesn't guarantee that the device won't be
126 * released by the iommu subsystem after being returned. The caller
127 * should use its own synchronization mechanism to avoid the device
128 * being released during its use if its possibly the case.
129 */
130struct device *device_rbtree_find(struct intel_iommu *iommu, u16 rid)
131{
132 struct device_domain_info *info = NULL;
133 struct rb_node *node;
134 unsigned long flags;
135
136 spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
137 node = rb_find(&rid, &iommu->device_rbtree, device_rid_cmp_key);
138 if (node)
139 info = rb_entry(node, struct device_domain_info, node);
140 spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
141
142 return info ? info->dev : NULL;
143}
144
145static int device_rbtree_insert(struct intel_iommu *iommu,
146 struct device_domain_info *info)
147{
148 struct rb_node *curr;
149 unsigned long flags;
150
151 spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
152 curr = rb_find_add(&info->node, &iommu->device_rbtree, device_rid_cmp);
153 spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
154 if (WARN_ON(curr))
155 return -EEXIST;
156
157 return 0;
158}
159
160static void device_rbtree_remove(struct device_domain_info *info)
161{
162 struct intel_iommu *iommu = info->iommu;
163 unsigned long flags;
164
165 spin_lock_irqsave(&iommu->device_rbtree_lock, flags);
166 rb_erase(&info->node, &iommu->device_rbtree);
167 spin_unlock_irqrestore(&iommu->device_rbtree_lock, flags);
168}
169
170struct dmar_rmrr_unit {
171 struct list_head list; /* list of rmrr units */
172 struct acpi_dmar_header *hdr; /* ACPI header */
173 u64 base_address; /* reserved base address*/
174 u64 end_address; /* reserved end address */
175 struct dmar_dev_scope *devices; /* target devices */
176 int devices_cnt; /* target device count */
177};
178
179struct dmar_atsr_unit {
180 struct list_head list; /* list of ATSR units */
181 struct acpi_dmar_header *hdr; /* ACPI header */
182 struct dmar_dev_scope *devices; /* target devices */
183 int devices_cnt; /* target device count */
184 u8 include_all:1; /* include all ports */
185};
186
187struct dmar_satc_unit {
188 struct list_head list; /* list of SATC units */
189 struct acpi_dmar_header *hdr; /* ACPI header */
190 struct dmar_dev_scope *devices; /* target devices */
191 struct intel_iommu *iommu; /* the corresponding iommu */
192 int devices_cnt; /* target device count */
193 u8 atc_required:1; /* ATS is required */
194};
195
196static LIST_HEAD(dmar_atsr_units);
197static LIST_HEAD(dmar_rmrr_units);
198static LIST_HEAD(dmar_satc_units);
199
200#define for_each_rmrr_units(rmrr) \
201 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
202
203static void intel_iommu_domain_free(struct iommu_domain *domain);
204
205int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);
206int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON);
207
208int intel_iommu_enabled = 0;
209EXPORT_SYMBOL_GPL(intel_iommu_enabled);
210
211static int intel_iommu_superpage = 1;
212static int iommu_identity_mapping;
213static int iommu_skip_te_disable;
214static int disable_igfx_iommu;
215
216#define IDENTMAP_AZALIA 4
217
218const struct iommu_ops intel_iommu_ops;
219static const struct iommu_dirty_ops intel_dirty_ops;
220
221static bool translation_pre_enabled(struct intel_iommu *iommu)
222{
223 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
224}
225
226static void clear_translation_pre_enabled(struct intel_iommu *iommu)
227{
228 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
229}
230
231static void init_translation_status(struct intel_iommu *iommu)
232{
233 u32 gsts;
234
235 gsts = readl(iommu->reg + DMAR_GSTS_REG);
236 if (gsts & DMA_GSTS_TES)
237 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
238}
239
240static int __init intel_iommu_setup(char *str)
241{
242 if (!str)
243 return -EINVAL;
244
245 while (*str) {
246 if (!strncmp(str, "on", 2)) {
247 dmar_disabled = 0;
248 pr_info("IOMMU enabled\n");
249 } else if (!strncmp(str, "off", 3)) {
250 dmar_disabled = 1;
251 no_platform_optin = 1;
252 pr_info("IOMMU disabled\n");
253 } else if (!strncmp(str, "igfx_off", 8)) {
254 disable_igfx_iommu = 1;
255 pr_info("Disable GFX device mapping\n");
256 } else if (!strncmp(str, "forcedac", 8)) {
257 pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n");
258 iommu_dma_forcedac = true;
259 } else if (!strncmp(str, "strict", 6)) {
260 pr_warn("intel_iommu=strict deprecated; use iommu.strict=1 instead\n");
261 iommu_set_dma_strict();
262 } else if (!strncmp(str, "sp_off", 6)) {
263 pr_info("Disable supported super page\n");
264 intel_iommu_superpage = 0;
265 } else if (!strncmp(str, "sm_on", 5)) {
266 pr_info("Enable scalable mode if hardware supports\n");
267 intel_iommu_sm = 1;
268 } else if (!strncmp(str, "sm_off", 6)) {
269 pr_info("Scalable mode is disallowed\n");
270 intel_iommu_sm = 0;
271 } else if (!strncmp(str, "tboot_noforce", 13)) {
272 pr_info("Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
273 intel_iommu_tboot_noforce = 1;
274 } else {
275 pr_notice("Unknown option - '%s'\n", str);
276 }
277
278 str += strcspn(str, ",");
279 while (*str == ',')
280 str++;
281 }
282
283 return 1;
284}
285__setup("intel_iommu=", intel_iommu_setup);
286
287static int domain_pfn_supported(struct dmar_domain *domain, unsigned long pfn)
288{
289 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
290
291 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
292}
293
294/*
295 * Calculate the Supported Adjusted Guest Address Widths of an IOMMU.
296 * Refer to 11.4.2 of the VT-d spec for the encoding of each bit of
297 * the returned SAGAW.
298 */
299static unsigned long __iommu_calculate_sagaw(struct intel_iommu *iommu)
300{
301 unsigned long fl_sagaw, sl_sagaw;
302
303 fl_sagaw = BIT(2) | (cap_fl5lp_support(iommu->cap) ? BIT(3) : 0);
304 sl_sagaw = cap_sagaw(iommu->cap);
305
306 /* Second level only. */
307 if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))
308 return sl_sagaw;
309
310 /* First level only. */
311 if (!ecap_slts(iommu->ecap))
312 return fl_sagaw;
313
314 return fl_sagaw & sl_sagaw;
315}
316
317static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
318{
319 unsigned long sagaw;
320 int agaw;
321
322 sagaw = __iommu_calculate_sagaw(iommu);
323 for (agaw = width_to_agaw(max_gaw); agaw >= 0; agaw--) {
324 if (test_bit(agaw, &sagaw))
325 break;
326 }
327
328 return agaw;
329}
330
331/*
332 * Calculate max SAGAW for each iommu.
333 */
334int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
335{
336 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
337}
338
339/*
340 * calculate agaw for each iommu.
341 * "SAGAW" may be different across iommus, use a default agaw, and
342 * get a supported less agaw for iommus that don't support the default agaw.
343 */
344int iommu_calculate_agaw(struct intel_iommu *iommu)
345{
346 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
347}
348
349static bool iommu_paging_structure_coherency(struct intel_iommu *iommu)
350{
351 return sm_supported(iommu) ?
352 ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap);
353}
354
355/* Return the super pagesize bitmap if supported. */
356static unsigned long domain_super_pgsize_bitmap(struct dmar_domain *domain)
357{
358 unsigned long bitmap = 0;
359
360 /*
361 * 1-level super page supports page size of 2MiB, 2-level super page
362 * supports page size of both 2MiB and 1GiB.
363 */
364 if (domain->iommu_superpage == 1)
365 bitmap |= SZ_2M;
366 else if (domain->iommu_superpage == 2)
367 bitmap |= SZ_2M | SZ_1G;
368
369 return bitmap;
370}
371
372struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
373 u8 devfn, int alloc)
374{
375 struct root_entry *root = &iommu->root_entry[bus];
376 struct context_entry *context;
377 u64 *entry;
378
379 /*
380 * Except that the caller requested to allocate a new entry,
381 * returning a copied context entry makes no sense.
382 */
383 if (!alloc && context_copied(iommu, bus, devfn))
384 return NULL;
385
386 entry = &root->lo;
387 if (sm_supported(iommu)) {
388 if (devfn >= 0x80) {
389 devfn -= 0x80;
390 entry = &root->hi;
391 }
392 devfn *= 2;
393 }
394 if (*entry & 1)
395 context = phys_to_virt(*entry & VTD_PAGE_MASK);
396 else {
397 unsigned long phy_addr;
398 if (!alloc)
399 return NULL;
400
401 context = iommu_alloc_page_node(iommu->node, GFP_ATOMIC);
402 if (!context)
403 return NULL;
404
405 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
406 phy_addr = virt_to_phys((void *)context);
407 *entry = phy_addr | 1;
408 __iommu_flush_cache(iommu, entry, sizeof(*entry));
409 }
410 return &context[devfn];
411}
412
413/**
414 * is_downstream_to_pci_bridge - test if a device belongs to the PCI
415 * sub-hierarchy of a candidate PCI-PCI bridge
416 * @dev: candidate PCI device belonging to @bridge PCI sub-hierarchy
417 * @bridge: the candidate PCI-PCI bridge
418 *
419 * Return: true if @dev belongs to @bridge PCI sub-hierarchy, else false.
420 */
421static bool
422is_downstream_to_pci_bridge(struct device *dev, struct device *bridge)
423{
424 struct pci_dev *pdev, *pbridge;
425
426 if (!dev_is_pci(dev) || !dev_is_pci(bridge))
427 return false;
428
429 pdev = to_pci_dev(dev);
430 pbridge = to_pci_dev(bridge);
431
432 if (pbridge->subordinate &&
433 pbridge->subordinate->number <= pdev->bus->number &&
434 pbridge->subordinate->busn_res.end >= pdev->bus->number)
435 return true;
436
437 return false;
438}
439
440static bool quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
441{
442 struct dmar_drhd_unit *drhd;
443 u32 vtbar;
444 int rc;
445
446 /* We know that this device on this chipset has its own IOMMU.
447 * If we find it under a different IOMMU, then the BIOS is lying
448 * to us. Hope that the IOMMU for this device is actually
449 * disabled, and it needs no translation...
450 */
451 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
452 if (rc) {
453 /* "can't" happen */
454 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
455 return false;
456 }
457 vtbar &= 0xffff0000;
458
459 /* we know that the this iommu should be at offset 0xa000 from vtbar */
460 drhd = dmar_find_matched_drhd_unit(pdev);
461 if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
462 pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
463 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
464 return true;
465 }
466
467 return false;
468}
469
470static bool iommu_is_dummy(struct intel_iommu *iommu, struct device *dev)
471{
472 if (!iommu || iommu->drhd->ignored)
473 return true;
474
475 if (dev_is_pci(dev)) {
476 struct pci_dev *pdev = to_pci_dev(dev);
477
478 if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
479 pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SNB &&
480 quirk_ioat_snb_local_iommu(pdev))
481 return true;
482 }
483
484 return false;
485}
486
487static struct intel_iommu *device_lookup_iommu(struct device *dev, u8 *bus, u8 *devfn)
488{
489 struct dmar_drhd_unit *drhd = NULL;
490 struct pci_dev *pdev = NULL;
491 struct intel_iommu *iommu;
492 struct device *tmp;
493 u16 segment = 0;
494 int i;
495
496 if (!dev)
497 return NULL;
498
499 if (dev_is_pci(dev)) {
500 struct pci_dev *pf_pdev;
501
502 pdev = pci_real_dma_dev(to_pci_dev(dev));
503
504 /* VFs aren't listed in scope tables; we need to look up
505 * the PF instead to find the IOMMU. */
506 pf_pdev = pci_physfn(pdev);
507 dev = &pf_pdev->dev;
508 segment = pci_domain_nr(pdev->bus);
509 } else if (has_acpi_companion(dev))
510 dev = &ACPI_COMPANION(dev)->dev;
511
512 rcu_read_lock();
513 for_each_iommu(iommu, drhd) {
514 if (pdev && segment != drhd->segment)
515 continue;
516
517 for_each_active_dev_scope(drhd->devices,
518 drhd->devices_cnt, i, tmp) {
519 if (tmp == dev) {
520 /* For a VF use its original BDF# not that of the PF
521 * which we used for the IOMMU lookup. Strictly speaking
522 * we could do this for all PCI devices; we only need to
523 * get the BDF# from the scope table for ACPI matches. */
524 if (pdev && pdev->is_virtfn)
525 goto got_pdev;
526
527 if (bus && devfn) {
528 *bus = drhd->devices[i].bus;
529 *devfn = drhd->devices[i].devfn;
530 }
531 goto out;
532 }
533
534 if (is_downstream_to_pci_bridge(dev, tmp))
535 goto got_pdev;
536 }
537
538 if (pdev && drhd->include_all) {
539got_pdev:
540 if (bus && devfn) {
541 *bus = pdev->bus->number;
542 *devfn = pdev->devfn;
543 }
544 goto out;
545 }
546 }
547 iommu = NULL;
548out:
549 if (iommu_is_dummy(iommu, dev))
550 iommu = NULL;
551
552 rcu_read_unlock();
553
554 return iommu;
555}
556
557static void domain_flush_cache(struct dmar_domain *domain,
558 void *addr, int size)
559{
560 if (!domain->iommu_coherency)
561 clflush_cache_range(addr, size);
562}
563
564static void free_context_table(struct intel_iommu *iommu)
565{
566 struct context_entry *context;
567 int i;
568
569 if (!iommu->root_entry)
570 return;
571
572 for (i = 0; i < ROOT_ENTRY_NR; i++) {
573 context = iommu_context_addr(iommu, i, 0, 0);
574 if (context)
575 iommu_free_page(context);
576
577 if (!sm_supported(iommu))
578 continue;
579
580 context = iommu_context_addr(iommu, i, 0x80, 0);
581 if (context)
582 iommu_free_page(context);
583 }
584
585 iommu_free_page(iommu->root_entry);
586 iommu->root_entry = NULL;
587}
588
589#ifdef CONFIG_DMAR_DEBUG
590static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn,
591 u8 bus, u8 devfn, struct dma_pte *parent, int level)
592{
593 struct dma_pte *pte;
594 int offset;
595
596 while (1) {
597 offset = pfn_level_offset(pfn, level);
598 pte = &parent[offset];
599
600 pr_info("pte level: %d, pte value: 0x%016llx\n", level, pte->val);
601
602 if (!dma_pte_present(pte)) {
603 pr_info("page table not present at level %d\n", level - 1);
604 break;
605 }
606
607 if (level == 1 || dma_pte_superpage(pte))
608 break;
609
610 parent = phys_to_virt(dma_pte_addr(pte));
611 level--;
612 }
613}
614
615void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,
616 unsigned long long addr, u32 pasid)
617{
618 struct pasid_dir_entry *dir, *pde;
619 struct pasid_entry *entries, *pte;
620 struct context_entry *ctx_entry;
621 struct root_entry *rt_entry;
622 int i, dir_index, index, level;
623 u8 devfn = source_id & 0xff;
624 u8 bus = source_id >> 8;
625 struct dma_pte *pgtable;
626
627 pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr);
628
629 /* root entry dump */
630 if (!iommu->root_entry) {
631 pr_info("root table is not present\n");
632 return;
633 }
634 rt_entry = &iommu->root_entry[bus];
635
636 if (sm_supported(iommu))
637 pr_info("scalable mode root entry: hi 0x%016llx, low 0x%016llx\n",
638 rt_entry->hi, rt_entry->lo);
639 else
640 pr_info("root entry: 0x%016llx", rt_entry->lo);
641
642 /* context entry dump */
643 ctx_entry = iommu_context_addr(iommu, bus, devfn, 0);
644 if (!ctx_entry) {
645 pr_info("context table is not present\n");
646 return;
647 }
648
649 pr_info("context entry: hi 0x%016llx, low 0x%016llx\n",
650 ctx_entry->hi, ctx_entry->lo);
651
652 /* legacy mode does not require PASID entries */
653 if (!sm_supported(iommu)) {
654 if (!context_present(ctx_entry)) {
655 pr_info("legacy mode page table is not present\n");
656 return;
657 }
658 level = agaw_to_level(ctx_entry->hi & 7);
659 pgtable = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
660 goto pgtable_walk;
661 }
662
663 if (!context_present(ctx_entry)) {
664 pr_info("pasid directory table is not present\n");
665 return;
666 }
667
668 /* get the pointer to pasid directory entry */
669 dir = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
670
671 /* For request-without-pasid, get the pasid from context entry */
672 if (intel_iommu_sm && pasid == IOMMU_PASID_INVALID)
673 pasid = IOMMU_NO_PASID;
674
675 dir_index = pasid >> PASID_PDE_SHIFT;
676 pde = &dir[dir_index];
677 pr_info("pasid dir entry: 0x%016llx\n", pde->val);
678
679 /* get the pointer to the pasid table entry */
680 entries = get_pasid_table_from_pde(pde);
681 if (!entries) {
682 pr_info("pasid table is not present\n");
683 return;
684 }
685 index = pasid & PASID_PTE_MASK;
686 pte = &entries[index];
687 for (i = 0; i < ARRAY_SIZE(pte->val); i++)
688 pr_info("pasid table entry[%d]: 0x%016llx\n", i, pte->val[i]);
689
690 if (!pasid_pte_is_present(pte)) {
691 pr_info("scalable mode page table is not present\n");
692 return;
693 }
694
695 if (pasid_pte_get_pgtt(pte) == PASID_ENTRY_PGTT_FL_ONLY) {
696 level = pte->val[2] & BIT_ULL(2) ? 5 : 4;
697 pgtable = phys_to_virt(pte->val[2] & VTD_PAGE_MASK);
698 } else {
699 level = agaw_to_level((pte->val[0] >> 2) & 0x7);
700 pgtable = phys_to_virt(pte->val[0] & VTD_PAGE_MASK);
701 }
702
703pgtable_walk:
704 pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn, pgtable, level);
705}
706#endif
707
708static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
709 unsigned long pfn, int *target_level,
710 gfp_t gfp)
711{
712 struct dma_pte *parent, *pte;
713 int level = agaw_to_level(domain->agaw);
714 int offset;
715
716 if (!domain_pfn_supported(domain, pfn))
717 /* Address beyond IOMMU's addressing capabilities. */
718 return NULL;
719
720 parent = domain->pgd;
721
722 while (1) {
723 void *tmp_page;
724
725 offset = pfn_level_offset(pfn, level);
726 pte = &parent[offset];
727 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
728 break;
729 if (level == *target_level)
730 break;
731
732 if (!dma_pte_present(pte)) {
733 uint64_t pteval, tmp;
734
735 tmp_page = iommu_alloc_page_node(domain->nid, gfp);
736
737 if (!tmp_page)
738 return NULL;
739
740 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
741 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
742 if (domain->use_first_level)
743 pteval |= DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
744
745 tmp = 0ULL;
746 if (!try_cmpxchg64(&pte->val, &tmp, pteval))
747 /* Someone else set it while we were thinking; use theirs. */
748 iommu_free_page(tmp_page);
749 else
750 domain_flush_cache(domain, pte, sizeof(*pte));
751 }
752 if (level == 1)
753 break;
754
755 parent = phys_to_virt(dma_pte_addr(pte));
756 level--;
757 }
758
759 if (!*target_level)
760 *target_level = level;
761
762 return pte;
763}
764
765/* return address's pte at specific level */
766static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
767 unsigned long pfn,
768 int level, int *large_page)
769{
770 struct dma_pte *parent, *pte;
771 int total = agaw_to_level(domain->agaw);
772 int offset;
773
774 parent = domain->pgd;
775 while (level <= total) {
776 offset = pfn_level_offset(pfn, total);
777 pte = &parent[offset];
778 if (level == total)
779 return pte;
780
781 if (!dma_pte_present(pte)) {
782 *large_page = total;
783 break;
784 }
785
786 if (dma_pte_superpage(pte)) {
787 *large_page = total;
788 return pte;
789 }
790
791 parent = phys_to_virt(dma_pte_addr(pte));
792 total--;
793 }
794 return NULL;
795}
796
797/* clear last level pte, a tlb flush should be followed */
798static void dma_pte_clear_range(struct dmar_domain *domain,
799 unsigned long start_pfn,
800 unsigned long last_pfn)
801{
802 unsigned int large_page;
803 struct dma_pte *first_pte, *pte;
804
805 if (WARN_ON(!domain_pfn_supported(domain, last_pfn)) ||
806 WARN_ON(start_pfn > last_pfn))
807 return;
808
809 /* we don't need lock here; nobody else touches the iova range */
810 do {
811 large_page = 1;
812 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
813 if (!pte) {
814 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
815 continue;
816 }
817 do {
818 dma_clear_pte(pte);
819 start_pfn += lvl_to_nr_pages(large_page);
820 pte++;
821 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
822
823 domain_flush_cache(domain, first_pte,
824 (void *)pte - (void *)first_pte);
825
826 } while (start_pfn && start_pfn <= last_pfn);
827}
828
829static void dma_pte_free_level(struct dmar_domain *domain, int level,
830 int retain_level, struct dma_pte *pte,
831 unsigned long pfn, unsigned long start_pfn,
832 unsigned long last_pfn)
833{
834 pfn = max(start_pfn, pfn);
835 pte = &pte[pfn_level_offset(pfn, level)];
836
837 do {
838 unsigned long level_pfn;
839 struct dma_pte *level_pte;
840
841 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
842 goto next;
843
844 level_pfn = pfn & level_mask(level);
845 level_pte = phys_to_virt(dma_pte_addr(pte));
846
847 if (level > 2) {
848 dma_pte_free_level(domain, level - 1, retain_level,
849 level_pte, level_pfn, start_pfn,
850 last_pfn);
851 }
852
853 /*
854 * Free the page table if we're below the level we want to
855 * retain and the range covers the entire table.
856 */
857 if (level < retain_level && !(start_pfn > level_pfn ||
858 last_pfn < level_pfn + level_size(level) - 1)) {
859 dma_clear_pte(pte);
860 domain_flush_cache(domain, pte, sizeof(*pte));
861 iommu_free_page(level_pte);
862 }
863next:
864 pfn += level_size(level);
865 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
866}
867
868/*
869 * clear last level (leaf) ptes and free page table pages below the
870 * level we wish to keep intact.
871 */
872static void dma_pte_free_pagetable(struct dmar_domain *domain,
873 unsigned long start_pfn,
874 unsigned long last_pfn,
875 int retain_level)
876{
877 dma_pte_clear_range(domain, start_pfn, last_pfn);
878
879 /* We don't need lock here; nobody else touches the iova range */
880 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
881 domain->pgd, 0, start_pfn, last_pfn);
882
883 /* free pgd */
884 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
885 iommu_free_page(domain->pgd);
886 domain->pgd = NULL;
887 }
888}
889
890/* When a page at a given level is being unlinked from its parent, we don't
891 need to *modify* it at all. All we need to do is make a list of all the
892 pages which can be freed just as soon as we've flushed the IOTLB and we
893 know the hardware page-walk will no longer touch them.
894 The 'pte' argument is the *parent* PTE, pointing to the page that is to
895 be freed. */
896static void dma_pte_list_pagetables(struct dmar_domain *domain,
897 int level, struct dma_pte *pte,
898 struct list_head *freelist)
899{
900 struct page *pg;
901
902 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
903 list_add_tail(&pg->lru, freelist);
904
905 if (level == 1)
906 return;
907
908 pte = page_address(pg);
909 do {
910 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
911 dma_pte_list_pagetables(domain, level - 1, pte, freelist);
912 pte++;
913 } while (!first_pte_in_page(pte));
914}
915
916static void dma_pte_clear_level(struct dmar_domain *domain, int level,
917 struct dma_pte *pte, unsigned long pfn,
918 unsigned long start_pfn, unsigned long last_pfn,
919 struct list_head *freelist)
920{
921 struct dma_pte *first_pte = NULL, *last_pte = NULL;
922
923 pfn = max(start_pfn, pfn);
924 pte = &pte[pfn_level_offset(pfn, level)];
925
926 do {
927 unsigned long level_pfn = pfn & level_mask(level);
928
929 if (!dma_pte_present(pte))
930 goto next;
931
932 /* If range covers entire pagetable, free it */
933 if (start_pfn <= level_pfn &&
934 last_pfn >= level_pfn + level_size(level) - 1) {
935 /* These suborbinate page tables are going away entirely. Don't
936 bother to clear them; we're just going to *free* them. */
937 if (level > 1 && !dma_pte_superpage(pte))
938 dma_pte_list_pagetables(domain, level - 1, pte, freelist);
939
940 dma_clear_pte(pte);
941 if (!first_pte)
942 first_pte = pte;
943 last_pte = pte;
944 } else if (level > 1) {
945 /* Recurse down into a level that isn't *entirely* obsolete */
946 dma_pte_clear_level(domain, level - 1,
947 phys_to_virt(dma_pte_addr(pte)),
948 level_pfn, start_pfn, last_pfn,
949 freelist);
950 }
951next:
952 pfn = level_pfn + level_size(level);
953 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
954
955 if (first_pte)
956 domain_flush_cache(domain, first_pte,
957 (void *)++last_pte - (void *)first_pte);
958}
959
960/* We can't just free the pages because the IOMMU may still be walking
961 the page tables, and may have cached the intermediate levels. The
962 pages can only be freed after the IOTLB flush has been done. */
963static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn,
964 unsigned long last_pfn, struct list_head *freelist)
965{
966 if (WARN_ON(!domain_pfn_supported(domain, last_pfn)) ||
967 WARN_ON(start_pfn > last_pfn))
968 return;
969
970 /* we don't need lock here; nobody else touches the iova range */
971 dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
972 domain->pgd, 0, start_pfn, last_pfn, freelist);
973
974 /* free pgd */
975 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
976 struct page *pgd_page = virt_to_page(domain->pgd);
977 list_add_tail(&pgd_page->lru, freelist);
978 domain->pgd = NULL;
979 }
980}
981
982/* iommu handling */
983static int iommu_alloc_root_entry(struct intel_iommu *iommu)
984{
985 struct root_entry *root;
986
987 root = iommu_alloc_page_node(iommu->node, GFP_ATOMIC);
988 if (!root) {
989 pr_err("Allocating root entry for %s failed\n",
990 iommu->name);
991 return -ENOMEM;
992 }
993
994 __iommu_flush_cache(iommu, root, ROOT_SIZE);
995 iommu->root_entry = root;
996
997 return 0;
998}
999
1000static void iommu_set_root_entry(struct intel_iommu *iommu)
1001{
1002 u64 addr;
1003 u32 sts;
1004 unsigned long flag;
1005
1006 addr = virt_to_phys(iommu->root_entry);
1007 if (sm_supported(iommu))
1008 addr |= DMA_RTADDR_SMT;
1009
1010 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1011 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1012
1013 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1014
1015 /* Make sure hardware complete it */
1016 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1017 readl, (sts & DMA_GSTS_RTPS), sts);
1018
1019 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1020
1021 /*
1022 * Hardware invalidates all DMA remapping hardware translation
1023 * caches as part of SRTP flow.
1024 */
1025 if (cap_esrtps(iommu->cap))
1026 return;
1027
1028 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1029 if (sm_supported(iommu))
1030 qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);
1031 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1032}
1033
1034void iommu_flush_write_buffer(struct intel_iommu *iommu)
1035{
1036 u32 val;
1037 unsigned long flag;
1038
1039 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1040 return;
1041
1042 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1043 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1044
1045 /* Make sure hardware complete it */
1046 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1047 readl, (!(val & DMA_GSTS_WBFS)), val);
1048
1049 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1050}
1051
1052/* return value determine if we need a write buffer flush */
1053static void __iommu_flush_context(struct intel_iommu *iommu,
1054 u16 did, u16 source_id, u8 function_mask,
1055 u64 type)
1056{
1057 u64 val = 0;
1058 unsigned long flag;
1059
1060 switch (type) {
1061 case DMA_CCMD_GLOBAL_INVL:
1062 val = DMA_CCMD_GLOBAL_INVL;
1063 break;
1064 case DMA_CCMD_DOMAIN_INVL:
1065 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1066 break;
1067 case DMA_CCMD_DEVICE_INVL:
1068 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1069 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1070 break;
1071 default:
1072 pr_warn("%s: Unexpected context-cache invalidation type 0x%llx\n",
1073 iommu->name, type);
1074 return;
1075 }
1076 val |= DMA_CCMD_ICC;
1077
1078 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1079 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1080
1081 /* Make sure hardware complete it */
1082 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1083 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1084
1085 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1086}
1087
1088void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did, u64 addr,
1089 unsigned int size_order, u64 type)
1090{
1091 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1092 u64 val = 0, val_iva = 0;
1093 unsigned long flag;
1094
1095 switch (type) {
1096 case DMA_TLB_GLOBAL_FLUSH:
1097 /* global flush doesn't need set IVA_REG */
1098 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1099 break;
1100 case DMA_TLB_DSI_FLUSH:
1101 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1102 break;
1103 case DMA_TLB_PSI_FLUSH:
1104 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1105 /* IH bit is passed in as part of address */
1106 val_iva = size_order | addr;
1107 break;
1108 default:
1109 pr_warn("%s: Unexpected iotlb invalidation type 0x%llx\n",
1110 iommu->name, type);
1111 return;
1112 }
1113
1114 if (cap_write_drain(iommu->cap))
1115 val |= DMA_TLB_WRITE_DRAIN;
1116
1117 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1118 /* Note: Only uses first TLB reg currently */
1119 if (val_iva)
1120 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1121 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1122
1123 /* Make sure hardware complete it */
1124 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1125 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1126
1127 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1128
1129 /* check IOTLB invalidation granularity */
1130 if (DMA_TLB_IAIG(val) == 0)
1131 pr_err("Flush IOTLB failed\n");
1132 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1133 pr_debug("TLB flush request %Lx, actual %Lx\n",
1134 (unsigned long long)DMA_TLB_IIRG(type),
1135 (unsigned long long)DMA_TLB_IAIG(val));
1136}
1137
1138static struct device_domain_info *
1139domain_lookup_dev_info(struct dmar_domain *domain,
1140 struct intel_iommu *iommu, u8 bus, u8 devfn)
1141{
1142 struct device_domain_info *info;
1143 unsigned long flags;
1144
1145 spin_lock_irqsave(&domain->lock, flags);
1146 list_for_each_entry(info, &domain->devices, link) {
1147 if (info->iommu == iommu && info->bus == bus &&
1148 info->devfn == devfn) {
1149 spin_unlock_irqrestore(&domain->lock, flags);
1150 return info;
1151 }
1152 }
1153 spin_unlock_irqrestore(&domain->lock, flags);
1154
1155 return NULL;
1156}
1157
1158/*
1159 * The extra devTLB flush quirk impacts those QAT devices with PCI device
1160 * IDs ranging from 0x4940 to 0x4943. It is exempted from risky_device()
1161 * check because it applies only to the built-in QAT devices and it doesn't
1162 * grant additional privileges.
1163 */
1164#define BUGGY_QAT_DEVID_MASK 0x4940
1165static bool dev_needs_extra_dtlb_flush(struct pci_dev *pdev)
1166{
1167 if (pdev->vendor != PCI_VENDOR_ID_INTEL)
1168 return false;
1169
1170 if ((pdev->device & 0xfffc) != BUGGY_QAT_DEVID_MASK)
1171 return false;
1172
1173 return true;
1174}
1175
1176static void iommu_enable_pci_caps(struct device_domain_info *info)
1177{
1178 struct pci_dev *pdev;
1179
1180 if (!dev_is_pci(info->dev))
1181 return;
1182
1183 pdev = to_pci_dev(info->dev);
1184 if (info->ats_supported && pci_ats_page_aligned(pdev) &&
1185 !pci_enable_ats(pdev, VTD_PAGE_SHIFT))
1186 info->ats_enabled = 1;
1187}
1188
1189static void iommu_disable_pci_caps(struct device_domain_info *info)
1190{
1191 struct pci_dev *pdev;
1192
1193 if (!dev_is_pci(info->dev))
1194 return;
1195
1196 pdev = to_pci_dev(info->dev);
1197
1198 if (info->ats_enabled) {
1199 pci_disable_ats(pdev);
1200 info->ats_enabled = 0;
1201 }
1202}
1203
1204static void intel_flush_iotlb_all(struct iommu_domain *domain)
1205{
1206 cache_tag_flush_all(to_dmar_domain(domain));
1207}
1208
1209static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1210{
1211 u32 pmen;
1212 unsigned long flags;
1213
1214 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1215 return;
1216
1217 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1218 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1219 pmen &= ~DMA_PMEN_EPM;
1220 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1221
1222 /* wait for the protected region status bit to clear */
1223 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1224 readl, !(pmen & DMA_PMEN_PRS), pmen);
1225
1226 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1227}
1228
1229static void iommu_enable_translation(struct intel_iommu *iommu)
1230{
1231 u32 sts;
1232 unsigned long flags;
1233
1234 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1235 iommu->gcmd |= DMA_GCMD_TE;
1236 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1237
1238 /* Make sure hardware complete it */
1239 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1240 readl, (sts & DMA_GSTS_TES), sts);
1241
1242 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1243}
1244
1245static void iommu_disable_translation(struct intel_iommu *iommu)
1246{
1247 u32 sts;
1248 unsigned long flag;
1249
1250 if (iommu_skip_te_disable && iommu->drhd->gfx_dedicated &&
1251 (cap_read_drain(iommu->cap) || cap_write_drain(iommu->cap)))
1252 return;
1253
1254 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1255 iommu->gcmd &= ~DMA_GCMD_TE;
1256 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1257
1258 /* Make sure hardware complete it */
1259 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1260 readl, (!(sts & DMA_GSTS_TES)), sts);
1261
1262 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1263}
1264
1265static int iommu_init_domains(struct intel_iommu *iommu)
1266{
1267 u32 ndomains;
1268
1269 ndomains = cap_ndoms(iommu->cap);
1270 pr_debug("%s: Number of Domains supported <%d>\n",
1271 iommu->name, ndomains);
1272
1273 spin_lock_init(&iommu->lock);
1274
1275 iommu->domain_ids = bitmap_zalloc(ndomains, GFP_KERNEL);
1276 if (!iommu->domain_ids)
1277 return -ENOMEM;
1278
1279 /*
1280 * If Caching mode is set, then invalid translations are tagged
1281 * with domain-id 0, hence we need to pre-allocate it. We also
1282 * use domain-id 0 as a marker for non-allocated domain-id, so
1283 * make sure it is not used for a real domain.
1284 */
1285 set_bit(0, iommu->domain_ids);
1286
1287 /*
1288 * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1289 * entry for first-level or pass-through translation modes should
1290 * be programmed with a domain id different from those used for
1291 * second-level or nested translation. We reserve a domain id for
1292 * this purpose. This domain id is also used for identity domain
1293 * in legacy mode.
1294 */
1295 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1296
1297 return 0;
1298}
1299
1300static void disable_dmar_iommu(struct intel_iommu *iommu)
1301{
1302 if (!iommu->domain_ids)
1303 return;
1304
1305 /*
1306 * All iommu domains must have been detached from the devices,
1307 * hence there should be no domain IDs in use.
1308 */
1309 if (WARN_ON(bitmap_weight(iommu->domain_ids, cap_ndoms(iommu->cap))
1310 > NUM_RESERVED_DID))
1311 return;
1312
1313 if (iommu->gcmd & DMA_GCMD_TE)
1314 iommu_disable_translation(iommu);
1315}
1316
1317static void free_dmar_iommu(struct intel_iommu *iommu)
1318{
1319 if (iommu->domain_ids) {
1320 bitmap_free(iommu->domain_ids);
1321 iommu->domain_ids = NULL;
1322 }
1323
1324 if (iommu->copied_tables) {
1325 bitmap_free(iommu->copied_tables);
1326 iommu->copied_tables = NULL;
1327 }
1328
1329 /* free context mapping */
1330 free_context_table(iommu);
1331
1332 if (ecap_prs(iommu->ecap))
1333 intel_iommu_finish_prq(iommu);
1334}
1335
1336/*
1337 * Check and return whether first level is used by default for
1338 * DMA translation.
1339 */
1340static bool first_level_by_default(struct intel_iommu *iommu)
1341{
1342 /* Only SL is available in legacy mode */
1343 if (!sm_supported(iommu))
1344 return false;
1345
1346 /* Only level (either FL or SL) is available, just use it */
1347 if (ecap_flts(iommu->ecap) ^ ecap_slts(iommu->ecap))
1348 return ecap_flts(iommu->ecap);
1349
1350 return true;
1351}
1352
1353int domain_attach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
1354{
1355 struct iommu_domain_info *info, *curr;
1356 unsigned long ndomains;
1357 int num, ret = -ENOSPC;
1358
1359 if (domain->domain.type == IOMMU_DOMAIN_SVA)
1360 return 0;
1361
1362 info = kzalloc(sizeof(*info), GFP_KERNEL);
1363 if (!info)
1364 return -ENOMEM;
1365
1366 spin_lock(&iommu->lock);
1367 curr = xa_load(&domain->iommu_array, iommu->seq_id);
1368 if (curr) {
1369 curr->refcnt++;
1370 spin_unlock(&iommu->lock);
1371 kfree(info);
1372 return 0;
1373 }
1374
1375 ndomains = cap_ndoms(iommu->cap);
1376 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1377 if (num >= ndomains) {
1378 pr_err("%s: No free domain ids\n", iommu->name);
1379 goto err_unlock;
1380 }
1381
1382 set_bit(num, iommu->domain_ids);
1383 info->refcnt = 1;
1384 info->did = num;
1385 info->iommu = iommu;
1386 curr = xa_cmpxchg(&domain->iommu_array, iommu->seq_id,
1387 NULL, info, GFP_ATOMIC);
1388 if (curr) {
1389 ret = xa_err(curr) ? : -EBUSY;
1390 goto err_clear;
1391 }
1392
1393 spin_unlock(&iommu->lock);
1394 return 0;
1395
1396err_clear:
1397 clear_bit(info->did, iommu->domain_ids);
1398err_unlock:
1399 spin_unlock(&iommu->lock);
1400 kfree(info);
1401 return ret;
1402}
1403
1404void domain_detach_iommu(struct dmar_domain *domain, struct intel_iommu *iommu)
1405{
1406 struct iommu_domain_info *info;
1407
1408 if (domain->domain.type == IOMMU_DOMAIN_SVA)
1409 return;
1410
1411 spin_lock(&iommu->lock);
1412 info = xa_load(&domain->iommu_array, iommu->seq_id);
1413 if (--info->refcnt == 0) {
1414 clear_bit(info->did, iommu->domain_ids);
1415 xa_erase(&domain->iommu_array, iommu->seq_id);
1416 domain->nid = NUMA_NO_NODE;
1417 kfree(info);
1418 }
1419 spin_unlock(&iommu->lock);
1420}
1421
1422static void domain_exit(struct dmar_domain *domain)
1423{
1424 if (domain->pgd) {
1425 LIST_HEAD(freelist);
1426
1427 domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist);
1428 iommu_put_pages_list(&freelist);
1429 }
1430
1431 if (WARN_ON(!list_empty(&domain->devices)))
1432 return;
1433
1434 kfree(domain->qi_batch);
1435 kfree(domain);
1436}
1437
1438/*
1439 * For kdump cases, old valid entries may be cached due to the
1440 * in-flight DMA and copied pgtable, but there is no unmapping
1441 * behaviour for them, thus we need an explicit cache flush for
1442 * the newly-mapped device. For kdump, at this point, the device
1443 * is supposed to finish reset at its driver probe stage, so no
1444 * in-flight DMA will exist, and we don't need to worry anymore
1445 * hereafter.
1446 */
1447static void copied_context_tear_down(struct intel_iommu *iommu,
1448 struct context_entry *context,
1449 u8 bus, u8 devfn)
1450{
1451 u16 did_old;
1452
1453 if (!context_copied(iommu, bus, devfn))
1454 return;
1455
1456 assert_spin_locked(&iommu->lock);
1457
1458 did_old = context_domain_id(context);
1459 context_clear_entry(context);
1460
1461 if (did_old < cap_ndoms(iommu->cap)) {
1462 iommu->flush.flush_context(iommu, did_old,
1463 PCI_DEVID(bus, devfn),
1464 DMA_CCMD_MASK_NOBIT,
1465 DMA_CCMD_DEVICE_INVL);
1466 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
1467 DMA_TLB_DSI_FLUSH);
1468 }
1469
1470 clear_context_copied(iommu, bus, devfn);
1471}
1472
1473/*
1474 * It's a non-present to present mapping. If hardware doesn't cache
1475 * non-present entry we only need to flush the write-buffer. If the
1476 * _does_ cache non-present entries, then it does so in the special
1477 * domain #0, which we have to flush:
1478 */
1479static void context_present_cache_flush(struct intel_iommu *iommu, u16 did,
1480 u8 bus, u8 devfn)
1481{
1482 if (cap_caching_mode(iommu->cap)) {
1483 iommu->flush.flush_context(iommu, 0,
1484 PCI_DEVID(bus, devfn),
1485 DMA_CCMD_MASK_NOBIT,
1486 DMA_CCMD_DEVICE_INVL);
1487 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
1488 } else {
1489 iommu_flush_write_buffer(iommu);
1490 }
1491}
1492
1493static int domain_context_mapping_one(struct dmar_domain *domain,
1494 struct intel_iommu *iommu,
1495 u8 bus, u8 devfn)
1496{
1497 struct device_domain_info *info =
1498 domain_lookup_dev_info(domain, iommu, bus, devfn);
1499 u16 did = domain_id_iommu(domain, iommu);
1500 int translation = CONTEXT_TT_MULTI_LEVEL;
1501 struct dma_pte *pgd = domain->pgd;
1502 struct context_entry *context;
1503 int ret;
1504
1505 pr_debug("Set context mapping for %02x:%02x.%d\n",
1506 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1507
1508 spin_lock(&iommu->lock);
1509 ret = -ENOMEM;
1510 context = iommu_context_addr(iommu, bus, devfn, 1);
1511 if (!context)
1512 goto out_unlock;
1513
1514 ret = 0;
1515 if (context_present(context) && !context_copied(iommu, bus, devfn))
1516 goto out_unlock;
1517
1518 copied_context_tear_down(iommu, context, bus, devfn);
1519 context_clear_entry(context);
1520 context_set_domain_id(context, did);
1521
1522 if (info && info->ats_supported)
1523 translation = CONTEXT_TT_DEV_IOTLB;
1524 else
1525 translation = CONTEXT_TT_MULTI_LEVEL;
1526
1527 context_set_address_root(context, virt_to_phys(pgd));
1528 context_set_address_width(context, domain->agaw);
1529 context_set_translation_type(context, translation);
1530 context_set_fault_enable(context);
1531 context_set_present(context);
1532 if (!ecap_coherent(iommu->ecap))
1533 clflush_cache_range(context, sizeof(*context));
1534 context_present_cache_flush(iommu, did, bus, devfn);
1535 ret = 0;
1536
1537out_unlock:
1538 spin_unlock(&iommu->lock);
1539
1540 return ret;
1541}
1542
1543static int domain_context_mapping_cb(struct pci_dev *pdev,
1544 u16 alias, void *opaque)
1545{
1546 struct device_domain_info *info = dev_iommu_priv_get(&pdev->dev);
1547 struct intel_iommu *iommu = info->iommu;
1548 struct dmar_domain *domain = opaque;
1549
1550 return domain_context_mapping_one(domain, iommu,
1551 PCI_BUS_NUM(alias), alias & 0xff);
1552}
1553
1554static int
1555domain_context_mapping(struct dmar_domain *domain, struct device *dev)
1556{
1557 struct device_domain_info *info = dev_iommu_priv_get(dev);
1558 struct intel_iommu *iommu = info->iommu;
1559 u8 bus = info->bus, devfn = info->devfn;
1560
1561 if (!dev_is_pci(dev))
1562 return domain_context_mapping_one(domain, iommu, bus, devfn);
1563
1564 return pci_for_each_dma_alias(to_pci_dev(dev),
1565 domain_context_mapping_cb, domain);
1566}
1567
1568/* Return largest possible superpage level for a given mapping */
1569static int hardware_largepage_caps(struct dmar_domain *domain, unsigned long iov_pfn,
1570 unsigned long phy_pfn, unsigned long pages)
1571{
1572 int support, level = 1;
1573 unsigned long pfnmerge;
1574
1575 support = domain->iommu_superpage;
1576
1577 /* To use a large page, the virtual *and* physical addresses
1578 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
1579 of them will mean we have to use smaller pages. So just
1580 merge them and check both at once. */
1581 pfnmerge = iov_pfn | phy_pfn;
1582
1583 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
1584 pages >>= VTD_STRIDE_SHIFT;
1585 if (!pages)
1586 break;
1587 pfnmerge >>= VTD_STRIDE_SHIFT;
1588 level++;
1589 support--;
1590 }
1591 return level;
1592}
1593
1594/*
1595 * Ensure that old small page tables are removed to make room for superpage(s).
1596 * We're going to add new large pages, so make sure we don't remove their parent
1597 * tables. The IOTLB/devTLBs should be flushed if any PDE/PTEs are cleared.
1598 */
1599static void switch_to_super_page(struct dmar_domain *domain,
1600 unsigned long start_pfn,
1601 unsigned long end_pfn, int level)
1602{
1603 unsigned long lvl_pages = lvl_to_nr_pages(level);
1604 struct dma_pte *pte = NULL;
1605
1606 while (start_pfn <= end_pfn) {
1607 if (!pte)
1608 pte = pfn_to_dma_pte(domain, start_pfn, &level,
1609 GFP_ATOMIC);
1610
1611 if (dma_pte_present(pte)) {
1612 dma_pte_free_pagetable(domain, start_pfn,
1613 start_pfn + lvl_pages - 1,
1614 level + 1);
1615
1616 cache_tag_flush_range(domain, start_pfn << VTD_PAGE_SHIFT,
1617 end_pfn << VTD_PAGE_SHIFT, 0);
1618 }
1619
1620 pte++;
1621 start_pfn += lvl_pages;
1622 if (first_pte_in_page(pte))
1623 pte = NULL;
1624 }
1625}
1626
1627static int
1628__domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
1629 unsigned long phys_pfn, unsigned long nr_pages, int prot,
1630 gfp_t gfp)
1631{
1632 struct dma_pte *first_pte = NULL, *pte = NULL;
1633 unsigned int largepage_lvl = 0;
1634 unsigned long lvl_pages = 0;
1635 phys_addr_t pteval;
1636 u64 attr;
1637
1638 if (unlikely(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1)))
1639 return -EINVAL;
1640
1641 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
1642 return -EINVAL;
1643
1644 if (!(prot & DMA_PTE_WRITE) && domain->nested_parent) {
1645 pr_err_ratelimited("Read-only mapping is disallowed on the domain which serves as the parent in a nested configuration, due to HW errata (ERRATA_772415_SPR17)\n");
1646 return -EINVAL;
1647 }
1648
1649 attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP);
1650 attr |= DMA_FL_PTE_PRESENT;
1651 if (domain->use_first_level) {
1652 attr |= DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
1653 if (prot & DMA_PTE_WRITE)
1654 attr |= DMA_FL_PTE_DIRTY;
1655 }
1656
1657 domain->has_mappings = true;
1658
1659 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | attr;
1660
1661 while (nr_pages > 0) {
1662 uint64_t tmp;
1663
1664 if (!pte) {
1665 largepage_lvl = hardware_largepage_caps(domain, iov_pfn,
1666 phys_pfn, nr_pages);
1667
1668 pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl,
1669 gfp);
1670 if (!pte)
1671 return -ENOMEM;
1672 first_pte = pte;
1673
1674 lvl_pages = lvl_to_nr_pages(largepage_lvl);
1675
1676 /* It is large page*/
1677 if (largepage_lvl > 1) {
1678 unsigned long end_pfn;
1679 unsigned long pages_to_remove;
1680
1681 pteval |= DMA_PTE_LARGE_PAGE;
1682 pages_to_remove = min_t(unsigned long, nr_pages,
1683 nr_pte_to_next_page(pte) * lvl_pages);
1684 end_pfn = iov_pfn + pages_to_remove - 1;
1685 switch_to_super_page(domain, iov_pfn, end_pfn, largepage_lvl);
1686 } else {
1687 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
1688 }
1689
1690 }
1691 /* We don't need lock here, nobody else
1692 * touches the iova range
1693 */
1694 tmp = 0ULL;
1695 if (!try_cmpxchg64_local(&pte->val, &tmp, pteval)) {
1696 static int dumps = 5;
1697 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
1698 iov_pfn, tmp, (unsigned long long)pteval);
1699 if (dumps) {
1700 dumps--;
1701 debug_dma_dump_mappings(NULL);
1702 }
1703 WARN_ON(1);
1704 }
1705
1706 nr_pages -= lvl_pages;
1707 iov_pfn += lvl_pages;
1708 phys_pfn += lvl_pages;
1709 pteval += lvl_pages * VTD_PAGE_SIZE;
1710
1711 /* If the next PTE would be the first in a new page, then we
1712 * need to flush the cache on the entries we've just written.
1713 * And then we'll need to recalculate 'pte', so clear it and
1714 * let it get set again in the if (!pte) block above.
1715 *
1716 * If we're done (!nr_pages) we need to flush the cache too.
1717 *
1718 * Also if we've been setting superpages, we may need to
1719 * recalculate 'pte' and switch back to smaller pages for the
1720 * end of the mapping, if the trailing size is not enough to
1721 * use another superpage (i.e. nr_pages < lvl_pages).
1722 */
1723 pte++;
1724 if (!nr_pages || first_pte_in_page(pte) ||
1725 (largepage_lvl > 1 && nr_pages < lvl_pages)) {
1726 domain_flush_cache(domain, first_pte,
1727 (void *)pte - (void *)first_pte);
1728 pte = NULL;
1729 }
1730 }
1731
1732 return 0;
1733}
1734
1735static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 devfn)
1736{
1737 struct intel_iommu *iommu = info->iommu;
1738 struct context_entry *context;
1739 u16 did;
1740
1741 spin_lock(&iommu->lock);
1742 context = iommu_context_addr(iommu, bus, devfn, 0);
1743 if (!context) {
1744 spin_unlock(&iommu->lock);
1745 return;
1746 }
1747
1748 did = context_domain_id(context);
1749 context_clear_entry(context);
1750 __iommu_flush_cache(iommu, context, sizeof(*context));
1751 spin_unlock(&iommu->lock);
1752 intel_context_flush_present(info, context, did, true);
1753}
1754
1755int __domain_setup_first_level(struct intel_iommu *iommu,
1756 struct device *dev, ioasid_t pasid,
1757 u16 did, pgd_t *pgd, int flags,
1758 struct iommu_domain *old)
1759{
1760 if (!old)
1761 return intel_pasid_setup_first_level(iommu, dev, pgd,
1762 pasid, did, flags);
1763 return intel_pasid_replace_first_level(iommu, dev, pgd, pasid, did,
1764 iommu_domain_did(old, iommu),
1765 flags);
1766}
1767
1768static int domain_setup_second_level(struct intel_iommu *iommu,
1769 struct dmar_domain *domain,
1770 struct device *dev, ioasid_t pasid,
1771 struct iommu_domain *old)
1772{
1773 if (!old)
1774 return intel_pasid_setup_second_level(iommu, domain,
1775 dev, pasid);
1776 return intel_pasid_replace_second_level(iommu, domain, dev,
1777 iommu_domain_did(old, iommu),
1778 pasid);
1779}
1780
1781static int domain_setup_passthrough(struct intel_iommu *iommu,
1782 struct device *dev, ioasid_t pasid,
1783 struct iommu_domain *old)
1784{
1785 if (!old)
1786 return intel_pasid_setup_pass_through(iommu, dev, pasid);
1787 return intel_pasid_replace_pass_through(iommu, dev,
1788 iommu_domain_did(old, iommu),
1789 pasid);
1790}
1791
1792static int domain_setup_first_level(struct intel_iommu *iommu,
1793 struct dmar_domain *domain,
1794 struct device *dev,
1795 u32 pasid, struct iommu_domain *old)
1796{
1797 struct dma_pte *pgd = domain->pgd;
1798 int level, flags = 0;
1799
1800 level = agaw_to_level(domain->agaw);
1801 if (level != 4 && level != 5)
1802 return -EINVAL;
1803
1804 if (level == 5)
1805 flags |= PASID_FLAG_FL5LP;
1806
1807 if (domain->force_snooping)
1808 flags |= PASID_FLAG_PAGE_SNOOP;
1809
1810 return __domain_setup_first_level(iommu, dev, pasid,
1811 domain_id_iommu(domain, iommu),
1812 (pgd_t *)pgd, flags, old);
1813}
1814
1815static int dmar_domain_attach_device(struct dmar_domain *domain,
1816 struct device *dev)
1817{
1818 struct device_domain_info *info = dev_iommu_priv_get(dev);
1819 struct intel_iommu *iommu = info->iommu;
1820 unsigned long flags;
1821 int ret;
1822
1823 ret = domain_attach_iommu(domain, iommu);
1824 if (ret)
1825 return ret;
1826
1827 info->domain = domain;
1828 spin_lock_irqsave(&domain->lock, flags);
1829 list_add(&info->link, &domain->devices);
1830 spin_unlock_irqrestore(&domain->lock, flags);
1831
1832 if (dev_is_real_dma_subdevice(dev))
1833 return 0;
1834
1835 if (!sm_supported(iommu))
1836 ret = domain_context_mapping(domain, dev);
1837 else if (domain->use_first_level)
1838 ret = domain_setup_first_level(iommu, domain, dev,
1839 IOMMU_NO_PASID, NULL);
1840 else
1841 ret = domain_setup_second_level(iommu, domain, dev,
1842 IOMMU_NO_PASID, NULL);
1843
1844 if (ret)
1845 goto out_block_translation;
1846
1847 iommu_enable_pci_caps(info);
1848
1849 ret = cache_tag_assign_domain(domain, dev, IOMMU_NO_PASID);
1850 if (ret)
1851 goto out_block_translation;
1852
1853 return 0;
1854
1855out_block_translation:
1856 device_block_translation(dev);
1857 return ret;
1858}
1859
1860/**
1861 * device_rmrr_is_relaxable - Test whether the RMRR of this device
1862 * is relaxable (ie. is allowed to be not enforced under some conditions)
1863 * @dev: device handle
1864 *
1865 * We assume that PCI USB devices with RMRRs have them largely
1866 * for historical reasons and that the RMRR space is not actively used post
1867 * boot. This exclusion may change if vendors begin to abuse it.
1868 *
1869 * The same exception is made for graphics devices, with the requirement that
1870 * any use of the RMRR regions will be torn down before assigning the device
1871 * to a guest.
1872 *
1873 * Return: true if the RMRR is relaxable, false otherwise
1874 */
1875static bool device_rmrr_is_relaxable(struct device *dev)
1876{
1877 struct pci_dev *pdev;
1878
1879 if (!dev_is_pci(dev))
1880 return false;
1881
1882 pdev = to_pci_dev(dev);
1883 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
1884 return true;
1885 else
1886 return false;
1887}
1888
1889static int device_def_domain_type(struct device *dev)
1890{
1891 struct device_domain_info *info = dev_iommu_priv_get(dev);
1892 struct intel_iommu *iommu = info->iommu;
1893
1894 /*
1895 * Hardware does not support the passthrough translation mode.
1896 * Always use a dynamaic mapping domain.
1897 */
1898 if (!ecap_pass_through(iommu->ecap))
1899 return IOMMU_DOMAIN_DMA;
1900
1901 if (dev_is_pci(dev)) {
1902 struct pci_dev *pdev = to_pci_dev(dev);
1903
1904 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
1905 return IOMMU_DOMAIN_IDENTITY;
1906 }
1907
1908 return 0;
1909}
1910
1911static void intel_iommu_init_qi(struct intel_iommu *iommu)
1912{
1913 /*
1914 * Start from the sane iommu hardware state.
1915 * If the queued invalidation is already initialized by us
1916 * (for example, while enabling interrupt-remapping) then
1917 * we got the things already rolling from a sane state.
1918 */
1919 if (!iommu->qi) {
1920 /*
1921 * Clear any previous faults.
1922 */
1923 dmar_fault(-1, iommu);
1924 /*
1925 * Disable queued invalidation if supported and already enabled
1926 * before OS handover.
1927 */
1928 dmar_disable_qi(iommu);
1929 }
1930
1931 if (dmar_enable_qi(iommu)) {
1932 /*
1933 * Queued Invalidate not enabled, use Register Based Invalidate
1934 */
1935 iommu->flush.flush_context = __iommu_flush_context;
1936 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
1937 pr_info("%s: Using Register based invalidation\n",
1938 iommu->name);
1939 } else {
1940 iommu->flush.flush_context = qi_flush_context;
1941 iommu->flush.flush_iotlb = qi_flush_iotlb;
1942 pr_info("%s: Using Queued invalidation\n", iommu->name);
1943 }
1944}
1945
1946static int copy_context_table(struct intel_iommu *iommu,
1947 struct root_entry *old_re,
1948 struct context_entry **tbl,
1949 int bus, bool ext)
1950{
1951 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
1952 struct context_entry *new_ce = NULL, ce;
1953 struct context_entry *old_ce = NULL;
1954 struct root_entry re;
1955 phys_addr_t old_ce_phys;
1956
1957 tbl_idx = ext ? bus * 2 : bus;
1958 memcpy(&re, old_re, sizeof(re));
1959
1960 for (devfn = 0; devfn < 256; devfn++) {
1961 /* First calculate the correct index */
1962 idx = (ext ? devfn * 2 : devfn) % 256;
1963
1964 if (idx == 0) {
1965 /* First save what we may have and clean up */
1966 if (new_ce) {
1967 tbl[tbl_idx] = new_ce;
1968 __iommu_flush_cache(iommu, new_ce,
1969 VTD_PAGE_SIZE);
1970 pos = 1;
1971 }
1972
1973 if (old_ce)
1974 memunmap(old_ce);
1975
1976 ret = 0;
1977 if (devfn < 0x80)
1978 old_ce_phys = root_entry_lctp(&re);
1979 else
1980 old_ce_phys = root_entry_uctp(&re);
1981
1982 if (!old_ce_phys) {
1983 if (ext && devfn == 0) {
1984 /* No LCTP, try UCTP */
1985 devfn = 0x7f;
1986 continue;
1987 } else {
1988 goto out;
1989 }
1990 }
1991
1992 ret = -ENOMEM;
1993 old_ce = memremap(old_ce_phys, PAGE_SIZE,
1994 MEMREMAP_WB);
1995 if (!old_ce)
1996 goto out;
1997
1998 new_ce = iommu_alloc_page_node(iommu->node, GFP_KERNEL);
1999 if (!new_ce)
2000 goto out_unmap;
2001
2002 ret = 0;
2003 }
2004
2005 /* Now copy the context entry */
2006 memcpy(&ce, old_ce + idx, sizeof(ce));
2007
2008 if (!context_present(&ce))
2009 continue;
2010
2011 did = context_domain_id(&ce);
2012 if (did >= 0 && did < cap_ndoms(iommu->cap))
2013 set_bit(did, iommu->domain_ids);
2014
2015 set_context_copied(iommu, bus, devfn);
2016 new_ce[idx] = ce;
2017 }
2018
2019 tbl[tbl_idx + pos] = new_ce;
2020
2021 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2022
2023out_unmap:
2024 memunmap(old_ce);
2025
2026out:
2027 return ret;
2028}
2029
2030static int copy_translation_tables(struct intel_iommu *iommu)
2031{
2032 struct context_entry **ctxt_tbls;
2033 struct root_entry *old_rt;
2034 phys_addr_t old_rt_phys;
2035 int ctxt_table_entries;
2036 u64 rtaddr_reg;
2037 int bus, ret;
2038 bool new_ext, ext;
2039
2040 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2041 ext = !!(rtaddr_reg & DMA_RTADDR_SMT);
2042 new_ext = !!sm_supported(iommu);
2043
2044 /*
2045 * The RTT bit can only be changed when translation is disabled,
2046 * but disabling translation means to open a window for data
2047 * corruption. So bail out and don't copy anything if we would
2048 * have to change the bit.
2049 */
2050 if (new_ext != ext)
2051 return -EINVAL;
2052
2053 iommu->copied_tables = bitmap_zalloc(BIT_ULL(16), GFP_KERNEL);
2054 if (!iommu->copied_tables)
2055 return -ENOMEM;
2056
2057 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2058 if (!old_rt_phys)
2059 return -EINVAL;
2060
2061 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
2062 if (!old_rt)
2063 return -ENOMEM;
2064
2065 /* This is too big for the stack - allocate it from slab */
2066 ctxt_table_entries = ext ? 512 : 256;
2067 ret = -ENOMEM;
2068 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
2069 if (!ctxt_tbls)
2070 goto out_unmap;
2071
2072 for (bus = 0; bus < 256; bus++) {
2073 ret = copy_context_table(iommu, &old_rt[bus],
2074 ctxt_tbls, bus, ext);
2075 if (ret) {
2076 pr_err("%s: Failed to copy context table for bus %d\n",
2077 iommu->name, bus);
2078 continue;
2079 }
2080 }
2081
2082 spin_lock(&iommu->lock);
2083
2084 /* Context tables are copied, now write them to the root_entry table */
2085 for (bus = 0; bus < 256; bus++) {
2086 int idx = ext ? bus * 2 : bus;
2087 u64 val;
2088
2089 if (ctxt_tbls[idx]) {
2090 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2091 iommu->root_entry[bus].lo = val;
2092 }
2093
2094 if (!ext || !ctxt_tbls[idx + 1])
2095 continue;
2096
2097 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2098 iommu->root_entry[bus].hi = val;
2099 }
2100
2101 spin_unlock(&iommu->lock);
2102
2103 kfree(ctxt_tbls);
2104
2105 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
2106
2107 ret = 0;
2108
2109out_unmap:
2110 memunmap(old_rt);
2111
2112 return ret;
2113}
2114
2115static int __init init_dmars(void)
2116{
2117 struct dmar_drhd_unit *drhd;
2118 struct intel_iommu *iommu;
2119 int ret;
2120
2121 ret = intel_cap_audit(CAP_AUDIT_STATIC_DMAR, NULL);
2122 if (ret)
2123 goto free_iommu;
2124
2125 for_each_iommu(iommu, drhd) {
2126 if (drhd->ignored) {
2127 iommu_disable_translation(iommu);
2128 continue;
2129 }
2130
2131 /*
2132 * Find the max pasid size of all IOMMU's in the system.
2133 * We need to ensure the system pasid table is no bigger
2134 * than the smallest supported.
2135 */
2136 if (pasid_supported(iommu)) {
2137 u32 temp = 2 << ecap_pss(iommu->ecap);
2138
2139 intel_pasid_max_id = min_t(u32, temp,
2140 intel_pasid_max_id);
2141 }
2142
2143 intel_iommu_init_qi(iommu);
2144
2145 ret = iommu_init_domains(iommu);
2146 if (ret)
2147 goto free_iommu;
2148
2149 init_translation_status(iommu);
2150
2151 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
2152 iommu_disable_translation(iommu);
2153 clear_translation_pre_enabled(iommu);
2154 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
2155 iommu->name);
2156 }
2157
2158 /*
2159 * TBD:
2160 * we could share the same root & context tables
2161 * among all IOMMU's. Need to Split it later.
2162 */
2163 ret = iommu_alloc_root_entry(iommu);
2164 if (ret)
2165 goto free_iommu;
2166
2167 if (translation_pre_enabled(iommu)) {
2168 pr_info("Translation already enabled - trying to copy translation structures\n");
2169
2170 ret = copy_translation_tables(iommu);
2171 if (ret) {
2172 /*
2173 * We found the IOMMU with translation
2174 * enabled - but failed to copy over the
2175 * old root-entry table. Try to proceed
2176 * by disabling translation now and
2177 * allocating a clean root-entry table.
2178 * This might cause DMAR faults, but
2179 * probably the dump will still succeed.
2180 */
2181 pr_err("Failed to copy translation tables from previous kernel for %s\n",
2182 iommu->name);
2183 iommu_disable_translation(iommu);
2184 clear_translation_pre_enabled(iommu);
2185 } else {
2186 pr_info("Copied translation tables from previous kernel for %s\n",
2187 iommu->name);
2188 }
2189 }
2190
2191 intel_svm_check(iommu);
2192 }
2193
2194 /*
2195 * Now that qi is enabled on all iommus, set the root entry and flush
2196 * caches. This is required on some Intel X58 chipsets, otherwise the
2197 * flush_context function will loop forever and the boot hangs.
2198 */
2199 for_each_active_iommu(iommu, drhd) {
2200 iommu_flush_write_buffer(iommu);
2201 iommu_set_root_entry(iommu);
2202 }
2203
2204 check_tylersburg_isoch();
2205
2206 /*
2207 * for each drhd
2208 * enable fault log
2209 * global invalidate context cache
2210 * global invalidate iotlb
2211 * enable translation
2212 */
2213 for_each_iommu(iommu, drhd) {
2214 if (drhd->ignored) {
2215 /*
2216 * we always have to disable PMRs or DMA may fail on
2217 * this device
2218 */
2219 if (force_on)
2220 iommu_disable_protect_mem_regions(iommu);
2221 continue;
2222 }
2223
2224 iommu_flush_write_buffer(iommu);
2225
2226 if (ecap_prs(iommu->ecap)) {
2227 /*
2228 * Call dmar_alloc_hwirq() with dmar_global_lock held,
2229 * could cause possible lock race condition.
2230 */
2231 up_write(&dmar_global_lock);
2232 ret = intel_iommu_enable_prq(iommu);
2233 down_write(&dmar_global_lock);
2234 if (ret)
2235 goto free_iommu;
2236 }
2237
2238 ret = dmar_set_interrupt(iommu);
2239 if (ret)
2240 goto free_iommu;
2241 }
2242
2243 return 0;
2244
2245free_iommu:
2246 for_each_active_iommu(iommu, drhd) {
2247 disable_dmar_iommu(iommu);
2248 free_dmar_iommu(iommu);
2249 }
2250
2251 return ret;
2252}
2253
2254static void __init init_no_remapping_devices(void)
2255{
2256 struct dmar_drhd_unit *drhd;
2257 struct device *dev;
2258 int i;
2259
2260 for_each_drhd_unit(drhd) {
2261 if (!drhd->include_all) {
2262 for_each_active_dev_scope(drhd->devices,
2263 drhd->devices_cnt, i, dev)
2264 break;
2265 /* ignore DMAR unit if no devices exist */
2266 if (i == drhd->devices_cnt)
2267 drhd->ignored = 1;
2268 }
2269 }
2270
2271 for_each_active_drhd_unit(drhd) {
2272 if (drhd->include_all)
2273 continue;
2274
2275 for_each_active_dev_scope(drhd->devices,
2276 drhd->devices_cnt, i, dev)
2277 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
2278 break;
2279 if (i < drhd->devices_cnt)
2280 continue;
2281
2282 /* This IOMMU has *only* gfx devices. Either bypass it or
2283 set the gfx_mapped flag, as appropriate */
2284 drhd->gfx_dedicated = 1;
2285 if (disable_igfx_iommu)
2286 drhd->ignored = 1;
2287 }
2288}
2289
2290#ifdef CONFIG_SUSPEND
2291static int init_iommu_hw(void)
2292{
2293 struct dmar_drhd_unit *drhd;
2294 struct intel_iommu *iommu = NULL;
2295 int ret;
2296
2297 for_each_active_iommu(iommu, drhd) {
2298 if (iommu->qi) {
2299 ret = dmar_reenable_qi(iommu);
2300 if (ret)
2301 return ret;
2302 }
2303 }
2304
2305 for_each_iommu(iommu, drhd) {
2306 if (drhd->ignored) {
2307 /*
2308 * we always have to disable PMRs or DMA may fail on
2309 * this device
2310 */
2311 if (force_on)
2312 iommu_disable_protect_mem_regions(iommu);
2313 continue;
2314 }
2315
2316 iommu_flush_write_buffer(iommu);
2317 iommu_set_root_entry(iommu);
2318 iommu_enable_translation(iommu);
2319 iommu_disable_protect_mem_regions(iommu);
2320 }
2321
2322 return 0;
2323}
2324
2325static void iommu_flush_all(void)
2326{
2327 struct dmar_drhd_unit *drhd;
2328 struct intel_iommu *iommu;
2329
2330 for_each_active_iommu(iommu, drhd) {
2331 iommu->flush.flush_context(iommu, 0, 0, 0,
2332 DMA_CCMD_GLOBAL_INVL);
2333 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
2334 DMA_TLB_GLOBAL_FLUSH);
2335 }
2336}
2337
2338static int iommu_suspend(void)
2339{
2340 struct dmar_drhd_unit *drhd;
2341 struct intel_iommu *iommu = NULL;
2342 unsigned long flag;
2343
2344 iommu_flush_all();
2345
2346 for_each_active_iommu(iommu, drhd) {
2347 iommu_disable_translation(iommu);
2348
2349 raw_spin_lock_irqsave(&iommu->register_lock, flag);
2350
2351 iommu->iommu_state[SR_DMAR_FECTL_REG] =
2352 readl(iommu->reg + DMAR_FECTL_REG);
2353 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
2354 readl(iommu->reg + DMAR_FEDATA_REG);
2355 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
2356 readl(iommu->reg + DMAR_FEADDR_REG);
2357 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
2358 readl(iommu->reg + DMAR_FEUADDR_REG);
2359
2360 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2361 }
2362 return 0;
2363}
2364
2365static void iommu_resume(void)
2366{
2367 struct dmar_drhd_unit *drhd;
2368 struct intel_iommu *iommu = NULL;
2369 unsigned long flag;
2370
2371 if (init_iommu_hw()) {
2372 if (force_on)
2373 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
2374 else
2375 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
2376 return;
2377 }
2378
2379 for_each_active_iommu(iommu, drhd) {
2380
2381 raw_spin_lock_irqsave(&iommu->register_lock, flag);
2382
2383 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
2384 iommu->reg + DMAR_FECTL_REG);
2385 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
2386 iommu->reg + DMAR_FEDATA_REG);
2387 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
2388 iommu->reg + DMAR_FEADDR_REG);
2389 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
2390 iommu->reg + DMAR_FEUADDR_REG);
2391
2392 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
2393 }
2394}
2395
2396static struct syscore_ops iommu_syscore_ops = {
2397 .resume = iommu_resume,
2398 .suspend = iommu_suspend,
2399};
2400
2401static void __init init_iommu_pm_ops(void)
2402{
2403 register_syscore_ops(&iommu_syscore_ops);
2404}
2405
2406#else
2407static inline void init_iommu_pm_ops(void) {}
2408#endif /* CONFIG_PM */
2409
2410static int __init rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
2411{
2412 if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) ||
2413 !IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) ||
2414 rmrr->end_address <= rmrr->base_address ||
2415 arch_rmrr_sanity_check(rmrr))
2416 return -EINVAL;
2417
2418 return 0;
2419}
2420
2421int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
2422{
2423 struct acpi_dmar_reserved_memory *rmrr;
2424 struct dmar_rmrr_unit *rmrru;
2425
2426 rmrr = (struct acpi_dmar_reserved_memory *)header;
2427 if (rmrr_sanity_check(rmrr)) {
2428 pr_warn(FW_BUG
2429 "Your BIOS is broken; bad RMRR [%#018Lx-%#018Lx]\n"
2430 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
2431 rmrr->base_address, rmrr->end_address,
2432 dmi_get_system_info(DMI_BIOS_VENDOR),
2433 dmi_get_system_info(DMI_BIOS_VERSION),
2434 dmi_get_system_info(DMI_PRODUCT_VERSION));
2435 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
2436 }
2437
2438 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
2439 if (!rmrru)
2440 goto out;
2441
2442 rmrru->hdr = header;
2443
2444 rmrru->base_address = rmrr->base_address;
2445 rmrru->end_address = rmrr->end_address;
2446
2447 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
2448 ((void *)rmrr) + rmrr->header.length,
2449 &rmrru->devices_cnt);
2450 if (rmrru->devices_cnt && rmrru->devices == NULL)
2451 goto free_rmrru;
2452
2453 list_add(&rmrru->list, &dmar_rmrr_units);
2454
2455 return 0;
2456free_rmrru:
2457 kfree(rmrru);
2458out:
2459 return -ENOMEM;
2460}
2461
2462static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
2463{
2464 struct dmar_atsr_unit *atsru;
2465 struct acpi_dmar_atsr *tmp;
2466
2467 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list,
2468 dmar_rcu_check()) {
2469 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
2470 if (atsr->segment != tmp->segment)
2471 continue;
2472 if (atsr->header.length != tmp->header.length)
2473 continue;
2474 if (memcmp(atsr, tmp, atsr->header.length) == 0)
2475 return atsru;
2476 }
2477
2478 return NULL;
2479}
2480
2481int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
2482{
2483 struct acpi_dmar_atsr *atsr;
2484 struct dmar_atsr_unit *atsru;
2485
2486 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
2487 return 0;
2488
2489 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
2490 atsru = dmar_find_atsr(atsr);
2491 if (atsru)
2492 return 0;
2493
2494 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
2495 if (!atsru)
2496 return -ENOMEM;
2497
2498 /*
2499 * If memory is allocated from slab by ACPI _DSM method, we need to
2500 * copy the memory content because the memory buffer will be freed
2501 * on return.
2502 */
2503 atsru->hdr = (void *)(atsru + 1);
2504 memcpy(atsru->hdr, hdr, hdr->length);
2505 atsru->include_all = atsr->flags & 0x1;
2506 if (!atsru->include_all) {
2507 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
2508 (void *)atsr + atsr->header.length,
2509 &atsru->devices_cnt);
2510 if (atsru->devices_cnt && atsru->devices == NULL) {
2511 kfree(atsru);
2512 return -ENOMEM;
2513 }
2514 }
2515
2516 list_add_rcu(&atsru->list, &dmar_atsr_units);
2517
2518 return 0;
2519}
2520
2521static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
2522{
2523 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
2524 kfree(atsru);
2525}
2526
2527int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
2528{
2529 struct acpi_dmar_atsr *atsr;
2530 struct dmar_atsr_unit *atsru;
2531
2532 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
2533 atsru = dmar_find_atsr(atsr);
2534 if (atsru) {
2535 list_del_rcu(&atsru->list);
2536 synchronize_rcu();
2537 intel_iommu_free_atsr(atsru);
2538 }
2539
2540 return 0;
2541}
2542
2543int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
2544{
2545 int i;
2546 struct device *dev;
2547 struct acpi_dmar_atsr *atsr;
2548 struct dmar_atsr_unit *atsru;
2549
2550 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
2551 atsru = dmar_find_atsr(atsr);
2552 if (!atsru)
2553 return 0;
2554
2555 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
2556 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
2557 i, dev)
2558 return -EBUSY;
2559 }
2560
2561 return 0;
2562}
2563
2564static struct dmar_satc_unit *dmar_find_satc(struct acpi_dmar_satc *satc)
2565{
2566 struct dmar_satc_unit *satcu;
2567 struct acpi_dmar_satc *tmp;
2568
2569 list_for_each_entry_rcu(satcu, &dmar_satc_units, list,
2570 dmar_rcu_check()) {
2571 tmp = (struct acpi_dmar_satc *)satcu->hdr;
2572 if (satc->segment != tmp->segment)
2573 continue;
2574 if (satc->header.length != tmp->header.length)
2575 continue;
2576 if (memcmp(satc, tmp, satc->header.length) == 0)
2577 return satcu;
2578 }
2579
2580 return NULL;
2581}
2582
2583int dmar_parse_one_satc(struct acpi_dmar_header *hdr, void *arg)
2584{
2585 struct acpi_dmar_satc *satc;
2586 struct dmar_satc_unit *satcu;
2587
2588 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
2589 return 0;
2590
2591 satc = container_of(hdr, struct acpi_dmar_satc, header);
2592 satcu = dmar_find_satc(satc);
2593 if (satcu)
2594 return 0;
2595
2596 satcu = kzalloc(sizeof(*satcu) + hdr->length, GFP_KERNEL);
2597 if (!satcu)
2598 return -ENOMEM;
2599
2600 satcu->hdr = (void *)(satcu + 1);
2601 memcpy(satcu->hdr, hdr, hdr->length);
2602 satcu->atc_required = satc->flags & 0x1;
2603 satcu->devices = dmar_alloc_dev_scope((void *)(satc + 1),
2604 (void *)satc + satc->header.length,
2605 &satcu->devices_cnt);
2606 if (satcu->devices_cnt && !satcu->devices) {
2607 kfree(satcu);
2608 return -ENOMEM;
2609 }
2610 list_add_rcu(&satcu->list, &dmar_satc_units);
2611
2612 return 0;
2613}
2614
2615static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
2616{
2617 struct intel_iommu *iommu = dmaru->iommu;
2618 int ret;
2619
2620 ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_DMAR, iommu);
2621 if (ret)
2622 goto out;
2623
2624 /*
2625 * Disable translation if already enabled prior to OS handover.
2626 */
2627 if (iommu->gcmd & DMA_GCMD_TE)
2628 iommu_disable_translation(iommu);
2629
2630 ret = iommu_init_domains(iommu);
2631 if (ret == 0)
2632 ret = iommu_alloc_root_entry(iommu);
2633 if (ret)
2634 goto out;
2635
2636 intel_svm_check(iommu);
2637
2638 if (dmaru->ignored) {
2639 /*
2640 * we always have to disable PMRs or DMA may fail on this device
2641 */
2642 if (force_on)
2643 iommu_disable_protect_mem_regions(iommu);
2644 return 0;
2645 }
2646
2647 intel_iommu_init_qi(iommu);
2648 iommu_flush_write_buffer(iommu);
2649
2650 if (ecap_prs(iommu->ecap)) {
2651 ret = intel_iommu_enable_prq(iommu);
2652 if (ret)
2653 goto disable_iommu;
2654 }
2655
2656 ret = dmar_set_interrupt(iommu);
2657 if (ret)
2658 goto disable_iommu;
2659
2660 iommu_set_root_entry(iommu);
2661 iommu_enable_translation(iommu);
2662
2663 iommu_disable_protect_mem_regions(iommu);
2664 return 0;
2665
2666disable_iommu:
2667 disable_dmar_iommu(iommu);
2668out:
2669 free_dmar_iommu(iommu);
2670 return ret;
2671}
2672
2673int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
2674{
2675 int ret = 0;
2676 struct intel_iommu *iommu = dmaru->iommu;
2677
2678 if (!intel_iommu_enabled)
2679 return 0;
2680 if (iommu == NULL)
2681 return -EINVAL;
2682
2683 if (insert) {
2684 ret = intel_iommu_add(dmaru);
2685 } else {
2686 disable_dmar_iommu(iommu);
2687 free_dmar_iommu(iommu);
2688 }
2689
2690 return ret;
2691}
2692
2693static void intel_iommu_free_dmars(void)
2694{
2695 struct dmar_rmrr_unit *rmrru, *rmrr_n;
2696 struct dmar_atsr_unit *atsru, *atsr_n;
2697 struct dmar_satc_unit *satcu, *satc_n;
2698
2699 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
2700 list_del(&rmrru->list);
2701 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
2702 kfree(rmrru);
2703 }
2704
2705 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
2706 list_del(&atsru->list);
2707 intel_iommu_free_atsr(atsru);
2708 }
2709 list_for_each_entry_safe(satcu, satc_n, &dmar_satc_units, list) {
2710 list_del(&satcu->list);
2711 dmar_free_dev_scope(&satcu->devices, &satcu->devices_cnt);
2712 kfree(satcu);
2713 }
2714}
2715
2716static struct dmar_satc_unit *dmar_find_matched_satc_unit(struct pci_dev *dev)
2717{
2718 struct dmar_satc_unit *satcu;
2719 struct acpi_dmar_satc *satc;
2720 struct device *tmp;
2721 int i;
2722
2723 dev = pci_physfn(dev);
2724 rcu_read_lock();
2725
2726 list_for_each_entry_rcu(satcu, &dmar_satc_units, list) {
2727 satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
2728 if (satc->segment != pci_domain_nr(dev->bus))
2729 continue;
2730 for_each_dev_scope(satcu->devices, satcu->devices_cnt, i, tmp)
2731 if (to_pci_dev(tmp) == dev)
2732 goto out;
2733 }
2734 satcu = NULL;
2735out:
2736 rcu_read_unlock();
2737 return satcu;
2738}
2739
2740static int dmar_ats_supported(struct pci_dev *dev, struct intel_iommu *iommu)
2741{
2742 int i, ret = 1;
2743 struct pci_bus *bus;
2744 struct pci_dev *bridge = NULL;
2745 struct device *tmp;
2746 struct acpi_dmar_atsr *atsr;
2747 struct dmar_atsr_unit *atsru;
2748 struct dmar_satc_unit *satcu;
2749
2750 dev = pci_physfn(dev);
2751 satcu = dmar_find_matched_satc_unit(dev);
2752 if (satcu)
2753 /*
2754 * This device supports ATS as it is in SATC table.
2755 * When IOMMU is in legacy mode, enabling ATS is done
2756 * automatically by HW for the device that requires
2757 * ATS, hence OS should not enable this device ATS
2758 * to avoid duplicated TLB invalidation.
2759 */
2760 return !(satcu->atc_required && !sm_supported(iommu));
2761
2762 for (bus = dev->bus; bus; bus = bus->parent) {
2763 bridge = bus->self;
2764 /* If it's an integrated device, allow ATS */
2765 if (!bridge)
2766 return 1;
2767 /* Connected via non-PCIe: no ATS */
2768 if (!pci_is_pcie(bridge) ||
2769 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
2770 return 0;
2771 /* If we found the root port, look it up in the ATSR */
2772 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
2773 break;
2774 }
2775
2776 rcu_read_lock();
2777 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
2778 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
2779 if (atsr->segment != pci_domain_nr(dev->bus))
2780 continue;
2781
2782 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
2783 if (tmp == &bridge->dev)
2784 goto out;
2785
2786 if (atsru->include_all)
2787 goto out;
2788 }
2789 ret = 0;
2790out:
2791 rcu_read_unlock();
2792
2793 return ret;
2794}
2795
2796int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
2797{
2798 int ret;
2799 struct dmar_rmrr_unit *rmrru;
2800 struct dmar_atsr_unit *atsru;
2801 struct dmar_satc_unit *satcu;
2802 struct acpi_dmar_atsr *atsr;
2803 struct acpi_dmar_reserved_memory *rmrr;
2804 struct acpi_dmar_satc *satc;
2805
2806 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
2807 return 0;
2808
2809 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
2810 rmrr = container_of(rmrru->hdr,
2811 struct acpi_dmar_reserved_memory, header);
2812 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
2813 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
2814 ((void *)rmrr) + rmrr->header.length,
2815 rmrr->segment, rmrru->devices,
2816 rmrru->devices_cnt);
2817 if (ret < 0)
2818 return ret;
2819 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
2820 dmar_remove_dev_scope(info, rmrr->segment,
2821 rmrru->devices, rmrru->devices_cnt);
2822 }
2823 }
2824
2825 list_for_each_entry(atsru, &dmar_atsr_units, list) {
2826 if (atsru->include_all)
2827 continue;
2828
2829 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
2830 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
2831 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
2832 (void *)atsr + atsr->header.length,
2833 atsr->segment, atsru->devices,
2834 atsru->devices_cnt);
2835 if (ret > 0)
2836 break;
2837 else if (ret < 0)
2838 return ret;
2839 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
2840 if (dmar_remove_dev_scope(info, atsr->segment,
2841 atsru->devices, atsru->devices_cnt))
2842 break;
2843 }
2844 }
2845 list_for_each_entry(satcu, &dmar_satc_units, list) {
2846 satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
2847 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
2848 ret = dmar_insert_dev_scope(info, (void *)(satc + 1),
2849 (void *)satc + satc->header.length,
2850 satc->segment, satcu->devices,
2851 satcu->devices_cnt);
2852 if (ret > 0)
2853 break;
2854 else if (ret < 0)
2855 return ret;
2856 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
2857 if (dmar_remove_dev_scope(info, satc->segment,
2858 satcu->devices, satcu->devices_cnt))
2859 break;
2860 }
2861 }
2862
2863 return 0;
2864}
2865
2866static void intel_disable_iommus(void)
2867{
2868 struct intel_iommu *iommu = NULL;
2869 struct dmar_drhd_unit *drhd;
2870
2871 for_each_iommu(iommu, drhd)
2872 iommu_disable_translation(iommu);
2873}
2874
2875void intel_iommu_shutdown(void)
2876{
2877 struct dmar_drhd_unit *drhd;
2878 struct intel_iommu *iommu = NULL;
2879
2880 if (no_iommu || dmar_disabled)
2881 return;
2882
2883 down_write(&dmar_global_lock);
2884
2885 /* Disable PMRs explicitly here. */
2886 for_each_iommu(iommu, drhd)
2887 iommu_disable_protect_mem_regions(iommu);
2888
2889 /* Make sure the IOMMUs are switched off */
2890 intel_disable_iommus();
2891
2892 up_write(&dmar_global_lock);
2893}
2894
2895static struct intel_iommu *dev_to_intel_iommu(struct device *dev)
2896{
2897 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
2898
2899 return container_of(iommu_dev, struct intel_iommu, iommu);
2900}
2901
2902static ssize_t version_show(struct device *dev,
2903 struct device_attribute *attr, char *buf)
2904{
2905 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
2906 u32 ver = readl(iommu->reg + DMAR_VER_REG);
2907 return sysfs_emit(buf, "%d:%d\n",
2908 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
2909}
2910static DEVICE_ATTR_RO(version);
2911
2912static ssize_t address_show(struct device *dev,
2913 struct device_attribute *attr, char *buf)
2914{
2915 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
2916 return sysfs_emit(buf, "%llx\n", iommu->reg_phys);
2917}
2918static DEVICE_ATTR_RO(address);
2919
2920static ssize_t cap_show(struct device *dev,
2921 struct device_attribute *attr, char *buf)
2922{
2923 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
2924 return sysfs_emit(buf, "%llx\n", iommu->cap);
2925}
2926static DEVICE_ATTR_RO(cap);
2927
2928static ssize_t ecap_show(struct device *dev,
2929 struct device_attribute *attr, char *buf)
2930{
2931 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
2932 return sysfs_emit(buf, "%llx\n", iommu->ecap);
2933}
2934static DEVICE_ATTR_RO(ecap);
2935
2936static ssize_t domains_supported_show(struct device *dev,
2937 struct device_attribute *attr, char *buf)
2938{
2939 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
2940 return sysfs_emit(buf, "%ld\n", cap_ndoms(iommu->cap));
2941}
2942static DEVICE_ATTR_RO(domains_supported);
2943
2944static ssize_t domains_used_show(struct device *dev,
2945 struct device_attribute *attr, char *buf)
2946{
2947 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
2948 return sysfs_emit(buf, "%d\n",
2949 bitmap_weight(iommu->domain_ids,
2950 cap_ndoms(iommu->cap)));
2951}
2952static DEVICE_ATTR_RO(domains_used);
2953
2954static struct attribute *intel_iommu_attrs[] = {
2955 &dev_attr_version.attr,
2956 &dev_attr_address.attr,
2957 &dev_attr_cap.attr,
2958 &dev_attr_ecap.attr,
2959 &dev_attr_domains_supported.attr,
2960 &dev_attr_domains_used.attr,
2961 NULL,
2962};
2963
2964static struct attribute_group intel_iommu_group = {
2965 .name = "intel-iommu",
2966 .attrs = intel_iommu_attrs,
2967};
2968
2969const struct attribute_group *intel_iommu_groups[] = {
2970 &intel_iommu_group,
2971 NULL,
2972};
2973
2974static bool has_external_pci(void)
2975{
2976 struct pci_dev *pdev = NULL;
2977
2978 for_each_pci_dev(pdev)
2979 if (pdev->external_facing) {
2980 pci_dev_put(pdev);
2981 return true;
2982 }
2983
2984 return false;
2985}
2986
2987static int __init platform_optin_force_iommu(void)
2988{
2989 if (!dmar_platform_optin() || no_platform_optin || !has_external_pci())
2990 return 0;
2991
2992 if (no_iommu || dmar_disabled)
2993 pr_info("Intel-IOMMU force enabled due to platform opt in\n");
2994
2995 /*
2996 * If Intel-IOMMU is disabled by default, we will apply identity
2997 * map for all devices except those marked as being untrusted.
2998 */
2999 if (dmar_disabled)
3000 iommu_set_default_passthrough(false);
3001
3002 dmar_disabled = 0;
3003 no_iommu = 0;
3004
3005 return 1;
3006}
3007
3008static int __init probe_acpi_namespace_devices(void)
3009{
3010 struct dmar_drhd_unit *drhd;
3011 /* To avoid a -Wunused-but-set-variable warning. */
3012 struct intel_iommu *iommu __maybe_unused;
3013 struct device *dev;
3014 int i, ret = 0;
3015
3016 for_each_active_iommu(iommu, drhd) {
3017 for_each_active_dev_scope(drhd->devices,
3018 drhd->devices_cnt, i, dev) {
3019 struct acpi_device_physical_node *pn;
3020 struct acpi_device *adev;
3021
3022 if (dev->bus != &acpi_bus_type)
3023 continue;
3024
3025 adev = to_acpi_device(dev);
3026 mutex_lock(&adev->physical_node_lock);
3027 list_for_each_entry(pn,
3028 &adev->physical_node_list, node) {
3029 ret = iommu_probe_device(pn->dev);
3030 if (ret)
3031 break;
3032 }
3033 mutex_unlock(&adev->physical_node_lock);
3034
3035 if (ret)
3036 return ret;
3037 }
3038 }
3039
3040 return 0;
3041}
3042
3043static __init int tboot_force_iommu(void)
3044{
3045 if (!tboot_enabled())
3046 return 0;
3047
3048 if (no_iommu || dmar_disabled)
3049 pr_warn("Forcing Intel-IOMMU to enabled\n");
3050
3051 dmar_disabled = 0;
3052 no_iommu = 0;
3053
3054 return 1;
3055}
3056
3057int __init intel_iommu_init(void)
3058{
3059 int ret = -ENODEV;
3060 struct dmar_drhd_unit *drhd;
3061 struct intel_iommu *iommu;
3062
3063 /*
3064 * Intel IOMMU is required for a TXT/tboot launch or platform
3065 * opt in, so enforce that.
3066 */
3067 force_on = (!intel_iommu_tboot_noforce && tboot_force_iommu()) ||
3068 platform_optin_force_iommu();
3069
3070 down_write(&dmar_global_lock);
3071 if (dmar_table_init()) {
3072 if (force_on)
3073 panic("tboot: Failed to initialize DMAR table\n");
3074 goto out_free_dmar;
3075 }
3076
3077 if (dmar_dev_scope_init() < 0) {
3078 if (force_on)
3079 panic("tboot: Failed to initialize DMAR device scope\n");
3080 goto out_free_dmar;
3081 }
3082
3083 up_write(&dmar_global_lock);
3084
3085 /*
3086 * The bus notifier takes the dmar_global_lock, so lockdep will
3087 * complain later when we register it under the lock.
3088 */
3089 dmar_register_bus_notifier();
3090
3091 down_write(&dmar_global_lock);
3092
3093 if (!no_iommu)
3094 intel_iommu_debugfs_init();
3095
3096 if (no_iommu || dmar_disabled) {
3097 /*
3098 * We exit the function here to ensure IOMMU's remapping and
3099 * mempool aren't setup, which means that the IOMMU's PMRs
3100 * won't be disabled via the call to init_dmars(). So disable
3101 * it explicitly here. The PMRs were setup by tboot prior to
3102 * calling SENTER, but the kernel is expected to reset/tear
3103 * down the PMRs.
3104 */
3105 if (intel_iommu_tboot_noforce) {
3106 for_each_iommu(iommu, drhd)
3107 iommu_disable_protect_mem_regions(iommu);
3108 }
3109
3110 /*
3111 * Make sure the IOMMUs are switched off, even when we
3112 * boot into a kexec kernel and the previous kernel left
3113 * them enabled
3114 */
3115 intel_disable_iommus();
3116 goto out_free_dmar;
3117 }
3118
3119 if (list_empty(&dmar_rmrr_units))
3120 pr_info("No RMRR found\n");
3121
3122 if (list_empty(&dmar_atsr_units))
3123 pr_info("No ATSR found\n");
3124
3125 if (list_empty(&dmar_satc_units))
3126 pr_info("No SATC found\n");
3127
3128 init_no_remapping_devices();
3129
3130 ret = init_dmars();
3131 if (ret) {
3132 if (force_on)
3133 panic("tboot: Failed to initialize DMARs\n");
3134 pr_err("Initialization failed\n");
3135 goto out_free_dmar;
3136 }
3137 up_write(&dmar_global_lock);
3138
3139 init_iommu_pm_ops();
3140
3141 down_read(&dmar_global_lock);
3142 for_each_active_iommu(iommu, drhd) {
3143 /*
3144 * The flush queue implementation does not perform
3145 * page-selective invalidations that are required for efficient
3146 * TLB flushes in virtual environments. The benefit of batching
3147 * is likely to be much lower than the overhead of synchronizing
3148 * the virtual and physical IOMMU page-tables.
3149 */
3150 if (cap_caching_mode(iommu->cap) &&
3151 !first_level_by_default(iommu)) {
3152 pr_info_once("IOMMU batching disallowed due to virtualization\n");
3153 iommu_set_dma_strict();
3154 }
3155 iommu_device_sysfs_add(&iommu->iommu, NULL,
3156 intel_iommu_groups,
3157 "%s", iommu->name);
3158 /*
3159 * The iommu device probe is protected by the iommu_probe_device_lock.
3160 * Release the dmar_global_lock before entering the device probe path
3161 * to avoid unnecessary lock order splat.
3162 */
3163 up_read(&dmar_global_lock);
3164 iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);
3165 down_read(&dmar_global_lock);
3166
3167 iommu_pmu_register(iommu);
3168 }
3169
3170 if (probe_acpi_namespace_devices())
3171 pr_warn("ACPI name space devices didn't probe correctly\n");
3172
3173 /* Finally, we enable the DMA remapping hardware. */
3174 for_each_iommu(iommu, drhd) {
3175 if (!drhd->ignored && !translation_pre_enabled(iommu))
3176 iommu_enable_translation(iommu);
3177
3178 iommu_disable_protect_mem_regions(iommu);
3179 }
3180 up_read(&dmar_global_lock);
3181
3182 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
3183
3184 intel_iommu_enabled = 1;
3185
3186 return 0;
3187
3188out_free_dmar:
3189 intel_iommu_free_dmars();
3190 up_write(&dmar_global_lock);
3191 return ret;
3192}
3193
3194static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
3195{
3196 struct device_domain_info *info = opaque;
3197
3198 domain_context_clear_one(info, PCI_BUS_NUM(alias), alias & 0xff);
3199 return 0;
3200}
3201
3202/*
3203 * NB - intel-iommu lacks any sort of reference counting for the users of
3204 * dependent devices. If multiple endpoints have intersecting dependent
3205 * devices, unbinding the driver from any one of them will possibly leave
3206 * the others unable to operate.
3207 */
3208static void domain_context_clear(struct device_domain_info *info)
3209{
3210 if (!dev_is_pci(info->dev)) {
3211 domain_context_clear_one(info, info->bus, info->devfn);
3212 return;
3213 }
3214
3215 pci_for_each_dma_alias(to_pci_dev(info->dev),
3216 &domain_context_clear_one_cb, info);
3217}
3218
3219/*
3220 * Clear the page table pointer in context or pasid table entries so that
3221 * all DMA requests without PASID from the device are blocked. If the page
3222 * table has been set, clean up the data structures.
3223 */
3224void device_block_translation(struct device *dev)
3225{
3226 struct device_domain_info *info = dev_iommu_priv_get(dev);
3227 struct intel_iommu *iommu = info->iommu;
3228 unsigned long flags;
3229
3230 if (info->domain)
3231 cache_tag_unassign_domain(info->domain, dev, IOMMU_NO_PASID);
3232
3233 iommu_disable_pci_caps(info);
3234 if (!dev_is_real_dma_subdevice(dev)) {
3235 if (sm_supported(iommu))
3236 intel_pasid_tear_down_entry(iommu, dev,
3237 IOMMU_NO_PASID, false);
3238 else
3239 domain_context_clear(info);
3240 }
3241
3242 if (!info->domain)
3243 return;
3244
3245 spin_lock_irqsave(&info->domain->lock, flags);
3246 list_del(&info->link);
3247 spin_unlock_irqrestore(&info->domain->lock, flags);
3248
3249 domain_detach_iommu(info->domain, iommu);
3250 info->domain = NULL;
3251}
3252
3253static int blocking_domain_attach_dev(struct iommu_domain *domain,
3254 struct device *dev)
3255{
3256 device_block_translation(dev);
3257 return 0;
3258}
3259
3260static struct iommu_domain blocking_domain = {
3261 .type = IOMMU_DOMAIN_BLOCKED,
3262 .ops = &(const struct iommu_domain_ops) {
3263 .attach_dev = blocking_domain_attach_dev,
3264 }
3265};
3266
3267static int iommu_superpage_capability(struct intel_iommu *iommu, bool first_stage)
3268{
3269 if (!intel_iommu_superpage)
3270 return 0;
3271
3272 if (first_stage)
3273 return cap_fl1gp_support(iommu->cap) ? 2 : 1;
3274
3275 return fls(cap_super_page_val(iommu->cap));
3276}
3277
3278static struct dmar_domain *paging_domain_alloc(struct device *dev, bool first_stage)
3279{
3280 struct device_domain_info *info = dev_iommu_priv_get(dev);
3281 struct intel_iommu *iommu = info->iommu;
3282 struct dmar_domain *domain;
3283 int addr_width;
3284
3285 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
3286 if (!domain)
3287 return ERR_PTR(-ENOMEM);
3288
3289 INIT_LIST_HEAD(&domain->devices);
3290 INIT_LIST_HEAD(&domain->dev_pasids);
3291 INIT_LIST_HEAD(&domain->cache_tags);
3292 spin_lock_init(&domain->lock);
3293 spin_lock_init(&domain->cache_lock);
3294 xa_init(&domain->iommu_array);
3295
3296 domain->nid = dev_to_node(dev);
3297 domain->use_first_level = first_stage;
3298
3299 /* calculate the address width */
3300 addr_width = agaw_to_width(iommu->agaw);
3301 if (addr_width > cap_mgaw(iommu->cap))
3302 addr_width = cap_mgaw(iommu->cap);
3303 domain->gaw = addr_width;
3304 domain->agaw = iommu->agaw;
3305 domain->max_addr = __DOMAIN_MAX_ADDR(addr_width);
3306
3307 /* iommu memory access coherency */
3308 domain->iommu_coherency = iommu_paging_structure_coherency(iommu);
3309
3310 /* pagesize bitmap */
3311 domain->domain.pgsize_bitmap = SZ_4K;
3312 domain->iommu_superpage = iommu_superpage_capability(iommu, first_stage);
3313 domain->domain.pgsize_bitmap |= domain_super_pgsize_bitmap(domain);
3314
3315 /*
3316 * IOVA aperture: First-level translation restricts the input-address
3317 * to a canonical address (i.e., address bits 63:N have the same value
3318 * as address bit [N-1], where N is 48-bits with 4-level paging and
3319 * 57-bits with 5-level paging). Hence, skip bit [N-1].
3320 */
3321 domain->domain.geometry.force_aperture = true;
3322 domain->domain.geometry.aperture_start = 0;
3323 if (first_stage)
3324 domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1);
3325 else
3326 domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw);
3327
3328 /* always allocate the top pgd */
3329 domain->pgd = iommu_alloc_page_node(domain->nid, GFP_KERNEL);
3330 if (!domain->pgd) {
3331 kfree(domain);
3332 return ERR_PTR(-ENOMEM);
3333 }
3334 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
3335
3336 return domain;
3337}
3338
3339static struct iommu_domain *
3340intel_iommu_domain_alloc_paging_flags(struct device *dev, u32 flags,
3341 const struct iommu_user_data *user_data)
3342{
3343 struct device_domain_info *info = dev_iommu_priv_get(dev);
3344 bool dirty_tracking = flags & IOMMU_HWPT_ALLOC_DIRTY_TRACKING;
3345 bool nested_parent = flags & IOMMU_HWPT_ALLOC_NEST_PARENT;
3346 struct intel_iommu *iommu = info->iommu;
3347 struct dmar_domain *dmar_domain;
3348 struct iommu_domain *domain;
3349 bool first_stage;
3350
3351 if (flags &
3352 (~(IOMMU_HWPT_ALLOC_NEST_PARENT | IOMMU_HWPT_ALLOC_DIRTY_TRACKING
3353 | IOMMU_HWPT_FAULT_ID_VALID)))
3354 return ERR_PTR(-EOPNOTSUPP);
3355 if (nested_parent && !nested_supported(iommu))
3356 return ERR_PTR(-EOPNOTSUPP);
3357 if (user_data || (dirty_tracking && !ssads_supported(iommu)))
3358 return ERR_PTR(-EOPNOTSUPP);
3359
3360 /*
3361 * Always allocate the guest compatible page table unless
3362 * IOMMU_HWPT_ALLOC_NEST_PARENT or IOMMU_HWPT_ALLOC_DIRTY_TRACKING
3363 * is specified.
3364 */
3365 if (nested_parent || dirty_tracking) {
3366 if (!sm_supported(iommu) || !ecap_slts(iommu->ecap))
3367 return ERR_PTR(-EOPNOTSUPP);
3368 first_stage = false;
3369 } else {
3370 first_stage = first_level_by_default(iommu);
3371 }
3372
3373 dmar_domain = paging_domain_alloc(dev, first_stage);
3374 if (IS_ERR(dmar_domain))
3375 return ERR_CAST(dmar_domain);
3376 domain = &dmar_domain->domain;
3377 domain->type = IOMMU_DOMAIN_UNMANAGED;
3378 domain->owner = &intel_iommu_ops;
3379 domain->ops = intel_iommu_ops.default_domain_ops;
3380
3381 if (nested_parent) {
3382 dmar_domain->nested_parent = true;
3383 INIT_LIST_HEAD(&dmar_domain->s1_domains);
3384 spin_lock_init(&dmar_domain->s1_lock);
3385 }
3386
3387 if (dirty_tracking) {
3388 if (dmar_domain->use_first_level) {
3389 iommu_domain_free(domain);
3390 return ERR_PTR(-EOPNOTSUPP);
3391 }
3392 domain->dirty_ops = &intel_dirty_ops;
3393 }
3394
3395 return domain;
3396}
3397
3398static void intel_iommu_domain_free(struct iommu_domain *domain)
3399{
3400 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3401
3402 WARN_ON(dmar_domain->nested_parent &&
3403 !list_empty(&dmar_domain->s1_domains));
3404 domain_exit(dmar_domain);
3405}
3406
3407int paging_domain_compatible(struct iommu_domain *domain, struct device *dev)
3408{
3409 struct device_domain_info *info = dev_iommu_priv_get(dev);
3410 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3411 struct intel_iommu *iommu = info->iommu;
3412 int addr_width;
3413
3414 if (WARN_ON_ONCE(!(domain->type & __IOMMU_DOMAIN_PAGING)))
3415 return -EPERM;
3416
3417 if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
3418 return -EINVAL;
3419
3420 if (domain->dirty_ops && !ssads_supported(iommu))
3421 return -EINVAL;
3422
3423 if (dmar_domain->iommu_coherency !=
3424 iommu_paging_structure_coherency(iommu))
3425 return -EINVAL;
3426
3427 if (dmar_domain->iommu_superpage !=
3428 iommu_superpage_capability(iommu, dmar_domain->use_first_level))
3429 return -EINVAL;
3430
3431 if (dmar_domain->use_first_level &&
3432 (!sm_supported(iommu) || !ecap_flts(iommu->ecap)))
3433 return -EINVAL;
3434
3435 /* check if this iommu agaw is sufficient for max mapped address */
3436 addr_width = agaw_to_width(iommu->agaw);
3437 if (addr_width > cap_mgaw(iommu->cap))
3438 addr_width = cap_mgaw(iommu->cap);
3439
3440 if (dmar_domain->gaw > addr_width || dmar_domain->agaw > iommu->agaw)
3441 return -EINVAL;
3442
3443 if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) &&
3444 context_copied(iommu, info->bus, info->devfn))
3445 return intel_pasid_setup_sm_context(dev);
3446
3447 return 0;
3448}
3449
3450static int intel_iommu_attach_device(struct iommu_domain *domain,
3451 struct device *dev)
3452{
3453 int ret;
3454
3455 device_block_translation(dev);
3456
3457 ret = paging_domain_compatible(domain, dev);
3458 if (ret)
3459 return ret;
3460
3461 return dmar_domain_attach_device(to_dmar_domain(domain), dev);
3462}
3463
3464static int intel_iommu_map(struct iommu_domain *domain,
3465 unsigned long iova, phys_addr_t hpa,
3466 size_t size, int iommu_prot, gfp_t gfp)
3467{
3468 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3469 u64 max_addr;
3470 int prot = 0;
3471
3472 if (iommu_prot & IOMMU_READ)
3473 prot |= DMA_PTE_READ;
3474 if (iommu_prot & IOMMU_WRITE)
3475 prot |= DMA_PTE_WRITE;
3476 if (dmar_domain->set_pte_snp)
3477 prot |= DMA_PTE_SNP;
3478
3479 max_addr = iova + size;
3480 if (dmar_domain->max_addr < max_addr) {
3481 u64 end;
3482
3483 /* check if minimum agaw is sufficient for mapped address */
3484 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
3485 if (end < max_addr) {
3486 pr_err("%s: iommu width (%d) is not "
3487 "sufficient for the mapped address (%llx)\n",
3488 __func__, dmar_domain->gaw, max_addr);
3489 return -EFAULT;
3490 }
3491 dmar_domain->max_addr = max_addr;
3492 }
3493 /* Round up size to next multiple of PAGE_SIZE, if it and
3494 the low bits of hpa would take us onto the next page */
3495 size = aligned_nrpages(hpa, size);
3496 return __domain_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
3497 hpa >> VTD_PAGE_SHIFT, size, prot, gfp);
3498}
3499
3500static int intel_iommu_map_pages(struct iommu_domain *domain,
3501 unsigned long iova, phys_addr_t paddr,
3502 size_t pgsize, size_t pgcount,
3503 int prot, gfp_t gfp, size_t *mapped)
3504{
3505 unsigned long pgshift = __ffs(pgsize);
3506 size_t size = pgcount << pgshift;
3507 int ret;
3508
3509 if (pgsize != SZ_4K && pgsize != SZ_2M && pgsize != SZ_1G)
3510 return -EINVAL;
3511
3512 if (!IS_ALIGNED(iova | paddr, pgsize))
3513 return -EINVAL;
3514
3515 ret = intel_iommu_map(domain, iova, paddr, size, prot, gfp);
3516 if (!ret && mapped)
3517 *mapped = size;
3518
3519 return ret;
3520}
3521
3522static size_t intel_iommu_unmap(struct iommu_domain *domain,
3523 unsigned long iova, size_t size,
3524 struct iommu_iotlb_gather *gather)
3525{
3526 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3527 unsigned long start_pfn, last_pfn;
3528 int level = 0;
3529
3530 /* Cope with horrid API which requires us to unmap more than the
3531 size argument if it happens to be a large-page mapping. */
3532 if (unlikely(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT,
3533 &level, GFP_ATOMIC)))
3534 return 0;
3535
3536 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
3537 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
3538
3539 start_pfn = iova >> VTD_PAGE_SHIFT;
3540 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
3541
3542 domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist);
3543
3544 if (dmar_domain->max_addr == iova + size)
3545 dmar_domain->max_addr = iova;
3546
3547 /*
3548 * We do not use page-selective IOTLB invalidation in flush queue,
3549 * so there is no need to track page and sync iotlb.
3550 */
3551 if (!iommu_iotlb_gather_queued(gather))
3552 iommu_iotlb_gather_add_page(domain, gather, iova, size);
3553
3554 return size;
3555}
3556
3557static size_t intel_iommu_unmap_pages(struct iommu_domain *domain,
3558 unsigned long iova,
3559 size_t pgsize, size_t pgcount,
3560 struct iommu_iotlb_gather *gather)
3561{
3562 unsigned long pgshift = __ffs(pgsize);
3563 size_t size = pgcount << pgshift;
3564
3565 return intel_iommu_unmap(domain, iova, size, gather);
3566}
3567
3568static void intel_iommu_tlb_sync(struct iommu_domain *domain,
3569 struct iommu_iotlb_gather *gather)
3570{
3571 cache_tag_flush_range(to_dmar_domain(domain), gather->start,
3572 gather->end, list_empty(&gather->freelist));
3573 iommu_put_pages_list(&gather->freelist);
3574}
3575
3576static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
3577 dma_addr_t iova)
3578{
3579 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3580 struct dma_pte *pte;
3581 int level = 0;
3582 u64 phys = 0;
3583
3584 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level,
3585 GFP_ATOMIC);
3586 if (pte && dma_pte_present(pte))
3587 phys = dma_pte_addr(pte) +
3588 (iova & (BIT_MASK(level_to_offset_bits(level) +
3589 VTD_PAGE_SHIFT) - 1));
3590
3591 return phys;
3592}
3593
3594static bool domain_support_force_snooping(struct dmar_domain *domain)
3595{
3596 struct device_domain_info *info;
3597 bool support = true;
3598
3599 assert_spin_locked(&domain->lock);
3600 list_for_each_entry(info, &domain->devices, link) {
3601 if (!ecap_sc_support(info->iommu->ecap)) {
3602 support = false;
3603 break;
3604 }
3605 }
3606
3607 return support;
3608}
3609
3610static void domain_set_force_snooping(struct dmar_domain *domain)
3611{
3612 struct device_domain_info *info;
3613
3614 assert_spin_locked(&domain->lock);
3615 /*
3616 * Second level page table supports per-PTE snoop control. The
3617 * iommu_map() interface will handle this by setting SNP bit.
3618 */
3619 if (!domain->use_first_level) {
3620 domain->set_pte_snp = true;
3621 return;
3622 }
3623
3624 list_for_each_entry(info, &domain->devices, link)
3625 intel_pasid_setup_page_snoop_control(info->iommu, info->dev,
3626 IOMMU_NO_PASID);
3627}
3628
3629static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain)
3630{
3631 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
3632 unsigned long flags;
3633
3634 if (dmar_domain->force_snooping)
3635 return true;
3636
3637 spin_lock_irqsave(&dmar_domain->lock, flags);
3638 if (!domain_support_force_snooping(dmar_domain) ||
3639 (!dmar_domain->use_first_level && dmar_domain->has_mappings)) {
3640 spin_unlock_irqrestore(&dmar_domain->lock, flags);
3641 return false;
3642 }
3643
3644 domain_set_force_snooping(dmar_domain);
3645 dmar_domain->force_snooping = true;
3646 spin_unlock_irqrestore(&dmar_domain->lock, flags);
3647
3648 return true;
3649}
3650
3651static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap)
3652{
3653 struct device_domain_info *info = dev_iommu_priv_get(dev);
3654
3655 switch (cap) {
3656 case IOMMU_CAP_CACHE_COHERENCY:
3657 case IOMMU_CAP_DEFERRED_FLUSH:
3658 return true;
3659 case IOMMU_CAP_PRE_BOOT_PROTECTION:
3660 return dmar_platform_optin();
3661 case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
3662 return ecap_sc_support(info->iommu->ecap);
3663 case IOMMU_CAP_DIRTY_TRACKING:
3664 return ssads_supported(info->iommu);
3665 default:
3666 return false;
3667 }
3668}
3669
3670static struct iommu_device *intel_iommu_probe_device(struct device *dev)
3671{
3672 struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;
3673 struct device_domain_info *info;
3674 struct intel_iommu *iommu;
3675 u8 bus, devfn;
3676 int ret;
3677
3678 iommu = device_lookup_iommu(dev, &bus, &devfn);
3679 if (!iommu || !iommu->iommu.ops)
3680 return ERR_PTR(-ENODEV);
3681
3682 info = kzalloc(sizeof(*info), GFP_KERNEL);
3683 if (!info)
3684 return ERR_PTR(-ENOMEM);
3685
3686 if (dev_is_real_dma_subdevice(dev)) {
3687 info->bus = pdev->bus->number;
3688 info->devfn = pdev->devfn;
3689 info->segment = pci_domain_nr(pdev->bus);
3690 } else {
3691 info->bus = bus;
3692 info->devfn = devfn;
3693 info->segment = iommu->segment;
3694 }
3695
3696 info->dev = dev;
3697 info->iommu = iommu;
3698 if (dev_is_pci(dev)) {
3699 if (ecap_dev_iotlb_support(iommu->ecap) &&
3700 pci_ats_supported(pdev) &&
3701 dmar_ats_supported(pdev, iommu)) {
3702 info->ats_supported = 1;
3703 info->dtlb_extra_inval = dev_needs_extra_dtlb_flush(pdev);
3704
3705 /*
3706 * For IOMMU that supports device IOTLB throttling
3707 * (DIT), we assign PFSID to the invalidation desc
3708 * of a VF such that IOMMU HW can gauge queue depth
3709 * at PF level. If DIT is not set, PFSID will be
3710 * treated as reserved, which should be set to 0.
3711 */
3712 if (ecap_dit(iommu->ecap))
3713 info->pfsid = pci_dev_id(pci_physfn(pdev));
3714 info->ats_qdep = pci_ats_queue_depth(pdev);
3715 }
3716 if (sm_supported(iommu)) {
3717 if (pasid_supported(iommu)) {
3718 int features = pci_pasid_features(pdev);
3719
3720 if (features >= 0)
3721 info->pasid_supported = features | 1;
3722 }
3723
3724 if (info->ats_supported && ecap_prs(iommu->ecap) &&
3725 pci_pri_supported(pdev))
3726 info->pri_supported = 1;
3727 }
3728 }
3729
3730 dev_iommu_priv_set(dev, info);
3731 if (pdev && pci_ats_supported(pdev)) {
3732 pci_prepare_ats(pdev, VTD_PAGE_SHIFT);
3733 ret = device_rbtree_insert(iommu, info);
3734 if (ret)
3735 goto free;
3736 }
3737
3738 if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
3739 ret = intel_pasid_alloc_table(dev);
3740 if (ret) {
3741 dev_err(dev, "PASID table allocation failed\n");
3742 goto clear_rbtree;
3743 }
3744
3745 if (!context_copied(iommu, info->bus, info->devfn)) {
3746 ret = intel_pasid_setup_sm_context(dev);
3747 if (ret)
3748 goto free_table;
3749 }
3750 }
3751
3752 intel_iommu_debugfs_create_dev(info);
3753
3754 /*
3755 * The PCIe spec, in its wisdom, declares that the behaviour of the
3756 * device is undefined if you enable PASID support after ATS support.
3757 * So always enable PASID support on devices which have it, even if
3758 * we can't yet know if we're ever going to use it.
3759 */
3760 if (info->pasid_supported &&
3761 !pci_enable_pasid(pdev, info->pasid_supported & ~1))
3762 info->pasid_enabled = 1;
3763
3764 return &iommu->iommu;
3765free_table:
3766 intel_pasid_free_table(dev);
3767clear_rbtree:
3768 device_rbtree_remove(info);
3769free:
3770 kfree(info);
3771
3772 return ERR_PTR(ret);
3773}
3774
3775static void intel_iommu_release_device(struct device *dev)
3776{
3777 struct device_domain_info *info = dev_iommu_priv_get(dev);
3778 struct intel_iommu *iommu = info->iommu;
3779
3780 if (info->pasid_enabled) {
3781 pci_disable_pasid(to_pci_dev(dev));
3782 info->pasid_enabled = 0;
3783 }
3784
3785 mutex_lock(&iommu->iopf_lock);
3786 if (dev_is_pci(dev) && pci_ats_supported(to_pci_dev(dev)))
3787 device_rbtree_remove(info);
3788 mutex_unlock(&iommu->iopf_lock);
3789
3790 if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev) &&
3791 !context_copied(iommu, info->bus, info->devfn))
3792 intel_pasid_teardown_sm_context(dev);
3793
3794 intel_pasid_free_table(dev);
3795 intel_iommu_debugfs_remove_dev(info);
3796 kfree(info);
3797 set_dma_ops(dev, NULL);
3798}
3799
3800static void intel_iommu_get_resv_regions(struct device *device,
3801 struct list_head *head)
3802{
3803 int prot = DMA_PTE_READ | DMA_PTE_WRITE;
3804 struct iommu_resv_region *reg;
3805 struct dmar_rmrr_unit *rmrr;
3806 struct device *i_dev;
3807 int i;
3808
3809 rcu_read_lock();
3810 for_each_rmrr_units(rmrr) {
3811 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
3812 i, i_dev) {
3813 struct iommu_resv_region *resv;
3814 enum iommu_resv_type type;
3815 size_t length;
3816
3817 if (i_dev != device &&
3818 !is_downstream_to_pci_bridge(device, i_dev))
3819 continue;
3820
3821 length = rmrr->end_address - rmrr->base_address + 1;
3822
3823 type = device_rmrr_is_relaxable(device) ?
3824 IOMMU_RESV_DIRECT_RELAXABLE : IOMMU_RESV_DIRECT;
3825
3826 resv = iommu_alloc_resv_region(rmrr->base_address,
3827 length, prot, type,
3828 GFP_ATOMIC);
3829 if (!resv)
3830 break;
3831
3832 list_add_tail(&resv->list, head);
3833 }
3834 }
3835 rcu_read_unlock();
3836
3837#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
3838 if (dev_is_pci(device)) {
3839 struct pci_dev *pdev = to_pci_dev(device);
3840
3841 if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) {
3842 reg = iommu_alloc_resv_region(0, 1UL << 24, prot,
3843 IOMMU_RESV_DIRECT_RELAXABLE,
3844 GFP_KERNEL);
3845 if (reg)
3846 list_add_tail(®->list, head);
3847 }
3848 }
3849#endif /* CONFIG_INTEL_IOMMU_FLOPPY_WA */
3850
3851 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
3852 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
3853 0, IOMMU_RESV_MSI, GFP_KERNEL);
3854 if (!reg)
3855 return;
3856 list_add_tail(®->list, head);
3857}
3858
3859static struct iommu_group *intel_iommu_device_group(struct device *dev)
3860{
3861 if (dev_is_pci(dev))
3862 return pci_device_group(dev);
3863 return generic_device_group(dev);
3864}
3865
3866static int intel_iommu_enable_sva(struct device *dev)
3867{
3868 struct device_domain_info *info = dev_iommu_priv_get(dev);
3869 struct intel_iommu *iommu;
3870
3871 if (!info || dmar_disabled)
3872 return -EINVAL;
3873
3874 iommu = info->iommu;
3875 if (!iommu)
3876 return -EINVAL;
3877
3878 if (!(iommu->flags & VTD_FLAG_SVM_CAPABLE))
3879 return -ENODEV;
3880
3881 if (!info->pasid_enabled || !info->ats_enabled)
3882 return -EINVAL;
3883
3884 /*
3885 * Devices having device-specific I/O fault handling should not
3886 * support PCI/PRI. The IOMMU side has no means to check the
3887 * capability of device-specific IOPF. Therefore, IOMMU can only
3888 * default that if the device driver enables SVA on a non-PRI
3889 * device, it will handle IOPF in its own way.
3890 */
3891 if (!info->pri_supported)
3892 return 0;
3893
3894 /* Devices supporting PRI should have it enabled. */
3895 if (!info->pri_enabled)
3896 return -EINVAL;
3897
3898 return 0;
3899}
3900
3901static int context_flip_pri(struct device_domain_info *info, bool enable)
3902{
3903 struct intel_iommu *iommu = info->iommu;
3904 u8 bus = info->bus, devfn = info->devfn;
3905 struct context_entry *context;
3906 u16 did;
3907
3908 spin_lock(&iommu->lock);
3909 if (context_copied(iommu, bus, devfn)) {
3910 spin_unlock(&iommu->lock);
3911 return -EINVAL;
3912 }
3913
3914 context = iommu_context_addr(iommu, bus, devfn, false);
3915 if (!context || !context_present(context)) {
3916 spin_unlock(&iommu->lock);
3917 return -ENODEV;
3918 }
3919 did = context_domain_id(context);
3920
3921 if (enable)
3922 context_set_sm_pre(context);
3923 else
3924 context_clear_sm_pre(context);
3925
3926 if (!ecap_coherent(iommu->ecap))
3927 clflush_cache_range(context, sizeof(*context));
3928 intel_context_flush_present(info, context, did, true);
3929 spin_unlock(&iommu->lock);
3930
3931 return 0;
3932}
3933
3934static int intel_iommu_enable_iopf(struct device *dev)
3935{
3936 struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;
3937 struct device_domain_info *info = dev_iommu_priv_get(dev);
3938 struct intel_iommu *iommu;
3939 int ret;
3940
3941 if (!pdev || !info || !info->ats_enabled || !info->pri_supported)
3942 return -ENODEV;
3943
3944 if (info->pri_enabled)
3945 return -EBUSY;
3946
3947 iommu = info->iommu;
3948 if (!iommu)
3949 return -EINVAL;
3950
3951 /* PASID is required in PRG Response Message. */
3952 if (info->pasid_enabled && !pci_prg_resp_pasid_required(pdev))
3953 return -EINVAL;
3954
3955 ret = pci_reset_pri(pdev);
3956 if (ret)
3957 return ret;
3958
3959 ret = iopf_queue_add_device(iommu->iopf_queue, dev);
3960 if (ret)
3961 return ret;
3962
3963 ret = context_flip_pri(info, true);
3964 if (ret)
3965 goto err_remove_device;
3966
3967 ret = pci_enable_pri(pdev, PRQ_DEPTH);
3968 if (ret)
3969 goto err_clear_pri;
3970
3971 info->pri_enabled = 1;
3972
3973 return 0;
3974err_clear_pri:
3975 context_flip_pri(info, false);
3976err_remove_device:
3977 iopf_queue_remove_device(iommu->iopf_queue, dev);
3978
3979 return ret;
3980}
3981
3982static int intel_iommu_disable_iopf(struct device *dev)
3983{
3984 struct device_domain_info *info = dev_iommu_priv_get(dev);
3985 struct intel_iommu *iommu = info->iommu;
3986
3987 if (!info->pri_enabled)
3988 return -EINVAL;
3989
3990 /* Disable new PRI reception: */
3991 context_flip_pri(info, false);
3992
3993 /*
3994 * Remove device from fault queue and acknowledge all outstanding
3995 * PRQs to the device:
3996 */
3997 iopf_queue_remove_device(iommu->iopf_queue, dev);
3998
3999 /*
4000 * PCIe spec states that by clearing PRI enable bit, the Page
4001 * Request Interface will not issue new page requests, but has
4002 * outstanding page requests that have been transmitted or are
4003 * queued for transmission. This is supposed to be called after
4004 * the device driver has stopped DMA, all PASIDs have been
4005 * unbound and the outstanding PRQs have been drained.
4006 */
4007 pci_disable_pri(to_pci_dev(dev));
4008 info->pri_enabled = 0;
4009
4010 return 0;
4011}
4012
4013static int
4014intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
4015{
4016 switch (feat) {
4017 case IOMMU_DEV_FEAT_IOPF:
4018 return intel_iommu_enable_iopf(dev);
4019
4020 case IOMMU_DEV_FEAT_SVA:
4021 return intel_iommu_enable_sva(dev);
4022
4023 default:
4024 return -ENODEV;
4025 }
4026}
4027
4028static int
4029intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
4030{
4031 switch (feat) {
4032 case IOMMU_DEV_FEAT_IOPF:
4033 return intel_iommu_disable_iopf(dev);
4034
4035 case IOMMU_DEV_FEAT_SVA:
4036 return 0;
4037
4038 default:
4039 return -ENODEV;
4040 }
4041}
4042
4043static bool intel_iommu_is_attach_deferred(struct device *dev)
4044{
4045 struct device_domain_info *info = dev_iommu_priv_get(dev);
4046
4047 return translation_pre_enabled(info->iommu) && !info->domain;
4048}
4049
4050/*
4051 * Check that the device does not live on an external facing PCI port that is
4052 * marked as untrusted. Such devices should not be able to apply quirks and
4053 * thus not be able to bypass the IOMMU restrictions.
4054 */
4055static bool risky_device(struct pci_dev *pdev)
4056{
4057 if (pdev->untrusted) {
4058 pci_info(pdev,
4059 "Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n",
4060 pdev->vendor, pdev->device);
4061 pci_info(pdev, "Please check with your BIOS/Platform vendor about this\n");
4062 return true;
4063 }
4064 return false;
4065}
4066
4067static int intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
4068 unsigned long iova, size_t size)
4069{
4070 cache_tag_flush_range_np(to_dmar_domain(domain), iova, iova + size - 1);
4071
4072 return 0;
4073}
4074
4075void domain_remove_dev_pasid(struct iommu_domain *domain,
4076 struct device *dev, ioasid_t pasid)
4077{
4078 struct device_domain_info *info = dev_iommu_priv_get(dev);
4079 struct dev_pasid_info *curr, *dev_pasid = NULL;
4080 struct intel_iommu *iommu = info->iommu;
4081 struct dmar_domain *dmar_domain;
4082 unsigned long flags;
4083
4084 if (!domain)
4085 return;
4086
4087 /* Identity domain has no meta data for pasid. */
4088 if (domain->type == IOMMU_DOMAIN_IDENTITY)
4089 return;
4090
4091 dmar_domain = to_dmar_domain(domain);
4092 spin_lock_irqsave(&dmar_domain->lock, flags);
4093 list_for_each_entry(curr, &dmar_domain->dev_pasids, link_domain) {
4094 if (curr->dev == dev && curr->pasid == pasid) {
4095 list_del(&curr->link_domain);
4096 dev_pasid = curr;
4097 break;
4098 }
4099 }
4100 spin_unlock_irqrestore(&dmar_domain->lock, flags);
4101
4102 cache_tag_unassign_domain(dmar_domain, dev, pasid);
4103 domain_detach_iommu(dmar_domain, iommu);
4104 if (!WARN_ON_ONCE(!dev_pasid)) {
4105 intel_iommu_debugfs_remove_dev_pasid(dev_pasid);
4106 kfree(dev_pasid);
4107 }
4108}
4109
4110static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid,
4111 struct iommu_domain *domain)
4112{
4113 struct device_domain_info *info = dev_iommu_priv_get(dev);
4114
4115 intel_pasid_tear_down_entry(info->iommu, dev, pasid, false);
4116 domain_remove_dev_pasid(domain, dev, pasid);
4117}
4118
4119struct dev_pasid_info *
4120domain_add_dev_pasid(struct iommu_domain *domain,
4121 struct device *dev, ioasid_t pasid)
4122{
4123 struct device_domain_info *info = dev_iommu_priv_get(dev);
4124 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4125 struct intel_iommu *iommu = info->iommu;
4126 struct dev_pasid_info *dev_pasid;
4127 unsigned long flags;
4128 int ret;
4129
4130 dev_pasid = kzalloc(sizeof(*dev_pasid), GFP_KERNEL);
4131 if (!dev_pasid)
4132 return ERR_PTR(-ENOMEM);
4133
4134 ret = domain_attach_iommu(dmar_domain, iommu);
4135 if (ret)
4136 goto out_free;
4137
4138 ret = cache_tag_assign_domain(dmar_domain, dev, pasid);
4139 if (ret)
4140 goto out_detach_iommu;
4141
4142 dev_pasid->dev = dev;
4143 dev_pasid->pasid = pasid;
4144 spin_lock_irqsave(&dmar_domain->lock, flags);
4145 list_add(&dev_pasid->link_domain, &dmar_domain->dev_pasids);
4146 spin_unlock_irqrestore(&dmar_domain->lock, flags);
4147
4148 return dev_pasid;
4149out_detach_iommu:
4150 domain_detach_iommu(dmar_domain, iommu);
4151out_free:
4152 kfree(dev_pasid);
4153 return ERR_PTR(ret);
4154}
4155
4156static int intel_iommu_set_dev_pasid(struct iommu_domain *domain,
4157 struct device *dev, ioasid_t pasid,
4158 struct iommu_domain *old)
4159{
4160 struct device_domain_info *info = dev_iommu_priv_get(dev);
4161 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4162 struct intel_iommu *iommu = info->iommu;
4163 struct dev_pasid_info *dev_pasid;
4164 int ret;
4165
4166 if (WARN_ON_ONCE(!(domain->type & __IOMMU_DOMAIN_PAGING)))
4167 return -EINVAL;
4168
4169 if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))
4170 return -EOPNOTSUPP;
4171
4172 if (domain->dirty_ops)
4173 return -EINVAL;
4174
4175 if (context_copied(iommu, info->bus, info->devfn))
4176 return -EBUSY;
4177
4178 ret = paging_domain_compatible(domain, dev);
4179 if (ret)
4180 return ret;
4181
4182 dev_pasid = domain_add_dev_pasid(domain, dev, pasid);
4183 if (IS_ERR(dev_pasid))
4184 return PTR_ERR(dev_pasid);
4185
4186 if (dmar_domain->use_first_level)
4187 ret = domain_setup_first_level(iommu, dmar_domain,
4188 dev, pasid, old);
4189 else
4190 ret = domain_setup_second_level(iommu, dmar_domain,
4191 dev, pasid, old);
4192 if (ret)
4193 goto out_remove_dev_pasid;
4194
4195 domain_remove_dev_pasid(old, dev, pasid);
4196
4197 intel_iommu_debugfs_create_dev_pasid(dev_pasid);
4198
4199 return 0;
4200
4201out_remove_dev_pasid:
4202 domain_remove_dev_pasid(domain, dev, pasid);
4203 return ret;
4204}
4205
4206static void *intel_iommu_hw_info(struct device *dev, u32 *length, u32 *type)
4207{
4208 struct device_domain_info *info = dev_iommu_priv_get(dev);
4209 struct intel_iommu *iommu = info->iommu;
4210 struct iommu_hw_info_vtd *vtd;
4211
4212 vtd = kzalloc(sizeof(*vtd), GFP_KERNEL);
4213 if (!vtd)
4214 return ERR_PTR(-ENOMEM);
4215
4216 vtd->flags = IOMMU_HW_INFO_VTD_ERRATA_772415_SPR17;
4217 vtd->cap_reg = iommu->cap;
4218 vtd->ecap_reg = iommu->ecap;
4219 *length = sizeof(*vtd);
4220 *type = IOMMU_HW_INFO_TYPE_INTEL_VTD;
4221 return vtd;
4222}
4223
4224/*
4225 * Set dirty tracking for the device list of a domain. The caller must
4226 * hold the domain->lock when calling it.
4227 */
4228static int device_set_dirty_tracking(struct list_head *devices, bool enable)
4229{
4230 struct device_domain_info *info;
4231 int ret = 0;
4232
4233 list_for_each_entry(info, devices, link) {
4234 ret = intel_pasid_setup_dirty_tracking(info->iommu, info->dev,
4235 IOMMU_NO_PASID, enable);
4236 if (ret)
4237 break;
4238 }
4239
4240 return ret;
4241}
4242
4243static int parent_domain_set_dirty_tracking(struct dmar_domain *domain,
4244 bool enable)
4245{
4246 struct dmar_domain *s1_domain;
4247 unsigned long flags;
4248 int ret;
4249
4250 spin_lock(&domain->s1_lock);
4251 list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) {
4252 spin_lock_irqsave(&s1_domain->lock, flags);
4253 ret = device_set_dirty_tracking(&s1_domain->devices, enable);
4254 spin_unlock_irqrestore(&s1_domain->lock, flags);
4255 if (ret)
4256 goto err_unwind;
4257 }
4258 spin_unlock(&domain->s1_lock);
4259 return 0;
4260
4261err_unwind:
4262 list_for_each_entry(s1_domain, &domain->s1_domains, s2_link) {
4263 spin_lock_irqsave(&s1_domain->lock, flags);
4264 device_set_dirty_tracking(&s1_domain->devices,
4265 domain->dirty_tracking);
4266 spin_unlock_irqrestore(&s1_domain->lock, flags);
4267 }
4268 spin_unlock(&domain->s1_lock);
4269 return ret;
4270}
4271
4272static int intel_iommu_set_dirty_tracking(struct iommu_domain *domain,
4273 bool enable)
4274{
4275 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4276 int ret;
4277
4278 spin_lock(&dmar_domain->lock);
4279 if (dmar_domain->dirty_tracking == enable)
4280 goto out_unlock;
4281
4282 ret = device_set_dirty_tracking(&dmar_domain->devices, enable);
4283 if (ret)
4284 goto err_unwind;
4285
4286 if (dmar_domain->nested_parent) {
4287 ret = parent_domain_set_dirty_tracking(dmar_domain, enable);
4288 if (ret)
4289 goto err_unwind;
4290 }
4291
4292 dmar_domain->dirty_tracking = enable;
4293out_unlock:
4294 spin_unlock(&dmar_domain->lock);
4295
4296 return 0;
4297
4298err_unwind:
4299 device_set_dirty_tracking(&dmar_domain->devices,
4300 dmar_domain->dirty_tracking);
4301 spin_unlock(&dmar_domain->lock);
4302 return ret;
4303}
4304
4305static int intel_iommu_read_and_clear_dirty(struct iommu_domain *domain,
4306 unsigned long iova, size_t size,
4307 unsigned long flags,
4308 struct iommu_dirty_bitmap *dirty)
4309{
4310 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4311 unsigned long end = iova + size - 1;
4312 unsigned long pgsize;
4313
4314 /*
4315 * IOMMUFD core calls into a dirty tracking disabled domain without an
4316 * IOVA bitmap set in order to clean dirty bits in all PTEs that might
4317 * have occurred when we stopped dirty tracking. This ensures that we
4318 * never inherit dirtied bits from a previous cycle.
4319 */
4320 if (!dmar_domain->dirty_tracking && dirty->bitmap)
4321 return -EINVAL;
4322
4323 do {
4324 struct dma_pte *pte;
4325 int lvl = 0;
4326
4327 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &lvl,
4328 GFP_ATOMIC);
4329 pgsize = level_size(lvl) << VTD_PAGE_SHIFT;
4330 if (!pte || !dma_pte_present(pte)) {
4331 iova += pgsize;
4332 continue;
4333 }
4334
4335 if (dma_sl_pte_test_and_clear_dirty(pte, flags))
4336 iommu_dirty_bitmap_record(dirty, iova, pgsize);
4337 iova += pgsize;
4338 } while (iova < end);
4339
4340 return 0;
4341}
4342
4343static const struct iommu_dirty_ops intel_dirty_ops = {
4344 .set_dirty_tracking = intel_iommu_set_dirty_tracking,
4345 .read_and_clear_dirty = intel_iommu_read_and_clear_dirty,
4346};
4347
4348static int context_setup_pass_through(struct device *dev, u8 bus, u8 devfn)
4349{
4350 struct device_domain_info *info = dev_iommu_priv_get(dev);
4351 struct intel_iommu *iommu = info->iommu;
4352 struct context_entry *context;
4353
4354 spin_lock(&iommu->lock);
4355 context = iommu_context_addr(iommu, bus, devfn, 1);
4356 if (!context) {
4357 spin_unlock(&iommu->lock);
4358 return -ENOMEM;
4359 }
4360
4361 if (context_present(context) && !context_copied(iommu, bus, devfn)) {
4362 spin_unlock(&iommu->lock);
4363 return 0;
4364 }
4365
4366 copied_context_tear_down(iommu, context, bus, devfn);
4367 context_clear_entry(context);
4368 context_set_domain_id(context, FLPT_DEFAULT_DID);
4369
4370 /*
4371 * In pass through mode, AW must be programmed to indicate the largest
4372 * AGAW value supported by hardware. And ASR is ignored by hardware.
4373 */
4374 context_set_address_width(context, iommu->msagaw);
4375 context_set_translation_type(context, CONTEXT_TT_PASS_THROUGH);
4376 context_set_fault_enable(context);
4377 context_set_present(context);
4378 if (!ecap_coherent(iommu->ecap))
4379 clflush_cache_range(context, sizeof(*context));
4380 context_present_cache_flush(iommu, FLPT_DEFAULT_DID, bus, devfn);
4381 spin_unlock(&iommu->lock);
4382
4383 return 0;
4384}
4385
4386static int context_setup_pass_through_cb(struct pci_dev *pdev, u16 alias, void *data)
4387{
4388 struct device *dev = data;
4389
4390 return context_setup_pass_through(dev, PCI_BUS_NUM(alias), alias & 0xff);
4391}
4392
4393static int device_setup_pass_through(struct device *dev)
4394{
4395 struct device_domain_info *info = dev_iommu_priv_get(dev);
4396
4397 if (!dev_is_pci(dev))
4398 return context_setup_pass_through(dev, info->bus, info->devfn);
4399
4400 return pci_for_each_dma_alias(to_pci_dev(dev),
4401 context_setup_pass_through_cb, dev);
4402}
4403
4404static int identity_domain_attach_dev(struct iommu_domain *domain, struct device *dev)
4405{
4406 struct device_domain_info *info = dev_iommu_priv_get(dev);
4407 struct intel_iommu *iommu = info->iommu;
4408 int ret;
4409
4410 device_block_translation(dev);
4411
4412 if (dev_is_real_dma_subdevice(dev))
4413 return 0;
4414
4415 if (sm_supported(iommu)) {
4416 ret = intel_pasid_setup_pass_through(iommu, dev, IOMMU_NO_PASID);
4417 if (!ret)
4418 iommu_enable_pci_caps(info);
4419 } else {
4420 ret = device_setup_pass_through(dev);
4421 }
4422
4423 return ret;
4424}
4425
4426static int identity_domain_set_dev_pasid(struct iommu_domain *domain,
4427 struct device *dev, ioasid_t pasid,
4428 struct iommu_domain *old)
4429{
4430 struct device_domain_info *info = dev_iommu_priv_get(dev);
4431 struct intel_iommu *iommu = info->iommu;
4432 int ret;
4433
4434 if (!pasid_supported(iommu) || dev_is_real_dma_subdevice(dev))
4435 return -EOPNOTSUPP;
4436
4437 ret = domain_setup_passthrough(iommu, dev, pasid, old);
4438 if (ret)
4439 return ret;
4440
4441 domain_remove_dev_pasid(old, dev, pasid);
4442 return 0;
4443}
4444
4445static struct iommu_domain identity_domain = {
4446 .type = IOMMU_DOMAIN_IDENTITY,
4447 .ops = &(const struct iommu_domain_ops) {
4448 .attach_dev = identity_domain_attach_dev,
4449 .set_dev_pasid = identity_domain_set_dev_pasid,
4450 },
4451};
4452
4453static struct iommu_domain *intel_iommu_domain_alloc_paging(struct device *dev)
4454{
4455 struct device_domain_info *info = dev_iommu_priv_get(dev);
4456 struct intel_iommu *iommu = info->iommu;
4457 struct dmar_domain *dmar_domain;
4458 bool first_stage;
4459
4460 first_stage = first_level_by_default(iommu);
4461 dmar_domain = paging_domain_alloc(dev, first_stage);
4462 if (IS_ERR(dmar_domain))
4463 return ERR_CAST(dmar_domain);
4464
4465 return &dmar_domain->domain;
4466}
4467
4468const struct iommu_ops intel_iommu_ops = {
4469 .blocked_domain = &blocking_domain,
4470 .release_domain = &blocking_domain,
4471 .identity_domain = &identity_domain,
4472 .capable = intel_iommu_capable,
4473 .hw_info = intel_iommu_hw_info,
4474 .domain_alloc_paging_flags = intel_iommu_domain_alloc_paging_flags,
4475 .domain_alloc_sva = intel_svm_domain_alloc,
4476 .domain_alloc_paging = intel_iommu_domain_alloc_paging,
4477 .domain_alloc_nested = intel_iommu_domain_alloc_nested,
4478 .probe_device = intel_iommu_probe_device,
4479 .release_device = intel_iommu_release_device,
4480 .get_resv_regions = intel_iommu_get_resv_regions,
4481 .device_group = intel_iommu_device_group,
4482 .dev_enable_feat = intel_iommu_dev_enable_feat,
4483 .dev_disable_feat = intel_iommu_dev_disable_feat,
4484 .is_attach_deferred = intel_iommu_is_attach_deferred,
4485 .def_domain_type = device_def_domain_type,
4486 .remove_dev_pasid = intel_iommu_remove_dev_pasid,
4487 .pgsize_bitmap = SZ_4K,
4488 .page_response = intel_iommu_page_response,
4489 .default_domain_ops = &(const struct iommu_domain_ops) {
4490 .attach_dev = intel_iommu_attach_device,
4491 .set_dev_pasid = intel_iommu_set_dev_pasid,
4492 .map_pages = intel_iommu_map_pages,
4493 .unmap_pages = intel_iommu_unmap_pages,
4494 .iotlb_sync_map = intel_iommu_iotlb_sync_map,
4495 .flush_iotlb_all = intel_flush_iotlb_all,
4496 .iotlb_sync = intel_iommu_tlb_sync,
4497 .iova_to_phys = intel_iommu_iova_to_phys,
4498 .free = intel_iommu_domain_free,
4499 .enforce_cache_coherency = intel_iommu_enforce_cache_coherency,
4500 }
4501};
4502
4503static void quirk_iommu_igfx(struct pci_dev *dev)
4504{
4505 if (risky_device(dev))
4506 return;
4507
4508 pci_info(dev, "Disabling IOMMU for graphics on this chipset\n");
4509 disable_igfx_iommu = 1;
4510}
4511
4512/* G4x/GM45 integrated gfx dmar support is totally busted. */
4513DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx);
4514DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx);
4515DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_igfx);
4516DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_igfx);
4517DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_igfx);
4518DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_igfx);
4519DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_igfx);
4520
4521/* Broadwell igfx malfunctions with dmar */
4522DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1606, quirk_iommu_igfx);
4523DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160B, quirk_iommu_igfx);
4524DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160E, quirk_iommu_igfx);
4525DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1602, quirk_iommu_igfx);
4526DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160A, quirk_iommu_igfx);
4527DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160D, quirk_iommu_igfx);
4528DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1616, quirk_iommu_igfx);
4529DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161B, quirk_iommu_igfx);
4530DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161E, quirk_iommu_igfx);
4531DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1612, quirk_iommu_igfx);
4532DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161A, quirk_iommu_igfx);
4533DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161D, quirk_iommu_igfx);
4534DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1626, quirk_iommu_igfx);
4535DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162B, quirk_iommu_igfx);
4536DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162E, quirk_iommu_igfx);
4537DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1622, quirk_iommu_igfx);
4538DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162A, quirk_iommu_igfx);
4539DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162D, quirk_iommu_igfx);
4540DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1636, quirk_iommu_igfx);
4541DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163B, quirk_iommu_igfx);
4542DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163E, quirk_iommu_igfx);
4543DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1632, quirk_iommu_igfx);
4544DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163A, quirk_iommu_igfx);
4545DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163D, quirk_iommu_igfx);
4546
4547static void quirk_iommu_rwbf(struct pci_dev *dev)
4548{
4549 if (risky_device(dev))
4550 return;
4551
4552 /*
4553 * Mobile 4 Series Chipset neglects to set RWBF capability,
4554 * but needs it. Same seems to hold for the desktop versions.
4555 */
4556 pci_info(dev, "Forcing write-buffer flush capability\n");
4557 rwbf_quirk = 1;
4558}
4559
4560DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4561DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4562DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4563DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4564DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4565DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4566DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4567
4568#define GGC 0x52
4569#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4570#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4571#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4572#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4573#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4574#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4575#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4576#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4577
4578static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4579{
4580 unsigned short ggc;
4581
4582 if (risky_device(dev))
4583 return;
4584
4585 if (pci_read_config_word(dev, GGC, &ggc))
4586 return;
4587
4588 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4589 pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4590 disable_igfx_iommu = 1;
4591 } else if (!disable_igfx_iommu) {
4592 /* we have to ensure the gfx device is idle before we flush */
4593 pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n");
4594 iommu_set_dma_strict();
4595 }
4596}
4597DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4598DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4599DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4600DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4601
4602static void quirk_igfx_skip_te_disable(struct pci_dev *dev)
4603{
4604 unsigned short ver;
4605
4606 if (!IS_GFX_DEVICE(dev))
4607 return;
4608
4609 ver = (dev->device >> 8) & 0xff;
4610 if (ver != 0x45 && ver != 0x46 && ver != 0x4c &&
4611 ver != 0x4e && ver != 0x8a && ver != 0x98 &&
4612 ver != 0x9a && ver != 0xa7 && ver != 0x7d)
4613 return;
4614
4615 if (risky_device(dev))
4616 return;
4617
4618 pci_info(dev, "Skip IOMMU disabling for graphics\n");
4619 iommu_skip_te_disable = 1;
4620}
4621DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, quirk_igfx_skip_te_disable);
4622
4623/* On Tylersburg chipsets, some BIOSes have been known to enable the
4624 ISOCH DMAR unit for the Azalia sound device, but not give it any
4625 TLB entries, which causes it to deadlock. Check for that. We do
4626 this in a function called from init_dmars(), instead of in a PCI
4627 quirk, because we don't want to print the obnoxious "BIOS broken"
4628 message if VT-d is actually disabled.
4629*/
4630static void __init check_tylersburg_isoch(void)
4631{
4632 struct pci_dev *pdev;
4633 uint32_t vtisochctrl;
4634
4635 /* If there's no Azalia in the system anyway, forget it. */
4636 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4637 if (!pdev)
4638 return;
4639
4640 if (risky_device(pdev)) {
4641 pci_dev_put(pdev);
4642 return;
4643 }
4644
4645 pci_dev_put(pdev);
4646
4647 /* System Management Registers. Might be hidden, in which case
4648 we can't do the sanity check. But that's OK, because the
4649 known-broken BIOSes _don't_ actually hide it, so far. */
4650 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4651 if (!pdev)
4652 return;
4653
4654 if (risky_device(pdev)) {
4655 pci_dev_put(pdev);
4656 return;
4657 }
4658
4659 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4660 pci_dev_put(pdev);
4661 return;
4662 }
4663
4664 pci_dev_put(pdev);
4665
4666 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4667 if (vtisochctrl & 1)
4668 return;
4669
4670 /* Drop all bits other than the number of TLB entries */
4671 vtisochctrl &= 0x1c;
4672
4673 /* If we have the recommended number of TLB entries (16), fine. */
4674 if (vtisochctrl == 0x10)
4675 return;
4676
4677 /* Zero TLB entries? You get to ride the short bus to school. */
4678 if (!vtisochctrl) {
4679 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4680 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4681 dmi_get_system_info(DMI_BIOS_VENDOR),
4682 dmi_get_system_info(DMI_BIOS_VERSION),
4683 dmi_get_system_info(DMI_PRODUCT_VERSION));
4684 iommu_identity_mapping |= IDENTMAP_AZALIA;
4685 return;
4686 }
4687
4688 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4689 vtisochctrl);
4690}
4691
4692/*
4693 * Here we deal with a device TLB defect where device may inadvertently issue ATS
4694 * invalidation completion before posted writes initiated with translated address
4695 * that utilized translations matching the invalidation address range, violating
4696 * the invalidation completion ordering.
4697 * Therefore, any use cases that cannot guarantee DMA is stopped before unmap is
4698 * vulnerable to this defect. In other words, any dTLB invalidation initiated not
4699 * under the control of the trusted/privileged host device driver must use this
4700 * quirk.
4701 * Device TLBs are invalidated under the following six conditions:
4702 * 1. Device driver does DMA API unmap IOVA
4703 * 2. Device driver unbind a PASID from a process, sva_unbind_device()
4704 * 3. PASID is torn down, after PASID cache is flushed. e.g. process
4705 * exit_mmap() due to crash
4706 * 4. Under SVA usage, called by mmu_notifier.invalidate_range() where
4707 * VM has to free pages that were unmapped
4708 * 5. Userspace driver unmaps a DMA buffer
4709 * 6. Cache invalidation in vSVA usage (upcoming)
4710 *
4711 * For #1 and #2, device drivers are responsible for stopping DMA traffic
4712 * before unmap/unbind. For #3, iommu driver gets mmu_notifier to
4713 * invalidate TLB the same way as normal user unmap which will use this quirk.
4714 * The dTLB invalidation after PASID cache flush does not need this quirk.
4715 *
4716 * As a reminder, #6 will *NEED* this quirk as we enable nested translation.
4717 */
4718void quirk_extra_dev_tlb_flush(struct device_domain_info *info,
4719 unsigned long address, unsigned long mask,
4720 u32 pasid, u16 qdep)
4721{
4722 u16 sid;
4723
4724 if (likely(!info->dtlb_extra_inval))
4725 return;
4726
4727 sid = PCI_DEVID(info->bus, info->devfn);
4728 if (pasid == IOMMU_NO_PASID) {
4729 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
4730 qdep, address, mask);
4731 } else {
4732 qi_flush_dev_iotlb_pasid(info->iommu, sid, info->pfsid,
4733 pasid, qdep, address, mask);
4734 }
4735}
4736
4737#define ecmd_get_status_code(res) (((res) & 0xff) >> 1)
4738
4739/*
4740 * Function to submit a command to the enhanced command interface. The
4741 * valid enhanced command descriptions are defined in Table 47 of the
4742 * VT-d spec. The VT-d hardware implementation may support some but not
4743 * all commands, which can be determined by checking the Enhanced
4744 * Command Capability Register.
4745 *
4746 * Return values:
4747 * - 0: Command successful without any error;
4748 * - Negative: software error value;
4749 * - Nonzero positive: failure status code defined in Table 48.
4750 */
4751int ecmd_submit_sync(struct intel_iommu *iommu, u8 ecmd, u64 oa, u64 ob)
4752{
4753 unsigned long flags;
4754 u64 res;
4755 int ret;
4756
4757 if (!cap_ecmds(iommu->cap))
4758 return -ENODEV;
4759
4760 raw_spin_lock_irqsave(&iommu->register_lock, flags);
4761
4762 res = dmar_readq(iommu->reg + DMAR_ECRSP_REG);
4763 if (res & DMA_ECMD_ECRSP_IP) {
4764 ret = -EBUSY;
4765 goto err;
4766 }
4767
4768 /*
4769 * Unconditionally write the operand B, because
4770 * - There is no side effect if an ecmd doesn't require an
4771 * operand B, but we set the register to some value.
4772 * - It's not invoked in any critical path. The extra MMIO
4773 * write doesn't bring any performance concerns.
4774 */
4775 dmar_writeq(iommu->reg + DMAR_ECEO_REG, ob);
4776 dmar_writeq(iommu->reg + DMAR_ECMD_REG, ecmd | (oa << DMA_ECMD_OA_SHIFT));
4777
4778 IOMMU_WAIT_OP(iommu, DMAR_ECRSP_REG, dmar_readq,
4779 !(res & DMA_ECMD_ECRSP_IP), res);
4780
4781 if (res & DMA_ECMD_ECRSP_IP) {
4782 ret = -ETIMEDOUT;
4783 goto err;
4784 }
4785
4786 ret = ecmd_get_status_code(res);
4787err:
4788 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
4789
4790 return ret;
4791}
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright © 2006-2014 Intel Corporation.
4 *
5 * Authors: David Woodhouse <dwmw2@infradead.org>,
6 * Ashok Raj <ashok.raj@intel.com>,
7 * Shaohua Li <shaohua.li@intel.com>,
8 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
9 * Fenghua Yu <fenghua.yu@intel.com>
10 * Joerg Roedel <jroedel@suse.de>
11 */
12
13#define pr_fmt(fmt) "DMAR: " fmt
14#define dev_fmt(fmt) pr_fmt(fmt)
15
16#include <linux/crash_dump.h>
17#include <linux/dma-direct.h>
18#include <linux/dmi.h>
19#include <linux/intel-svm.h>
20#include <linux/memory.h>
21#include <linux/pci.h>
22#include <linux/pci-ats.h>
23#include <linux/spinlock.h>
24#include <linux/syscore_ops.h>
25#include <linux/tboot.h>
26
27#include "iommu.h"
28#include "../dma-iommu.h"
29#include "../irq_remapping.h"
30#include "../iommu-sva.h"
31#include "pasid.h"
32#include "cap_audit.h"
33
34#define ROOT_SIZE VTD_PAGE_SIZE
35#define CONTEXT_SIZE VTD_PAGE_SIZE
36
37#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
38#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
39#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
40#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
41
42#define IOAPIC_RANGE_START (0xfee00000)
43#define IOAPIC_RANGE_END (0xfeefffff)
44#define IOVA_START_ADDR (0x1000)
45
46#define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
47
48#define MAX_AGAW_WIDTH 64
49#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
50
51#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << ((gaw) - VTD_PAGE_SHIFT)) - 1)
52#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << (gaw)) - 1)
53
54/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
55 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
56#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
57 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
58#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
59
60/* IO virtual address start page frame number */
61#define IOVA_START_PFN (1)
62
63#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
64
65/* page table handling */
66#define LEVEL_STRIDE (9)
67#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
68
69static inline int agaw_to_level(int agaw)
70{
71 return agaw + 2;
72}
73
74static inline int agaw_to_width(int agaw)
75{
76 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
77}
78
79static inline int width_to_agaw(int width)
80{
81 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
82}
83
84static inline unsigned int level_to_offset_bits(int level)
85{
86 return (level - 1) * LEVEL_STRIDE;
87}
88
89static inline int pfn_level_offset(u64 pfn, int level)
90{
91 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
92}
93
94static inline u64 level_mask(int level)
95{
96 return -1ULL << level_to_offset_bits(level);
97}
98
99static inline u64 level_size(int level)
100{
101 return 1ULL << level_to_offset_bits(level);
102}
103
104static inline u64 align_to_level(u64 pfn, int level)
105{
106 return (pfn + level_size(level) - 1) & level_mask(level);
107}
108
109static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
110{
111 return 1UL << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
112}
113
114/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
115 are never going to work. */
116static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
117{
118 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
119}
120static inline unsigned long page_to_dma_pfn(struct page *pg)
121{
122 return mm_to_dma_pfn(page_to_pfn(pg));
123}
124static inline unsigned long virt_to_dma_pfn(void *p)
125{
126 return page_to_dma_pfn(virt_to_page(p));
127}
128
129static void __init check_tylersburg_isoch(void);
130static int rwbf_quirk;
131
132/*
133 * set to 1 to panic kernel if can't successfully enable VT-d
134 * (used when kernel is launched w/ TXT)
135 */
136static int force_on = 0;
137static int intel_iommu_tboot_noforce;
138static int no_platform_optin;
139
140#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
141
142/*
143 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
144 * if marked present.
145 */
146static phys_addr_t root_entry_lctp(struct root_entry *re)
147{
148 if (!(re->lo & 1))
149 return 0;
150
151 return re->lo & VTD_PAGE_MASK;
152}
153
154/*
155 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
156 * if marked present.
157 */
158static phys_addr_t root_entry_uctp(struct root_entry *re)
159{
160 if (!(re->hi & 1))
161 return 0;
162
163 return re->hi & VTD_PAGE_MASK;
164}
165
166static inline void context_set_present(struct context_entry *context)
167{
168 context->lo |= 1;
169}
170
171static inline void context_set_fault_enable(struct context_entry *context)
172{
173 context->lo &= (((u64)-1) << 2) | 1;
174}
175
176static inline void context_set_translation_type(struct context_entry *context,
177 unsigned long value)
178{
179 context->lo &= (((u64)-1) << 4) | 3;
180 context->lo |= (value & 3) << 2;
181}
182
183static inline void context_set_address_root(struct context_entry *context,
184 unsigned long value)
185{
186 context->lo &= ~VTD_PAGE_MASK;
187 context->lo |= value & VTD_PAGE_MASK;
188}
189
190static inline void context_set_address_width(struct context_entry *context,
191 unsigned long value)
192{
193 context->hi |= value & 7;
194}
195
196static inline void context_set_domain_id(struct context_entry *context,
197 unsigned long value)
198{
199 context->hi |= (value & ((1 << 16) - 1)) << 8;
200}
201
202static inline void context_set_pasid(struct context_entry *context)
203{
204 context->lo |= CONTEXT_PASIDE;
205}
206
207static inline int context_domain_id(struct context_entry *c)
208{
209 return((c->hi >> 8) & 0xffff);
210}
211
212static inline void context_clear_entry(struct context_entry *context)
213{
214 context->lo = 0;
215 context->hi = 0;
216}
217
218static inline bool context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
219{
220 if (!iommu->copied_tables)
221 return false;
222
223 return test_bit(((long)bus << 8) | devfn, iommu->copied_tables);
224}
225
226static inline void
227set_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
228{
229 set_bit(((long)bus << 8) | devfn, iommu->copied_tables);
230}
231
232static inline void
233clear_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
234{
235 clear_bit(((long)bus << 8) | devfn, iommu->copied_tables);
236}
237
238/*
239 * This domain is a statically identity mapping domain.
240 * 1. This domain creats a static 1:1 mapping to all usable memory.
241 * 2. It maps to each iommu if successful.
242 * 3. Each iommu mapps to this domain if successful.
243 */
244static struct dmar_domain *si_domain;
245static int hw_pass_through = 1;
246
247struct dmar_rmrr_unit {
248 struct list_head list; /* list of rmrr units */
249 struct acpi_dmar_header *hdr; /* ACPI header */
250 u64 base_address; /* reserved base address*/
251 u64 end_address; /* reserved end address */
252 struct dmar_dev_scope *devices; /* target devices */
253 int devices_cnt; /* target device count */
254};
255
256struct dmar_atsr_unit {
257 struct list_head list; /* list of ATSR units */
258 struct acpi_dmar_header *hdr; /* ACPI header */
259 struct dmar_dev_scope *devices; /* target devices */
260 int devices_cnt; /* target device count */
261 u8 include_all:1; /* include all ports */
262};
263
264struct dmar_satc_unit {
265 struct list_head list; /* list of SATC units */
266 struct acpi_dmar_header *hdr; /* ACPI header */
267 struct dmar_dev_scope *devices; /* target devices */
268 struct intel_iommu *iommu; /* the corresponding iommu */
269 int devices_cnt; /* target device count */
270 u8 atc_required:1; /* ATS is required */
271};
272
273static LIST_HEAD(dmar_atsr_units);
274static LIST_HEAD(dmar_rmrr_units);
275static LIST_HEAD(dmar_satc_units);
276
277#define for_each_rmrr_units(rmrr) \
278 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
279
280static void device_block_translation(struct device *dev);
281static void intel_iommu_domain_free(struct iommu_domain *domain);
282
283int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);
284int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON);
285
286int intel_iommu_enabled = 0;
287EXPORT_SYMBOL_GPL(intel_iommu_enabled);
288
289static int dmar_map_gfx = 1;
290static int intel_iommu_superpage = 1;
291static int iommu_identity_mapping;
292static int iommu_skip_te_disable;
293
294#define IDENTMAP_GFX 2
295#define IDENTMAP_AZALIA 4
296
297const struct iommu_ops intel_iommu_ops;
298
299static bool translation_pre_enabled(struct intel_iommu *iommu)
300{
301 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
302}
303
304static void clear_translation_pre_enabled(struct intel_iommu *iommu)
305{
306 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
307}
308
309static void init_translation_status(struct intel_iommu *iommu)
310{
311 u32 gsts;
312
313 gsts = readl(iommu->reg + DMAR_GSTS_REG);
314 if (gsts & DMA_GSTS_TES)
315 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
316}
317
318static int __init intel_iommu_setup(char *str)
319{
320 if (!str)
321 return -EINVAL;
322
323 while (*str) {
324 if (!strncmp(str, "on", 2)) {
325 dmar_disabled = 0;
326 pr_info("IOMMU enabled\n");
327 } else if (!strncmp(str, "off", 3)) {
328 dmar_disabled = 1;
329 no_platform_optin = 1;
330 pr_info("IOMMU disabled\n");
331 } else if (!strncmp(str, "igfx_off", 8)) {
332 dmar_map_gfx = 0;
333 pr_info("Disable GFX device mapping\n");
334 } else if (!strncmp(str, "forcedac", 8)) {
335 pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n");
336 iommu_dma_forcedac = true;
337 } else if (!strncmp(str, "strict", 6)) {
338 pr_warn("intel_iommu=strict deprecated; use iommu.strict=1 instead\n");
339 iommu_set_dma_strict();
340 } else if (!strncmp(str, "sp_off", 6)) {
341 pr_info("Disable supported super page\n");
342 intel_iommu_superpage = 0;
343 } else if (!strncmp(str, "sm_on", 5)) {
344 pr_info("Enable scalable mode if hardware supports\n");
345 intel_iommu_sm = 1;
346 } else if (!strncmp(str, "sm_off", 6)) {
347 pr_info("Scalable mode is disallowed\n");
348 intel_iommu_sm = 0;
349 } else if (!strncmp(str, "tboot_noforce", 13)) {
350 pr_info("Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
351 intel_iommu_tboot_noforce = 1;
352 } else {
353 pr_notice("Unknown option - '%s'\n", str);
354 }
355
356 str += strcspn(str, ",");
357 while (*str == ',')
358 str++;
359 }
360
361 return 1;
362}
363__setup("intel_iommu=", intel_iommu_setup);
364
365void *alloc_pgtable_page(int node)
366{
367 struct page *page;
368 void *vaddr = NULL;
369
370 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
371 if (page)
372 vaddr = page_address(page);
373 return vaddr;
374}
375
376void free_pgtable_page(void *vaddr)
377{
378 free_page((unsigned long)vaddr);
379}
380
381static inline int domain_type_is_si(struct dmar_domain *domain)
382{
383 return domain->domain.type == IOMMU_DOMAIN_IDENTITY;
384}
385
386static inline int domain_pfn_supported(struct dmar_domain *domain,
387 unsigned long pfn)
388{
389 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
390
391 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
392}
393
394/*
395 * Calculate the Supported Adjusted Guest Address Widths of an IOMMU.
396 * Refer to 11.4.2 of the VT-d spec for the encoding of each bit of
397 * the returned SAGAW.
398 */
399static unsigned long __iommu_calculate_sagaw(struct intel_iommu *iommu)
400{
401 unsigned long fl_sagaw, sl_sagaw;
402
403 fl_sagaw = BIT(2) | (cap_fl5lp_support(iommu->cap) ? BIT(3) : 0);
404 sl_sagaw = cap_sagaw(iommu->cap);
405
406 /* Second level only. */
407 if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))
408 return sl_sagaw;
409
410 /* First level only. */
411 if (!ecap_slts(iommu->ecap))
412 return fl_sagaw;
413
414 return fl_sagaw & sl_sagaw;
415}
416
417static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
418{
419 unsigned long sagaw;
420 int agaw;
421
422 sagaw = __iommu_calculate_sagaw(iommu);
423 for (agaw = width_to_agaw(max_gaw); agaw >= 0; agaw--) {
424 if (test_bit(agaw, &sagaw))
425 break;
426 }
427
428 return agaw;
429}
430
431/*
432 * Calculate max SAGAW for each iommu.
433 */
434int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
435{
436 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
437}
438
439/*
440 * calculate agaw for each iommu.
441 * "SAGAW" may be different across iommus, use a default agaw, and
442 * get a supported less agaw for iommus that don't support the default agaw.
443 */
444int iommu_calculate_agaw(struct intel_iommu *iommu)
445{
446 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
447}
448
449static inline bool iommu_paging_structure_coherency(struct intel_iommu *iommu)
450{
451 return sm_supported(iommu) ?
452 ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap);
453}
454
455static void domain_update_iommu_coherency(struct dmar_domain *domain)
456{
457 struct iommu_domain_info *info;
458 struct dmar_drhd_unit *drhd;
459 struct intel_iommu *iommu;
460 bool found = false;
461 unsigned long i;
462
463 domain->iommu_coherency = true;
464 xa_for_each(&domain->iommu_array, i, info) {
465 found = true;
466 if (!iommu_paging_structure_coherency(info->iommu)) {
467 domain->iommu_coherency = false;
468 break;
469 }
470 }
471 if (found)
472 return;
473
474 /* No hardware attached; use lowest common denominator */
475 rcu_read_lock();
476 for_each_active_iommu(iommu, drhd) {
477 if (!iommu_paging_structure_coherency(iommu)) {
478 domain->iommu_coherency = false;
479 break;
480 }
481 }
482 rcu_read_unlock();
483}
484
485static int domain_update_iommu_superpage(struct dmar_domain *domain,
486 struct intel_iommu *skip)
487{
488 struct dmar_drhd_unit *drhd;
489 struct intel_iommu *iommu;
490 int mask = 0x3;
491
492 if (!intel_iommu_superpage)
493 return 0;
494
495 /* set iommu_superpage to the smallest common denominator */
496 rcu_read_lock();
497 for_each_active_iommu(iommu, drhd) {
498 if (iommu != skip) {
499 if (domain && domain->use_first_level) {
500 if (!cap_fl1gp_support(iommu->cap))
501 mask = 0x1;
502 } else {
503 mask &= cap_super_page_val(iommu->cap);
504 }
505
506 if (!mask)
507 break;
508 }
509 }
510 rcu_read_unlock();
511
512 return fls(mask);
513}
514
515static int domain_update_device_node(struct dmar_domain *domain)
516{
517 struct device_domain_info *info;
518 int nid = NUMA_NO_NODE;
519 unsigned long flags;
520
521 spin_lock_irqsave(&domain->lock, flags);
522 list_for_each_entry(info, &domain->devices, link) {
523 /*
524 * There could possibly be multiple device numa nodes as devices
525 * within the same domain may sit behind different IOMMUs. There
526 * isn't perfect answer in such situation, so we select first
527 * come first served policy.
528 */
529 nid = dev_to_node(info->dev);
530 if (nid != NUMA_NO_NODE)
531 break;
532 }
533 spin_unlock_irqrestore(&domain->lock, flags);
534
535 return nid;
536}
537
538static void domain_update_iotlb(struct dmar_domain *domain);
539
540/* Return the super pagesize bitmap if supported. */
541static unsigned long domain_super_pgsize_bitmap(struct dmar_domain *domain)
542{
543 unsigned long bitmap = 0;
544
545 /*
546 * 1-level super page supports page size of 2MiB, 2-level super page
547 * supports page size of both 2MiB and 1GiB.
548 */
549 if (domain->iommu_superpage == 1)
550 bitmap |= SZ_2M;
551 else if (domain->iommu_superpage == 2)
552 bitmap |= SZ_2M | SZ_1G;
553
554 return bitmap;
555}
556
557/* Some capabilities may be different across iommus */
558static void domain_update_iommu_cap(struct dmar_domain *domain)
559{
560 domain_update_iommu_coherency(domain);
561 domain->iommu_superpage = domain_update_iommu_superpage(domain, NULL);
562
563 /*
564 * If RHSA is missing, we should default to the device numa domain
565 * as fall back.
566 */
567 if (domain->nid == NUMA_NO_NODE)
568 domain->nid = domain_update_device_node(domain);
569
570 /*
571 * First-level translation restricts the input-address to a
572 * canonical address (i.e., address bits 63:N have the same
573 * value as address bit [N-1], where N is 48-bits with 4-level
574 * paging and 57-bits with 5-level paging). Hence, skip bit
575 * [N-1].
576 */
577 if (domain->use_first_level)
578 domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1);
579 else
580 domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw);
581
582 domain->domain.pgsize_bitmap |= domain_super_pgsize_bitmap(domain);
583 domain_update_iotlb(domain);
584}
585
586struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
587 u8 devfn, int alloc)
588{
589 struct root_entry *root = &iommu->root_entry[bus];
590 struct context_entry *context;
591 u64 *entry;
592
593 /*
594 * Except that the caller requested to allocate a new entry,
595 * returning a copied context entry makes no sense.
596 */
597 if (!alloc && context_copied(iommu, bus, devfn))
598 return NULL;
599
600 entry = &root->lo;
601 if (sm_supported(iommu)) {
602 if (devfn >= 0x80) {
603 devfn -= 0x80;
604 entry = &root->hi;
605 }
606 devfn *= 2;
607 }
608 if (*entry & 1)
609 context = phys_to_virt(*entry & VTD_PAGE_MASK);
610 else {
611 unsigned long phy_addr;
612 if (!alloc)
613 return NULL;
614
615 context = alloc_pgtable_page(iommu->node);
616 if (!context)
617 return NULL;
618
619 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
620 phy_addr = virt_to_phys((void *)context);
621 *entry = phy_addr | 1;
622 __iommu_flush_cache(iommu, entry, sizeof(*entry));
623 }
624 return &context[devfn];
625}
626
627/**
628 * is_downstream_to_pci_bridge - test if a device belongs to the PCI
629 * sub-hierarchy of a candidate PCI-PCI bridge
630 * @dev: candidate PCI device belonging to @bridge PCI sub-hierarchy
631 * @bridge: the candidate PCI-PCI bridge
632 *
633 * Return: true if @dev belongs to @bridge PCI sub-hierarchy, else false.
634 */
635static bool
636is_downstream_to_pci_bridge(struct device *dev, struct device *bridge)
637{
638 struct pci_dev *pdev, *pbridge;
639
640 if (!dev_is_pci(dev) || !dev_is_pci(bridge))
641 return false;
642
643 pdev = to_pci_dev(dev);
644 pbridge = to_pci_dev(bridge);
645
646 if (pbridge->subordinate &&
647 pbridge->subordinate->number <= pdev->bus->number &&
648 pbridge->subordinate->busn_res.end >= pdev->bus->number)
649 return true;
650
651 return false;
652}
653
654static bool quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
655{
656 struct dmar_drhd_unit *drhd;
657 u32 vtbar;
658 int rc;
659
660 /* We know that this device on this chipset has its own IOMMU.
661 * If we find it under a different IOMMU, then the BIOS is lying
662 * to us. Hope that the IOMMU for this device is actually
663 * disabled, and it needs no translation...
664 */
665 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
666 if (rc) {
667 /* "can't" happen */
668 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
669 return false;
670 }
671 vtbar &= 0xffff0000;
672
673 /* we know that the this iommu should be at offset 0xa000 from vtbar */
674 drhd = dmar_find_matched_drhd_unit(pdev);
675 if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
676 pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
677 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
678 return true;
679 }
680
681 return false;
682}
683
684static bool iommu_is_dummy(struct intel_iommu *iommu, struct device *dev)
685{
686 if (!iommu || iommu->drhd->ignored)
687 return true;
688
689 if (dev_is_pci(dev)) {
690 struct pci_dev *pdev = to_pci_dev(dev);
691
692 if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
693 pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SNB &&
694 quirk_ioat_snb_local_iommu(pdev))
695 return true;
696 }
697
698 return false;
699}
700
701struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
702{
703 struct dmar_drhd_unit *drhd = NULL;
704 struct pci_dev *pdev = NULL;
705 struct intel_iommu *iommu;
706 struct device *tmp;
707 u16 segment = 0;
708 int i;
709
710 if (!dev)
711 return NULL;
712
713 if (dev_is_pci(dev)) {
714 struct pci_dev *pf_pdev;
715
716 pdev = pci_real_dma_dev(to_pci_dev(dev));
717
718 /* VFs aren't listed in scope tables; we need to look up
719 * the PF instead to find the IOMMU. */
720 pf_pdev = pci_physfn(pdev);
721 dev = &pf_pdev->dev;
722 segment = pci_domain_nr(pdev->bus);
723 } else if (has_acpi_companion(dev))
724 dev = &ACPI_COMPANION(dev)->dev;
725
726 rcu_read_lock();
727 for_each_iommu(iommu, drhd) {
728 if (pdev && segment != drhd->segment)
729 continue;
730
731 for_each_active_dev_scope(drhd->devices,
732 drhd->devices_cnt, i, tmp) {
733 if (tmp == dev) {
734 /* For a VF use its original BDF# not that of the PF
735 * which we used for the IOMMU lookup. Strictly speaking
736 * we could do this for all PCI devices; we only need to
737 * get the BDF# from the scope table for ACPI matches. */
738 if (pdev && pdev->is_virtfn)
739 goto got_pdev;
740
741 if (bus && devfn) {
742 *bus = drhd->devices[i].bus;
743 *devfn = drhd->devices[i].devfn;
744 }
745 goto out;
746 }
747
748 if (is_downstream_to_pci_bridge(dev, tmp))
749 goto got_pdev;
750 }
751
752 if (pdev && drhd->include_all) {
753got_pdev:
754 if (bus && devfn) {
755 *bus = pdev->bus->number;
756 *devfn = pdev->devfn;
757 }
758 goto out;
759 }
760 }
761 iommu = NULL;
762out:
763 if (iommu_is_dummy(iommu, dev))
764 iommu = NULL;
765
766 rcu_read_unlock();
767
768 return iommu;
769}
770
771static void domain_flush_cache(struct dmar_domain *domain,
772 void *addr, int size)
773{
774 if (!domain->iommu_coherency)
775 clflush_cache_range(addr, size);
776}
777
778static void free_context_table(struct intel_iommu *iommu)
779{
780 struct context_entry *context;
781 int i;
782
783 if (!iommu->root_entry)
784 return;
785
786 for (i = 0; i < ROOT_ENTRY_NR; i++) {
787 context = iommu_context_addr(iommu, i, 0, 0);
788 if (context)
789 free_pgtable_page(context);
790
791 if (!sm_supported(iommu))
792 continue;
793
794 context = iommu_context_addr(iommu, i, 0x80, 0);
795 if (context)
796 free_pgtable_page(context);
797 }
798
799 free_pgtable_page(iommu->root_entry);
800 iommu->root_entry = NULL;
801}
802
803#ifdef CONFIG_DMAR_DEBUG
804static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn,
805 u8 bus, u8 devfn, struct dma_pte *parent, int level)
806{
807 struct dma_pte *pte;
808 int offset;
809
810 while (1) {
811 offset = pfn_level_offset(pfn, level);
812 pte = &parent[offset];
813 if (!pte || (dma_pte_superpage(pte) || !dma_pte_present(pte))) {
814 pr_info("PTE not present at level %d\n", level);
815 break;
816 }
817
818 pr_info("pte level: %d, pte value: 0x%016llx\n", level, pte->val);
819
820 if (level == 1)
821 break;
822
823 parent = phys_to_virt(dma_pte_addr(pte));
824 level--;
825 }
826}
827
828void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,
829 unsigned long long addr, u32 pasid)
830{
831 struct pasid_dir_entry *dir, *pde;
832 struct pasid_entry *entries, *pte;
833 struct context_entry *ctx_entry;
834 struct root_entry *rt_entry;
835 int i, dir_index, index, level;
836 u8 devfn = source_id & 0xff;
837 u8 bus = source_id >> 8;
838 struct dma_pte *pgtable;
839
840 pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr);
841
842 /* root entry dump */
843 rt_entry = &iommu->root_entry[bus];
844 if (!rt_entry) {
845 pr_info("root table entry is not present\n");
846 return;
847 }
848
849 if (sm_supported(iommu))
850 pr_info("scalable mode root entry: hi 0x%016llx, low 0x%016llx\n",
851 rt_entry->hi, rt_entry->lo);
852 else
853 pr_info("root entry: 0x%016llx", rt_entry->lo);
854
855 /* context entry dump */
856 ctx_entry = iommu_context_addr(iommu, bus, devfn, 0);
857 if (!ctx_entry) {
858 pr_info("context table entry is not present\n");
859 return;
860 }
861
862 pr_info("context entry: hi 0x%016llx, low 0x%016llx\n",
863 ctx_entry->hi, ctx_entry->lo);
864
865 /* legacy mode does not require PASID entries */
866 if (!sm_supported(iommu)) {
867 level = agaw_to_level(ctx_entry->hi & 7);
868 pgtable = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
869 goto pgtable_walk;
870 }
871
872 /* get the pointer to pasid directory entry */
873 dir = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
874 if (!dir) {
875 pr_info("pasid directory entry is not present\n");
876 return;
877 }
878 /* For request-without-pasid, get the pasid from context entry */
879 if (intel_iommu_sm && pasid == INVALID_IOASID)
880 pasid = PASID_RID2PASID;
881
882 dir_index = pasid >> PASID_PDE_SHIFT;
883 pde = &dir[dir_index];
884 pr_info("pasid dir entry: 0x%016llx\n", pde->val);
885
886 /* get the pointer to the pasid table entry */
887 entries = get_pasid_table_from_pde(pde);
888 if (!entries) {
889 pr_info("pasid table entry is not present\n");
890 return;
891 }
892 index = pasid & PASID_PTE_MASK;
893 pte = &entries[index];
894 for (i = 0; i < ARRAY_SIZE(pte->val); i++)
895 pr_info("pasid table entry[%d]: 0x%016llx\n", i, pte->val[i]);
896
897 if (pasid_pte_get_pgtt(pte) == PASID_ENTRY_PGTT_FL_ONLY) {
898 level = pte->val[2] & BIT_ULL(2) ? 5 : 4;
899 pgtable = phys_to_virt(pte->val[2] & VTD_PAGE_MASK);
900 } else {
901 level = agaw_to_level((pte->val[0] >> 2) & 0x7);
902 pgtable = phys_to_virt(pte->val[0] & VTD_PAGE_MASK);
903 }
904
905pgtable_walk:
906 pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn, pgtable, level);
907}
908#endif
909
910static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
911 unsigned long pfn, int *target_level)
912{
913 struct dma_pte *parent, *pte;
914 int level = agaw_to_level(domain->agaw);
915 int offset;
916
917 BUG_ON(!domain->pgd);
918
919 if (!domain_pfn_supported(domain, pfn))
920 /* Address beyond IOMMU's addressing capabilities. */
921 return NULL;
922
923 parent = domain->pgd;
924
925 while (1) {
926 void *tmp_page;
927
928 offset = pfn_level_offset(pfn, level);
929 pte = &parent[offset];
930 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
931 break;
932 if (level == *target_level)
933 break;
934
935 if (!dma_pte_present(pte)) {
936 uint64_t pteval;
937
938 tmp_page = alloc_pgtable_page(domain->nid);
939
940 if (!tmp_page)
941 return NULL;
942
943 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
944 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
945 if (domain->use_first_level)
946 pteval |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
947
948 if (cmpxchg64(&pte->val, 0ULL, pteval))
949 /* Someone else set it while we were thinking; use theirs. */
950 free_pgtable_page(tmp_page);
951 else
952 domain_flush_cache(domain, pte, sizeof(*pte));
953 }
954 if (level == 1)
955 break;
956
957 parent = phys_to_virt(dma_pte_addr(pte));
958 level--;
959 }
960
961 if (!*target_level)
962 *target_level = level;
963
964 return pte;
965}
966
967/* return address's pte at specific level */
968static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
969 unsigned long pfn,
970 int level, int *large_page)
971{
972 struct dma_pte *parent, *pte;
973 int total = agaw_to_level(domain->agaw);
974 int offset;
975
976 parent = domain->pgd;
977 while (level <= total) {
978 offset = pfn_level_offset(pfn, total);
979 pte = &parent[offset];
980 if (level == total)
981 return pte;
982
983 if (!dma_pte_present(pte)) {
984 *large_page = total;
985 break;
986 }
987
988 if (dma_pte_superpage(pte)) {
989 *large_page = total;
990 return pte;
991 }
992
993 parent = phys_to_virt(dma_pte_addr(pte));
994 total--;
995 }
996 return NULL;
997}
998
999/* clear last level pte, a tlb flush should be followed */
1000static void dma_pte_clear_range(struct dmar_domain *domain,
1001 unsigned long start_pfn,
1002 unsigned long last_pfn)
1003{
1004 unsigned int large_page;
1005 struct dma_pte *first_pte, *pte;
1006
1007 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1008 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1009 BUG_ON(start_pfn > last_pfn);
1010
1011 /* we don't need lock here; nobody else touches the iova range */
1012 do {
1013 large_page = 1;
1014 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1015 if (!pte) {
1016 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1017 continue;
1018 }
1019 do {
1020 dma_clear_pte(pte);
1021 start_pfn += lvl_to_nr_pages(large_page);
1022 pte++;
1023 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1024
1025 domain_flush_cache(domain, first_pte,
1026 (void *)pte - (void *)first_pte);
1027
1028 } while (start_pfn && start_pfn <= last_pfn);
1029}
1030
1031static void dma_pte_free_level(struct dmar_domain *domain, int level,
1032 int retain_level, struct dma_pte *pte,
1033 unsigned long pfn, unsigned long start_pfn,
1034 unsigned long last_pfn)
1035{
1036 pfn = max(start_pfn, pfn);
1037 pte = &pte[pfn_level_offset(pfn, level)];
1038
1039 do {
1040 unsigned long level_pfn;
1041 struct dma_pte *level_pte;
1042
1043 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1044 goto next;
1045
1046 level_pfn = pfn & level_mask(level);
1047 level_pte = phys_to_virt(dma_pte_addr(pte));
1048
1049 if (level > 2) {
1050 dma_pte_free_level(domain, level - 1, retain_level,
1051 level_pte, level_pfn, start_pfn,
1052 last_pfn);
1053 }
1054
1055 /*
1056 * Free the page table if we're below the level we want to
1057 * retain and the range covers the entire table.
1058 */
1059 if (level < retain_level && !(start_pfn > level_pfn ||
1060 last_pfn < level_pfn + level_size(level) - 1)) {
1061 dma_clear_pte(pte);
1062 domain_flush_cache(domain, pte, sizeof(*pte));
1063 free_pgtable_page(level_pte);
1064 }
1065next:
1066 pfn += level_size(level);
1067 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1068}
1069
1070/*
1071 * clear last level (leaf) ptes and free page table pages below the
1072 * level we wish to keep intact.
1073 */
1074static void dma_pte_free_pagetable(struct dmar_domain *domain,
1075 unsigned long start_pfn,
1076 unsigned long last_pfn,
1077 int retain_level)
1078{
1079 dma_pte_clear_range(domain, start_pfn, last_pfn);
1080
1081 /* We don't need lock here; nobody else touches the iova range */
1082 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1083 domain->pgd, 0, start_pfn, last_pfn);
1084
1085 /* free pgd */
1086 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1087 free_pgtable_page(domain->pgd);
1088 domain->pgd = NULL;
1089 }
1090}
1091
1092/* When a page at a given level is being unlinked from its parent, we don't
1093 need to *modify* it at all. All we need to do is make a list of all the
1094 pages which can be freed just as soon as we've flushed the IOTLB and we
1095 know the hardware page-walk will no longer touch them.
1096 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1097 be freed. */
1098static void dma_pte_list_pagetables(struct dmar_domain *domain,
1099 int level, struct dma_pte *pte,
1100 struct list_head *freelist)
1101{
1102 struct page *pg;
1103
1104 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1105 list_add_tail(&pg->lru, freelist);
1106
1107 if (level == 1)
1108 return;
1109
1110 pte = page_address(pg);
1111 do {
1112 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1113 dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1114 pte++;
1115 } while (!first_pte_in_page(pte));
1116}
1117
1118static void dma_pte_clear_level(struct dmar_domain *domain, int level,
1119 struct dma_pte *pte, unsigned long pfn,
1120 unsigned long start_pfn, unsigned long last_pfn,
1121 struct list_head *freelist)
1122{
1123 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1124
1125 pfn = max(start_pfn, pfn);
1126 pte = &pte[pfn_level_offset(pfn, level)];
1127
1128 do {
1129 unsigned long level_pfn = pfn & level_mask(level);
1130
1131 if (!dma_pte_present(pte))
1132 goto next;
1133
1134 /* If range covers entire pagetable, free it */
1135 if (start_pfn <= level_pfn &&
1136 last_pfn >= level_pfn + level_size(level) - 1) {
1137 /* These suborbinate page tables are going away entirely. Don't
1138 bother to clear them; we're just going to *free* them. */
1139 if (level > 1 && !dma_pte_superpage(pte))
1140 dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1141
1142 dma_clear_pte(pte);
1143 if (!first_pte)
1144 first_pte = pte;
1145 last_pte = pte;
1146 } else if (level > 1) {
1147 /* Recurse down into a level that isn't *entirely* obsolete */
1148 dma_pte_clear_level(domain, level - 1,
1149 phys_to_virt(dma_pte_addr(pte)),
1150 level_pfn, start_pfn, last_pfn,
1151 freelist);
1152 }
1153next:
1154 pfn = level_pfn + level_size(level);
1155 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1156
1157 if (first_pte)
1158 domain_flush_cache(domain, first_pte,
1159 (void *)++last_pte - (void *)first_pte);
1160}
1161
1162/* We can't just free the pages because the IOMMU may still be walking
1163 the page tables, and may have cached the intermediate levels. The
1164 pages can only be freed after the IOTLB flush has been done. */
1165static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn,
1166 unsigned long last_pfn, struct list_head *freelist)
1167{
1168 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1169 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1170 BUG_ON(start_pfn > last_pfn);
1171
1172 /* we don't need lock here; nobody else touches the iova range */
1173 dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1174 domain->pgd, 0, start_pfn, last_pfn, freelist);
1175
1176 /* free pgd */
1177 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1178 struct page *pgd_page = virt_to_page(domain->pgd);
1179 list_add_tail(&pgd_page->lru, freelist);
1180 domain->pgd = NULL;
1181 }
1182}
1183
1184/* iommu handling */
1185static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1186{
1187 struct root_entry *root;
1188
1189 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1190 if (!root) {
1191 pr_err("Allocating root entry for %s failed\n",
1192 iommu->name);
1193 return -ENOMEM;
1194 }
1195
1196 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1197 iommu->root_entry = root;
1198
1199 return 0;
1200}
1201
1202static void iommu_set_root_entry(struct intel_iommu *iommu)
1203{
1204 u64 addr;
1205 u32 sts;
1206 unsigned long flag;
1207
1208 addr = virt_to_phys(iommu->root_entry);
1209 if (sm_supported(iommu))
1210 addr |= DMA_RTADDR_SMT;
1211
1212 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1213 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1214
1215 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1216
1217 /* Make sure hardware complete it */
1218 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1219 readl, (sts & DMA_GSTS_RTPS), sts);
1220
1221 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1222
1223 /*
1224 * Hardware invalidates all DMA remapping hardware translation
1225 * caches as part of SRTP flow.
1226 */
1227 if (cap_esrtps(iommu->cap))
1228 return;
1229
1230 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1231 if (sm_supported(iommu))
1232 qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);
1233 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1234}
1235
1236void iommu_flush_write_buffer(struct intel_iommu *iommu)
1237{
1238 u32 val;
1239 unsigned long flag;
1240
1241 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1242 return;
1243
1244 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1245 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1246
1247 /* Make sure hardware complete it */
1248 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1249 readl, (!(val & DMA_GSTS_WBFS)), val);
1250
1251 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1252}
1253
1254/* return value determine if we need a write buffer flush */
1255static void __iommu_flush_context(struct intel_iommu *iommu,
1256 u16 did, u16 source_id, u8 function_mask,
1257 u64 type)
1258{
1259 u64 val = 0;
1260 unsigned long flag;
1261
1262 switch (type) {
1263 case DMA_CCMD_GLOBAL_INVL:
1264 val = DMA_CCMD_GLOBAL_INVL;
1265 break;
1266 case DMA_CCMD_DOMAIN_INVL:
1267 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1268 break;
1269 case DMA_CCMD_DEVICE_INVL:
1270 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1271 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1272 break;
1273 default:
1274 BUG();
1275 }
1276 val |= DMA_CCMD_ICC;
1277
1278 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1279 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1280
1281 /* Make sure hardware complete it */
1282 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1283 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1284
1285 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1286}
1287
1288/* return value determine if we need a write buffer flush */
1289static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1290 u64 addr, unsigned int size_order, u64 type)
1291{
1292 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1293 u64 val = 0, val_iva = 0;
1294 unsigned long flag;
1295
1296 switch (type) {
1297 case DMA_TLB_GLOBAL_FLUSH:
1298 /* global flush doesn't need set IVA_REG */
1299 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1300 break;
1301 case DMA_TLB_DSI_FLUSH:
1302 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1303 break;
1304 case DMA_TLB_PSI_FLUSH:
1305 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1306 /* IH bit is passed in as part of address */
1307 val_iva = size_order | addr;
1308 break;
1309 default:
1310 BUG();
1311 }
1312 /* Note: set drain read/write */
1313#if 0
1314 /*
1315 * This is probably to be super secure.. Looks like we can
1316 * ignore it without any impact.
1317 */
1318 if (cap_read_drain(iommu->cap))
1319 val |= DMA_TLB_READ_DRAIN;
1320#endif
1321 if (cap_write_drain(iommu->cap))
1322 val |= DMA_TLB_WRITE_DRAIN;
1323
1324 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1325 /* Note: Only uses first TLB reg currently */
1326 if (val_iva)
1327 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1328 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1329
1330 /* Make sure hardware complete it */
1331 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1332 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1333
1334 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1335
1336 /* check IOTLB invalidation granularity */
1337 if (DMA_TLB_IAIG(val) == 0)
1338 pr_err("Flush IOTLB failed\n");
1339 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1340 pr_debug("TLB flush request %Lx, actual %Lx\n",
1341 (unsigned long long)DMA_TLB_IIRG(type),
1342 (unsigned long long)DMA_TLB_IAIG(val));
1343}
1344
1345static struct device_domain_info *
1346domain_lookup_dev_info(struct dmar_domain *domain,
1347 struct intel_iommu *iommu, u8 bus, u8 devfn)
1348{
1349 struct device_domain_info *info;
1350 unsigned long flags;
1351
1352 spin_lock_irqsave(&domain->lock, flags);
1353 list_for_each_entry(info, &domain->devices, link) {
1354 if (info->iommu == iommu && info->bus == bus &&
1355 info->devfn == devfn) {
1356 spin_unlock_irqrestore(&domain->lock, flags);
1357 return info;
1358 }
1359 }
1360 spin_unlock_irqrestore(&domain->lock, flags);
1361
1362 return NULL;
1363}
1364
1365static void domain_update_iotlb(struct dmar_domain *domain)
1366{
1367 struct device_domain_info *info;
1368 bool has_iotlb_device = false;
1369 unsigned long flags;
1370
1371 spin_lock_irqsave(&domain->lock, flags);
1372 list_for_each_entry(info, &domain->devices, link) {
1373 if (info->ats_enabled) {
1374 has_iotlb_device = true;
1375 break;
1376 }
1377 }
1378 domain->has_iotlb_device = has_iotlb_device;
1379 spin_unlock_irqrestore(&domain->lock, flags);
1380}
1381
1382/*
1383 * The extra devTLB flush quirk impacts those QAT devices with PCI device
1384 * IDs ranging from 0x4940 to 0x4943. It is exempted from risky_device()
1385 * check because it applies only to the built-in QAT devices and it doesn't
1386 * grant additional privileges.
1387 */
1388#define BUGGY_QAT_DEVID_MASK 0x4940
1389static bool dev_needs_extra_dtlb_flush(struct pci_dev *pdev)
1390{
1391 if (pdev->vendor != PCI_VENDOR_ID_INTEL)
1392 return false;
1393
1394 if ((pdev->device & 0xfffc) != BUGGY_QAT_DEVID_MASK)
1395 return false;
1396
1397 return true;
1398}
1399
1400static void iommu_enable_pci_caps(struct device_domain_info *info)
1401{
1402 struct pci_dev *pdev;
1403
1404 if (!dev_is_pci(info->dev))
1405 return;
1406
1407 pdev = to_pci_dev(info->dev);
1408 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1409 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1410 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1411 * reserved, which should be set to 0.
1412 */
1413 if (!ecap_dit(info->iommu->ecap))
1414 info->pfsid = 0;
1415 else {
1416 struct pci_dev *pf_pdev;
1417
1418 /* pdev will be returned if device is not a vf */
1419 pf_pdev = pci_physfn(pdev);
1420 info->pfsid = pci_dev_id(pf_pdev);
1421 }
1422
1423 /* The PCIe spec, in its wisdom, declares that the behaviour of
1424 the device if you enable PASID support after ATS support is
1425 undefined. So always enable PASID support on devices which
1426 have it, even if we can't yet know if we're ever going to
1427 use it. */
1428 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1429 info->pasid_enabled = 1;
1430
1431 if (info->pri_supported &&
1432 (info->pasid_enabled ? pci_prg_resp_pasid_required(pdev) : 1) &&
1433 !pci_reset_pri(pdev) && !pci_enable_pri(pdev, PRQ_DEPTH))
1434 info->pri_enabled = 1;
1435
1436 if (info->ats_supported && pci_ats_page_aligned(pdev) &&
1437 !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1438 info->ats_enabled = 1;
1439 domain_update_iotlb(info->domain);
1440 info->ats_qdep = pci_ats_queue_depth(pdev);
1441 }
1442}
1443
1444static void iommu_disable_pci_caps(struct device_domain_info *info)
1445{
1446 struct pci_dev *pdev;
1447
1448 if (!dev_is_pci(info->dev))
1449 return;
1450
1451 pdev = to_pci_dev(info->dev);
1452
1453 if (info->ats_enabled) {
1454 pci_disable_ats(pdev);
1455 info->ats_enabled = 0;
1456 domain_update_iotlb(info->domain);
1457 }
1458
1459 if (info->pri_enabled) {
1460 pci_disable_pri(pdev);
1461 info->pri_enabled = 0;
1462 }
1463
1464 if (info->pasid_enabled) {
1465 pci_disable_pasid(pdev);
1466 info->pasid_enabled = 0;
1467 }
1468}
1469
1470static void __iommu_flush_dev_iotlb(struct device_domain_info *info,
1471 u64 addr, unsigned int mask)
1472{
1473 u16 sid, qdep;
1474
1475 if (!info || !info->ats_enabled)
1476 return;
1477
1478 sid = info->bus << 8 | info->devfn;
1479 qdep = info->ats_qdep;
1480 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1481 qdep, addr, mask);
1482 quirk_extra_dev_tlb_flush(info, addr, mask, PASID_RID2PASID, qdep);
1483}
1484
1485static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1486 u64 addr, unsigned mask)
1487{
1488 struct device_domain_info *info;
1489 unsigned long flags;
1490
1491 if (!domain->has_iotlb_device)
1492 return;
1493
1494 spin_lock_irqsave(&domain->lock, flags);
1495 list_for_each_entry(info, &domain->devices, link)
1496 __iommu_flush_dev_iotlb(info, addr, mask);
1497 spin_unlock_irqrestore(&domain->lock, flags);
1498}
1499
1500static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1501 struct dmar_domain *domain,
1502 unsigned long pfn, unsigned int pages,
1503 int ih, int map)
1504{
1505 unsigned int aligned_pages = __roundup_pow_of_two(pages);
1506 unsigned int mask = ilog2(aligned_pages);
1507 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1508 u16 did = domain_id_iommu(domain, iommu);
1509
1510 BUG_ON(pages == 0);
1511
1512 if (ih)
1513 ih = 1 << 6;
1514
1515 if (domain->use_first_level) {
1516 qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, pages, ih);
1517 } else {
1518 unsigned long bitmask = aligned_pages - 1;
1519
1520 /*
1521 * PSI masks the low order bits of the base address. If the
1522 * address isn't aligned to the mask, then compute a mask value
1523 * needed to ensure the target range is flushed.
1524 */
1525 if (unlikely(bitmask & pfn)) {
1526 unsigned long end_pfn = pfn + pages - 1, shared_bits;
1527
1528 /*
1529 * Since end_pfn <= pfn + bitmask, the only way bits
1530 * higher than bitmask can differ in pfn and end_pfn is
1531 * by carrying. This means after masking out bitmask,
1532 * high bits starting with the first set bit in
1533 * shared_bits are all equal in both pfn and end_pfn.
1534 */
1535 shared_bits = ~(pfn ^ end_pfn) & ~bitmask;
1536 mask = shared_bits ? __ffs(shared_bits) : BITS_PER_LONG;
1537 }
1538
1539 /*
1540 * Fallback to domain selective flush if no PSI support or
1541 * the size is too big.
1542 */
1543 if (!cap_pgsel_inv(iommu->cap) ||
1544 mask > cap_max_amask_val(iommu->cap))
1545 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1546 DMA_TLB_DSI_FLUSH);
1547 else
1548 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1549 DMA_TLB_PSI_FLUSH);
1550 }
1551
1552 /*
1553 * In caching mode, changes of pages from non-present to present require
1554 * flush. However, device IOTLB doesn't need to be flushed in this case.
1555 */
1556 if (!cap_caching_mode(iommu->cap) || !map)
1557 iommu_flush_dev_iotlb(domain, addr, mask);
1558}
1559
1560/* Notification for newly created mappings */
1561static inline void __mapping_notify_one(struct intel_iommu *iommu,
1562 struct dmar_domain *domain,
1563 unsigned long pfn, unsigned int pages)
1564{
1565 /*
1566 * It's a non-present to present mapping. Only flush if caching mode
1567 * and second level.
1568 */
1569 if (cap_caching_mode(iommu->cap) && !domain->use_first_level)
1570 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1571 else
1572 iommu_flush_write_buffer(iommu);
1573}
1574
1575static void intel_flush_iotlb_all(struct iommu_domain *domain)
1576{
1577 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
1578 struct iommu_domain_info *info;
1579 unsigned long idx;
1580
1581 xa_for_each(&dmar_domain->iommu_array, idx, info) {
1582 struct intel_iommu *iommu = info->iommu;
1583 u16 did = domain_id_iommu(dmar_domain, iommu);
1584
1585 if (dmar_domain->use_first_level)
1586 qi_flush_piotlb(iommu, did, PASID_RID2PASID, 0, -1, 0);
1587 else
1588 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1589 DMA_TLB_DSI_FLUSH);
1590
1591 if (!cap_caching_mode(iommu->cap))
1592 iommu_flush_dev_iotlb(dmar_domain, 0, MAX_AGAW_PFN_WIDTH);
1593 }
1594}
1595
1596static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1597{
1598 u32 pmen;
1599 unsigned long flags;
1600
1601 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1602 return;
1603
1604 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1605 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1606 pmen &= ~DMA_PMEN_EPM;
1607 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1608
1609 /* wait for the protected region status bit to clear */
1610 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1611 readl, !(pmen & DMA_PMEN_PRS), pmen);
1612
1613 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1614}
1615
1616static void iommu_enable_translation(struct intel_iommu *iommu)
1617{
1618 u32 sts;
1619 unsigned long flags;
1620
1621 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1622 iommu->gcmd |= DMA_GCMD_TE;
1623 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1624
1625 /* Make sure hardware complete it */
1626 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1627 readl, (sts & DMA_GSTS_TES), sts);
1628
1629 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1630}
1631
1632static void iommu_disable_translation(struct intel_iommu *iommu)
1633{
1634 u32 sts;
1635 unsigned long flag;
1636
1637 if (iommu_skip_te_disable && iommu->drhd->gfx_dedicated &&
1638 (cap_read_drain(iommu->cap) || cap_write_drain(iommu->cap)))
1639 return;
1640
1641 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1642 iommu->gcmd &= ~DMA_GCMD_TE;
1643 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1644
1645 /* Make sure hardware complete it */
1646 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1647 readl, (!(sts & DMA_GSTS_TES)), sts);
1648
1649 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1650}
1651
1652static int iommu_init_domains(struct intel_iommu *iommu)
1653{
1654 u32 ndomains;
1655
1656 ndomains = cap_ndoms(iommu->cap);
1657 pr_debug("%s: Number of Domains supported <%d>\n",
1658 iommu->name, ndomains);
1659
1660 spin_lock_init(&iommu->lock);
1661
1662 iommu->domain_ids = bitmap_zalloc(ndomains, GFP_KERNEL);
1663 if (!iommu->domain_ids)
1664 return -ENOMEM;
1665
1666 /*
1667 * If Caching mode is set, then invalid translations are tagged
1668 * with domain-id 0, hence we need to pre-allocate it. We also
1669 * use domain-id 0 as a marker for non-allocated domain-id, so
1670 * make sure it is not used for a real domain.
1671 */
1672 set_bit(0, iommu->domain_ids);
1673
1674 /*
1675 * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1676 * entry for first-level or pass-through translation modes should
1677 * be programmed with a domain id different from those used for
1678 * second-level or nested translation. We reserve a domain id for
1679 * this purpose.
1680 */
1681 if (sm_supported(iommu))
1682 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1683
1684 return 0;
1685}
1686
1687static void disable_dmar_iommu(struct intel_iommu *iommu)
1688{
1689 if (!iommu->domain_ids)
1690 return;
1691
1692 /*
1693 * All iommu domains must have been detached from the devices,
1694 * hence there should be no domain IDs in use.
1695 */
1696 if (WARN_ON(bitmap_weight(iommu->domain_ids, cap_ndoms(iommu->cap))
1697 > NUM_RESERVED_DID))
1698 return;
1699
1700 if (iommu->gcmd & DMA_GCMD_TE)
1701 iommu_disable_translation(iommu);
1702}
1703
1704static void free_dmar_iommu(struct intel_iommu *iommu)
1705{
1706 if (iommu->domain_ids) {
1707 bitmap_free(iommu->domain_ids);
1708 iommu->domain_ids = NULL;
1709 }
1710
1711 if (iommu->copied_tables) {
1712 bitmap_free(iommu->copied_tables);
1713 iommu->copied_tables = NULL;
1714 }
1715
1716 /* free context mapping */
1717 free_context_table(iommu);
1718
1719#ifdef CONFIG_INTEL_IOMMU_SVM
1720 if (pasid_supported(iommu)) {
1721 if (ecap_prs(iommu->ecap))
1722 intel_svm_finish_prq(iommu);
1723 }
1724 if (vccap_pasid(iommu->vccap))
1725 ioasid_unregister_allocator(&iommu->pasid_allocator);
1726
1727#endif
1728}
1729
1730/*
1731 * Check and return whether first level is used by default for
1732 * DMA translation.
1733 */
1734static bool first_level_by_default(unsigned int type)
1735{
1736 /* Only SL is available in legacy mode */
1737 if (!scalable_mode_support())
1738 return false;
1739
1740 /* Only level (either FL or SL) is available, just use it */
1741 if (intel_cap_flts_sanity() ^ intel_cap_slts_sanity())
1742 return intel_cap_flts_sanity();
1743
1744 /* Both levels are available, decide it based on domain type */
1745 return type != IOMMU_DOMAIN_UNMANAGED;
1746}
1747
1748static struct dmar_domain *alloc_domain(unsigned int type)
1749{
1750 struct dmar_domain *domain;
1751
1752 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
1753 if (!domain)
1754 return NULL;
1755
1756 domain->nid = NUMA_NO_NODE;
1757 if (first_level_by_default(type))
1758 domain->use_first_level = true;
1759 domain->has_iotlb_device = false;
1760 INIT_LIST_HEAD(&domain->devices);
1761 spin_lock_init(&domain->lock);
1762 xa_init(&domain->iommu_array);
1763
1764 return domain;
1765}
1766
1767static int domain_attach_iommu(struct dmar_domain *domain,
1768 struct intel_iommu *iommu)
1769{
1770 struct iommu_domain_info *info, *curr;
1771 unsigned long ndomains;
1772 int num, ret = -ENOSPC;
1773
1774 info = kzalloc(sizeof(*info), GFP_KERNEL);
1775 if (!info)
1776 return -ENOMEM;
1777
1778 spin_lock(&iommu->lock);
1779 curr = xa_load(&domain->iommu_array, iommu->seq_id);
1780 if (curr) {
1781 curr->refcnt++;
1782 spin_unlock(&iommu->lock);
1783 kfree(info);
1784 return 0;
1785 }
1786
1787 ndomains = cap_ndoms(iommu->cap);
1788 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1789 if (num >= ndomains) {
1790 pr_err("%s: No free domain ids\n", iommu->name);
1791 goto err_unlock;
1792 }
1793
1794 set_bit(num, iommu->domain_ids);
1795 info->refcnt = 1;
1796 info->did = num;
1797 info->iommu = iommu;
1798 curr = xa_cmpxchg(&domain->iommu_array, iommu->seq_id,
1799 NULL, info, GFP_ATOMIC);
1800 if (curr) {
1801 ret = xa_err(curr) ? : -EBUSY;
1802 goto err_clear;
1803 }
1804 domain_update_iommu_cap(domain);
1805
1806 spin_unlock(&iommu->lock);
1807 return 0;
1808
1809err_clear:
1810 clear_bit(info->did, iommu->domain_ids);
1811err_unlock:
1812 spin_unlock(&iommu->lock);
1813 kfree(info);
1814 return ret;
1815}
1816
1817static void domain_detach_iommu(struct dmar_domain *domain,
1818 struct intel_iommu *iommu)
1819{
1820 struct iommu_domain_info *info;
1821
1822 spin_lock(&iommu->lock);
1823 info = xa_load(&domain->iommu_array, iommu->seq_id);
1824 if (--info->refcnt == 0) {
1825 clear_bit(info->did, iommu->domain_ids);
1826 xa_erase(&domain->iommu_array, iommu->seq_id);
1827 domain->nid = NUMA_NO_NODE;
1828 domain_update_iommu_cap(domain);
1829 kfree(info);
1830 }
1831 spin_unlock(&iommu->lock);
1832}
1833
1834static inline int guestwidth_to_adjustwidth(int gaw)
1835{
1836 int agaw;
1837 int r = (gaw - 12) % 9;
1838
1839 if (r == 0)
1840 agaw = gaw;
1841 else
1842 agaw = gaw + 9 - r;
1843 if (agaw > 64)
1844 agaw = 64;
1845 return agaw;
1846}
1847
1848static void domain_exit(struct dmar_domain *domain)
1849{
1850 if (domain->pgd) {
1851 LIST_HEAD(freelist);
1852
1853 domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist);
1854 put_pages_list(&freelist);
1855 }
1856
1857 if (WARN_ON(!list_empty(&domain->devices)))
1858 return;
1859
1860 kfree(domain);
1861}
1862
1863/*
1864 * Get the PASID directory size for scalable mode context entry.
1865 * Value of X in the PDTS field of a scalable mode context entry
1866 * indicates PASID directory with 2^(X + 7) entries.
1867 */
1868static inline unsigned long context_get_sm_pds(struct pasid_table *table)
1869{
1870 unsigned long pds, max_pde;
1871
1872 max_pde = table->max_pasid >> PASID_PDE_SHIFT;
1873 pds = find_first_bit(&max_pde, MAX_NR_PASID_BITS);
1874 if (pds < 7)
1875 return 0;
1876
1877 return pds - 7;
1878}
1879
1880/*
1881 * Set the RID_PASID field of a scalable mode context entry. The
1882 * IOMMU hardware will use the PASID value set in this field for
1883 * DMA translations of DMA requests without PASID.
1884 */
1885static inline void
1886context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid)
1887{
1888 context->hi |= pasid & ((1 << 20) - 1);
1889}
1890
1891/*
1892 * Set the DTE(Device-TLB Enable) field of a scalable mode context
1893 * entry.
1894 */
1895static inline void context_set_sm_dte(struct context_entry *context)
1896{
1897 context->lo |= (1 << 2);
1898}
1899
1900/*
1901 * Set the PRE(Page Request Enable) field of a scalable mode context
1902 * entry.
1903 */
1904static inline void context_set_sm_pre(struct context_entry *context)
1905{
1906 context->lo |= (1 << 4);
1907}
1908
1909/* Convert value to context PASID directory size field coding. */
1910#define context_pdts(pds) (((pds) & 0x7) << 9)
1911
1912static int domain_context_mapping_one(struct dmar_domain *domain,
1913 struct intel_iommu *iommu,
1914 struct pasid_table *table,
1915 u8 bus, u8 devfn)
1916{
1917 struct device_domain_info *info =
1918 domain_lookup_dev_info(domain, iommu, bus, devfn);
1919 u16 did = domain_id_iommu(domain, iommu);
1920 int translation = CONTEXT_TT_MULTI_LEVEL;
1921 struct context_entry *context;
1922 int ret;
1923
1924 WARN_ON(did == 0);
1925
1926 if (hw_pass_through && domain_type_is_si(domain))
1927 translation = CONTEXT_TT_PASS_THROUGH;
1928
1929 pr_debug("Set context mapping for %02x:%02x.%d\n",
1930 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1931
1932 BUG_ON(!domain->pgd);
1933
1934 spin_lock(&iommu->lock);
1935 ret = -ENOMEM;
1936 context = iommu_context_addr(iommu, bus, devfn, 1);
1937 if (!context)
1938 goto out_unlock;
1939
1940 ret = 0;
1941 if (context_present(context) && !context_copied(iommu, bus, devfn))
1942 goto out_unlock;
1943
1944 /*
1945 * For kdump cases, old valid entries may be cached due to the
1946 * in-flight DMA and copied pgtable, but there is no unmapping
1947 * behaviour for them, thus we need an explicit cache flush for
1948 * the newly-mapped device. For kdump, at this point, the device
1949 * is supposed to finish reset at its driver probe stage, so no
1950 * in-flight DMA will exist, and we don't need to worry anymore
1951 * hereafter.
1952 */
1953 if (context_copied(iommu, bus, devfn)) {
1954 u16 did_old = context_domain_id(context);
1955
1956 if (did_old < cap_ndoms(iommu->cap)) {
1957 iommu->flush.flush_context(iommu, did_old,
1958 (((u16)bus) << 8) | devfn,
1959 DMA_CCMD_MASK_NOBIT,
1960 DMA_CCMD_DEVICE_INVL);
1961 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
1962 DMA_TLB_DSI_FLUSH);
1963 }
1964
1965 clear_context_copied(iommu, bus, devfn);
1966 }
1967
1968 context_clear_entry(context);
1969
1970 if (sm_supported(iommu)) {
1971 unsigned long pds;
1972
1973 WARN_ON(!table);
1974
1975 /* Setup the PASID DIR pointer: */
1976 pds = context_get_sm_pds(table);
1977 context->lo = (u64)virt_to_phys(table->table) |
1978 context_pdts(pds);
1979
1980 /* Setup the RID_PASID field: */
1981 context_set_sm_rid2pasid(context, PASID_RID2PASID);
1982
1983 /*
1984 * Setup the Device-TLB enable bit and Page request
1985 * Enable bit:
1986 */
1987 if (info && info->ats_supported)
1988 context_set_sm_dte(context);
1989 if (info && info->pri_supported)
1990 context_set_sm_pre(context);
1991 if (info && info->pasid_supported)
1992 context_set_pasid(context);
1993 } else {
1994 struct dma_pte *pgd = domain->pgd;
1995 int agaw;
1996
1997 context_set_domain_id(context, did);
1998
1999 if (translation != CONTEXT_TT_PASS_THROUGH) {
2000 /*
2001 * Skip top levels of page tables for iommu which has
2002 * less agaw than default. Unnecessary for PT mode.
2003 */
2004 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2005 ret = -ENOMEM;
2006 pgd = phys_to_virt(dma_pte_addr(pgd));
2007 if (!dma_pte_present(pgd))
2008 goto out_unlock;
2009 }
2010
2011 if (info && info->ats_supported)
2012 translation = CONTEXT_TT_DEV_IOTLB;
2013 else
2014 translation = CONTEXT_TT_MULTI_LEVEL;
2015
2016 context_set_address_root(context, virt_to_phys(pgd));
2017 context_set_address_width(context, agaw);
2018 } else {
2019 /*
2020 * In pass through mode, AW must be programmed to
2021 * indicate the largest AGAW value supported by
2022 * hardware. And ASR is ignored by hardware.
2023 */
2024 context_set_address_width(context, iommu->msagaw);
2025 }
2026
2027 context_set_translation_type(context, translation);
2028 }
2029
2030 context_set_fault_enable(context);
2031 context_set_present(context);
2032 if (!ecap_coherent(iommu->ecap))
2033 clflush_cache_range(context, sizeof(*context));
2034
2035 /*
2036 * It's a non-present to present mapping. If hardware doesn't cache
2037 * non-present entry we only need to flush the write-buffer. If the
2038 * _does_ cache non-present entries, then it does so in the special
2039 * domain #0, which we have to flush:
2040 */
2041 if (cap_caching_mode(iommu->cap)) {
2042 iommu->flush.flush_context(iommu, 0,
2043 (((u16)bus) << 8) | devfn,
2044 DMA_CCMD_MASK_NOBIT,
2045 DMA_CCMD_DEVICE_INVL);
2046 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2047 } else {
2048 iommu_flush_write_buffer(iommu);
2049 }
2050
2051 ret = 0;
2052
2053out_unlock:
2054 spin_unlock(&iommu->lock);
2055
2056 return ret;
2057}
2058
2059struct domain_context_mapping_data {
2060 struct dmar_domain *domain;
2061 struct intel_iommu *iommu;
2062 struct pasid_table *table;
2063};
2064
2065static int domain_context_mapping_cb(struct pci_dev *pdev,
2066 u16 alias, void *opaque)
2067{
2068 struct domain_context_mapping_data *data = opaque;
2069
2070 return domain_context_mapping_one(data->domain, data->iommu,
2071 data->table, PCI_BUS_NUM(alias),
2072 alias & 0xff);
2073}
2074
2075static int
2076domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2077{
2078 struct domain_context_mapping_data data;
2079 struct pasid_table *table;
2080 struct intel_iommu *iommu;
2081 u8 bus, devfn;
2082
2083 iommu = device_to_iommu(dev, &bus, &devfn);
2084 if (!iommu)
2085 return -ENODEV;
2086
2087 table = intel_pasid_get_table(dev);
2088
2089 if (!dev_is_pci(dev))
2090 return domain_context_mapping_one(domain, iommu, table,
2091 bus, devfn);
2092
2093 data.domain = domain;
2094 data.iommu = iommu;
2095 data.table = table;
2096
2097 return pci_for_each_dma_alias(to_pci_dev(dev),
2098 &domain_context_mapping_cb, &data);
2099}
2100
2101/* Returns a number of VTD pages, but aligned to MM page size */
2102static inline unsigned long aligned_nrpages(unsigned long host_addr,
2103 size_t size)
2104{
2105 host_addr &= ~PAGE_MASK;
2106 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2107}
2108
2109/* Return largest possible superpage level for a given mapping */
2110static inline int hardware_largepage_caps(struct dmar_domain *domain,
2111 unsigned long iov_pfn,
2112 unsigned long phy_pfn,
2113 unsigned long pages)
2114{
2115 int support, level = 1;
2116 unsigned long pfnmerge;
2117
2118 support = domain->iommu_superpage;
2119
2120 /* To use a large page, the virtual *and* physical addresses
2121 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2122 of them will mean we have to use smaller pages. So just
2123 merge them and check both at once. */
2124 pfnmerge = iov_pfn | phy_pfn;
2125
2126 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2127 pages >>= VTD_STRIDE_SHIFT;
2128 if (!pages)
2129 break;
2130 pfnmerge >>= VTD_STRIDE_SHIFT;
2131 level++;
2132 support--;
2133 }
2134 return level;
2135}
2136
2137/*
2138 * Ensure that old small page tables are removed to make room for superpage(s).
2139 * We're going to add new large pages, so make sure we don't remove their parent
2140 * tables. The IOTLB/devTLBs should be flushed if any PDE/PTEs are cleared.
2141 */
2142static void switch_to_super_page(struct dmar_domain *domain,
2143 unsigned long start_pfn,
2144 unsigned long end_pfn, int level)
2145{
2146 unsigned long lvl_pages = lvl_to_nr_pages(level);
2147 struct iommu_domain_info *info;
2148 struct dma_pte *pte = NULL;
2149 unsigned long i;
2150
2151 while (start_pfn <= end_pfn) {
2152 if (!pte)
2153 pte = pfn_to_dma_pte(domain, start_pfn, &level);
2154
2155 if (dma_pte_present(pte)) {
2156 dma_pte_free_pagetable(domain, start_pfn,
2157 start_pfn + lvl_pages - 1,
2158 level + 1);
2159
2160 xa_for_each(&domain->iommu_array, i, info)
2161 iommu_flush_iotlb_psi(info->iommu, domain,
2162 start_pfn, lvl_pages,
2163 0, 0);
2164 }
2165
2166 pte++;
2167 start_pfn += lvl_pages;
2168 if (first_pte_in_page(pte))
2169 pte = NULL;
2170 }
2171}
2172
2173static int
2174__domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2175 unsigned long phys_pfn, unsigned long nr_pages, int prot)
2176{
2177 struct dma_pte *first_pte = NULL, *pte = NULL;
2178 unsigned int largepage_lvl = 0;
2179 unsigned long lvl_pages = 0;
2180 phys_addr_t pteval;
2181 u64 attr;
2182
2183 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2184
2185 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2186 return -EINVAL;
2187
2188 attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP);
2189 attr |= DMA_FL_PTE_PRESENT;
2190 if (domain->use_first_level) {
2191 attr |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
2192 if (prot & DMA_PTE_WRITE)
2193 attr |= DMA_FL_PTE_DIRTY;
2194 }
2195
2196 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | attr;
2197
2198 while (nr_pages > 0) {
2199 uint64_t tmp;
2200
2201 if (!pte) {
2202 largepage_lvl = hardware_largepage_caps(domain, iov_pfn,
2203 phys_pfn, nr_pages);
2204
2205 pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2206 if (!pte)
2207 return -ENOMEM;
2208 first_pte = pte;
2209
2210 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2211
2212 /* It is large page*/
2213 if (largepage_lvl > 1) {
2214 unsigned long end_pfn;
2215 unsigned long pages_to_remove;
2216
2217 pteval |= DMA_PTE_LARGE_PAGE;
2218 pages_to_remove = min_t(unsigned long, nr_pages,
2219 nr_pte_to_next_page(pte) * lvl_pages);
2220 end_pfn = iov_pfn + pages_to_remove - 1;
2221 switch_to_super_page(domain, iov_pfn, end_pfn, largepage_lvl);
2222 } else {
2223 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2224 }
2225
2226 }
2227 /* We don't need lock here, nobody else
2228 * touches the iova range
2229 */
2230 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2231 if (tmp) {
2232 static int dumps = 5;
2233 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2234 iov_pfn, tmp, (unsigned long long)pteval);
2235 if (dumps) {
2236 dumps--;
2237 debug_dma_dump_mappings(NULL);
2238 }
2239 WARN_ON(1);
2240 }
2241
2242 nr_pages -= lvl_pages;
2243 iov_pfn += lvl_pages;
2244 phys_pfn += lvl_pages;
2245 pteval += lvl_pages * VTD_PAGE_SIZE;
2246
2247 /* If the next PTE would be the first in a new page, then we
2248 * need to flush the cache on the entries we've just written.
2249 * And then we'll need to recalculate 'pte', so clear it and
2250 * let it get set again in the if (!pte) block above.
2251 *
2252 * If we're done (!nr_pages) we need to flush the cache too.
2253 *
2254 * Also if we've been setting superpages, we may need to
2255 * recalculate 'pte' and switch back to smaller pages for the
2256 * end of the mapping, if the trailing size is not enough to
2257 * use another superpage (i.e. nr_pages < lvl_pages).
2258 */
2259 pte++;
2260 if (!nr_pages || first_pte_in_page(pte) ||
2261 (largepage_lvl > 1 && nr_pages < lvl_pages)) {
2262 domain_flush_cache(domain, first_pte,
2263 (void *)pte - (void *)first_pte);
2264 pte = NULL;
2265 }
2266 }
2267
2268 return 0;
2269}
2270
2271static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 devfn)
2272{
2273 struct intel_iommu *iommu = info->iommu;
2274 struct context_entry *context;
2275 u16 did_old;
2276
2277 if (!iommu)
2278 return;
2279
2280 spin_lock(&iommu->lock);
2281 context = iommu_context_addr(iommu, bus, devfn, 0);
2282 if (!context) {
2283 spin_unlock(&iommu->lock);
2284 return;
2285 }
2286
2287 if (sm_supported(iommu)) {
2288 if (hw_pass_through && domain_type_is_si(info->domain))
2289 did_old = FLPT_DEFAULT_DID;
2290 else
2291 did_old = domain_id_iommu(info->domain, iommu);
2292 } else {
2293 did_old = context_domain_id(context);
2294 }
2295
2296 context_clear_entry(context);
2297 __iommu_flush_cache(iommu, context, sizeof(*context));
2298 spin_unlock(&iommu->lock);
2299 iommu->flush.flush_context(iommu,
2300 did_old,
2301 (((u16)bus) << 8) | devfn,
2302 DMA_CCMD_MASK_NOBIT,
2303 DMA_CCMD_DEVICE_INVL);
2304
2305 if (sm_supported(iommu))
2306 qi_flush_pasid_cache(iommu, did_old, QI_PC_ALL_PASIDS, 0);
2307
2308 iommu->flush.flush_iotlb(iommu,
2309 did_old,
2310 0,
2311 0,
2312 DMA_TLB_DSI_FLUSH);
2313
2314 __iommu_flush_dev_iotlb(info, 0, MAX_AGAW_PFN_WIDTH);
2315}
2316
2317static int domain_setup_first_level(struct intel_iommu *iommu,
2318 struct dmar_domain *domain,
2319 struct device *dev,
2320 u32 pasid)
2321{
2322 struct dma_pte *pgd = domain->pgd;
2323 int agaw, level;
2324 int flags = 0;
2325
2326 /*
2327 * Skip top levels of page tables for iommu which has
2328 * less agaw than default. Unnecessary for PT mode.
2329 */
2330 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2331 pgd = phys_to_virt(dma_pte_addr(pgd));
2332 if (!dma_pte_present(pgd))
2333 return -ENOMEM;
2334 }
2335
2336 level = agaw_to_level(agaw);
2337 if (level != 4 && level != 5)
2338 return -EINVAL;
2339
2340 if (pasid != PASID_RID2PASID)
2341 flags |= PASID_FLAG_SUPERVISOR_MODE;
2342 if (level == 5)
2343 flags |= PASID_FLAG_FL5LP;
2344
2345 if (domain->force_snooping)
2346 flags |= PASID_FLAG_PAGE_SNOOP;
2347
2348 return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid,
2349 domain_id_iommu(domain, iommu),
2350 flags);
2351}
2352
2353static bool dev_is_real_dma_subdevice(struct device *dev)
2354{
2355 return dev && dev_is_pci(dev) &&
2356 pci_real_dma_dev(to_pci_dev(dev)) != to_pci_dev(dev);
2357}
2358
2359static int iommu_domain_identity_map(struct dmar_domain *domain,
2360 unsigned long first_vpfn,
2361 unsigned long last_vpfn)
2362{
2363 /*
2364 * RMRR range might have overlap with physical memory range,
2365 * clear it first
2366 */
2367 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2368
2369 return __domain_mapping(domain, first_vpfn,
2370 first_vpfn, last_vpfn - first_vpfn + 1,
2371 DMA_PTE_READ|DMA_PTE_WRITE);
2372}
2373
2374static int md_domain_init(struct dmar_domain *domain, int guest_width);
2375
2376static int __init si_domain_init(int hw)
2377{
2378 struct dmar_rmrr_unit *rmrr;
2379 struct device *dev;
2380 int i, nid, ret;
2381
2382 si_domain = alloc_domain(IOMMU_DOMAIN_IDENTITY);
2383 if (!si_domain)
2384 return -EFAULT;
2385
2386 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2387 domain_exit(si_domain);
2388 si_domain = NULL;
2389 return -EFAULT;
2390 }
2391
2392 if (hw)
2393 return 0;
2394
2395 for_each_online_node(nid) {
2396 unsigned long start_pfn, end_pfn;
2397 int i;
2398
2399 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2400 ret = iommu_domain_identity_map(si_domain,
2401 mm_to_dma_pfn(start_pfn),
2402 mm_to_dma_pfn(end_pfn));
2403 if (ret)
2404 return ret;
2405 }
2406 }
2407
2408 /*
2409 * Identity map the RMRRs so that devices with RMRRs could also use
2410 * the si_domain.
2411 */
2412 for_each_rmrr_units(rmrr) {
2413 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2414 i, dev) {
2415 unsigned long long start = rmrr->base_address;
2416 unsigned long long end = rmrr->end_address;
2417
2418 if (WARN_ON(end < start ||
2419 end >> agaw_to_width(si_domain->agaw)))
2420 continue;
2421
2422 ret = iommu_domain_identity_map(si_domain,
2423 mm_to_dma_pfn(start >> PAGE_SHIFT),
2424 mm_to_dma_pfn(end >> PAGE_SHIFT));
2425 if (ret)
2426 return ret;
2427 }
2428 }
2429
2430 return 0;
2431}
2432
2433static int dmar_domain_attach_device(struct dmar_domain *domain,
2434 struct device *dev)
2435{
2436 struct device_domain_info *info = dev_iommu_priv_get(dev);
2437 struct intel_iommu *iommu;
2438 unsigned long flags;
2439 u8 bus, devfn;
2440 int ret;
2441
2442 iommu = device_to_iommu(dev, &bus, &devfn);
2443 if (!iommu)
2444 return -ENODEV;
2445
2446 ret = domain_attach_iommu(domain, iommu);
2447 if (ret)
2448 return ret;
2449 info->domain = domain;
2450 spin_lock_irqsave(&domain->lock, flags);
2451 list_add(&info->link, &domain->devices);
2452 spin_unlock_irqrestore(&domain->lock, flags);
2453
2454 /* PASID table is mandatory for a PCI device in scalable mode. */
2455 if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
2456 /* Setup the PASID entry for requests without PASID: */
2457 if (hw_pass_through && domain_type_is_si(domain))
2458 ret = intel_pasid_setup_pass_through(iommu, domain,
2459 dev, PASID_RID2PASID);
2460 else if (domain->use_first_level)
2461 ret = domain_setup_first_level(iommu, domain, dev,
2462 PASID_RID2PASID);
2463 else
2464 ret = intel_pasid_setup_second_level(iommu, domain,
2465 dev, PASID_RID2PASID);
2466 if (ret) {
2467 dev_err(dev, "Setup RID2PASID failed\n");
2468 device_block_translation(dev);
2469 return ret;
2470 }
2471 }
2472
2473 ret = domain_context_mapping(domain, dev);
2474 if (ret) {
2475 dev_err(dev, "Domain context map failed\n");
2476 device_block_translation(dev);
2477 return ret;
2478 }
2479
2480 iommu_enable_pci_caps(info);
2481
2482 return 0;
2483}
2484
2485static bool device_has_rmrr(struct device *dev)
2486{
2487 struct dmar_rmrr_unit *rmrr;
2488 struct device *tmp;
2489 int i;
2490
2491 rcu_read_lock();
2492 for_each_rmrr_units(rmrr) {
2493 /*
2494 * Return TRUE if this RMRR contains the device that
2495 * is passed in.
2496 */
2497 for_each_active_dev_scope(rmrr->devices,
2498 rmrr->devices_cnt, i, tmp)
2499 if (tmp == dev ||
2500 is_downstream_to_pci_bridge(dev, tmp)) {
2501 rcu_read_unlock();
2502 return true;
2503 }
2504 }
2505 rcu_read_unlock();
2506 return false;
2507}
2508
2509/**
2510 * device_rmrr_is_relaxable - Test whether the RMRR of this device
2511 * is relaxable (ie. is allowed to be not enforced under some conditions)
2512 * @dev: device handle
2513 *
2514 * We assume that PCI USB devices with RMRRs have them largely
2515 * for historical reasons and that the RMRR space is not actively used post
2516 * boot. This exclusion may change if vendors begin to abuse it.
2517 *
2518 * The same exception is made for graphics devices, with the requirement that
2519 * any use of the RMRR regions will be torn down before assigning the device
2520 * to a guest.
2521 *
2522 * Return: true if the RMRR is relaxable, false otherwise
2523 */
2524static bool device_rmrr_is_relaxable(struct device *dev)
2525{
2526 struct pci_dev *pdev;
2527
2528 if (!dev_is_pci(dev))
2529 return false;
2530
2531 pdev = to_pci_dev(dev);
2532 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2533 return true;
2534 else
2535 return false;
2536}
2537
2538/*
2539 * There are a couple cases where we need to restrict the functionality of
2540 * devices associated with RMRRs. The first is when evaluating a device for
2541 * identity mapping because problems exist when devices are moved in and out
2542 * of domains and their respective RMRR information is lost. This means that
2543 * a device with associated RMRRs will never be in a "passthrough" domain.
2544 * The second is use of the device through the IOMMU API. This interface
2545 * expects to have full control of the IOVA space for the device. We cannot
2546 * satisfy both the requirement that RMRR access is maintained and have an
2547 * unencumbered IOVA space. We also have no ability to quiesce the device's
2548 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2549 * We therefore prevent devices associated with an RMRR from participating in
2550 * the IOMMU API, which eliminates them from device assignment.
2551 *
2552 * In both cases, devices which have relaxable RMRRs are not concerned by this
2553 * restriction. See device_rmrr_is_relaxable comment.
2554 */
2555static bool device_is_rmrr_locked(struct device *dev)
2556{
2557 if (!device_has_rmrr(dev))
2558 return false;
2559
2560 if (device_rmrr_is_relaxable(dev))
2561 return false;
2562
2563 return true;
2564}
2565
2566/*
2567 * Return the required default domain type for a specific device.
2568 *
2569 * @dev: the device in query
2570 * @startup: true if this is during early boot
2571 *
2572 * Returns:
2573 * - IOMMU_DOMAIN_DMA: device requires a dynamic mapping domain
2574 * - IOMMU_DOMAIN_IDENTITY: device requires an identical mapping domain
2575 * - 0: both identity and dynamic domains work for this device
2576 */
2577static int device_def_domain_type(struct device *dev)
2578{
2579 if (dev_is_pci(dev)) {
2580 struct pci_dev *pdev = to_pci_dev(dev);
2581
2582 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2583 return IOMMU_DOMAIN_IDENTITY;
2584
2585 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2586 return IOMMU_DOMAIN_IDENTITY;
2587 }
2588
2589 return 0;
2590}
2591
2592static void intel_iommu_init_qi(struct intel_iommu *iommu)
2593{
2594 /*
2595 * Start from the sane iommu hardware state.
2596 * If the queued invalidation is already initialized by us
2597 * (for example, while enabling interrupt-remapping) then
2598 * we got the things already rolling from a sane state.
2599 */
2600 if (!iommu->qi) {
2601 /*
2602 * Clear any previous faults.
2603 */
2604 dmar_fault(-1, iommu);
2605 /*
2606 * Disable queued invalidation if supported and already enabled
2607 * before OS handover.
2608 */
2609 dmar_disable_qi(iommu);
2610 }
2611
2612 if (dmar_enable_qi(iommu)) {
2613 /*
2614 * Queued Invalidate not enabled, use Register Based Invalidate
2615 */
2616 iommu->flush.flush_context = __iommu_flush_context;
2617 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2618 pr_info("%s: Using Register based invalidation\n",
2619 iommu->name);
2620 } else {
2621 iommu->flush.flush_context = qi_flush_context;
2622 iommu->flush.flush_iotlb = qi_flush_iotlb;
2623 pr_info("%s: Using Queued invalidation\n", iommu->name);
2624 }
2625}
2626
2627static int copy_context_table(struct intel_iommu *iommu,
2628 struct root_entry *old_re,
2629 struct context_entry **tbl,
2630 int bus, bool ext)
2631{
2632 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2633 struct context_entry *new_ce = NULL, ce;
2634 struct context_entry *old_ce = NULL;
2635 struct root_entry re;
2636 phys_addr_t old_ce_phys;
2637
2638 tbl_idx = ext ? bus * 2 : bus;
2639 memcpy(&re, old_re, sizeof(re));
2640
2641 for (devfn = 0; devfn < 256; devfn++) {
2642 /* First calculate the correct index */
2643 idx = (ext ? devfn * 2 : devfn) % 256;
2644
2645 if (idx == 0) {
2646 /* First save what we may have and clean up */
2647 if (new_ce) {
2648 tbl[tbl_idx] = new_ce;
2649 __iommu_flush_cache(iommu, new_ce,
2650 VTD_PAGE_SIZE);
2651 pos = 1;
2652 }
2653
2654 if (old_ce)
2655 memunmap(old_ce);
2656
2657 ret = 0;
2658 if (devfn < 0x80)
2659 old_ce_phys = root_entry_lctp(&re);
2660 else
2661 old_ce_phys = root_entry_uctp(&re);
2662
2663 if (!old_ce_phys) {
2664 if (ext && devfn == 0) {
2665 /* No LCTP, try UCTP */
2666 devfn = 0x7f;
2667 continue;
2668 } else {
2669 goto out;
2670 }
2671 }
2672
2673 ret = -ENOMEM;
2674 old_ce = memremap(old_ce_phys, PAGE_SIZE,
2675 MEMREMAP_WB);
2676 if (!old_ce)
2677 goto out;
2678
2679 new_ce = alloc_pgtable_page(iommu->node);
2680 if (!new_ce)
2681 goto out_unmap;
2682
2683 ret = 0;
2684 }
2685
2686 /* Now copy the context entry */
2687 memcpy(&ce, old_ce + idx, sizeof(ce));
2688
2689 if (!context_present(&ce))
2690 continue;
2691
2692 did = context_domain_id(&ce);
2693 if (did >= 0 && did < cap_ndoms(iommu->cap))
2694 set_bit(did, iommu->domain_ids);
2695
2696 set_context_copied(iommu, bus, devfn);
2697 new_ce[idx] = ce;
2698 }
2699
2700 tbl[tbl_idx + pos] = new_ce;
2701
2702 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2703
2704out_unmap:
2705 memunmap(old_ce);
2706
2707out:
2708 return ret;
2709}
2710
2711static int copy_translation_tables(struct intel_iommu *iommu)
2712{
2713 struct context_entry **ctxt_tbls;
2714 struct root_entry *old_rt;
2715 phys_addr_t old_rt_phys;
2716 int ctxt_table_entries;
2717 u64 rtaddr_reg;
2718 int bus, ret;
2719 bool new_ext, ext;
2720
2721 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2722 ext = !!(rtaddr_reg & DMA_RTADDR_SMT);
2723 new_ext = !!sm_supported(iommu);
2724
2725 /*
2726 * The RTT bit can only be changed when translation is disabled,
2727 * but disabling translation means to open a window for data
2728 * corruption. So bail out and don't copy anything if we would
2729 * have to change the bit.
2730 */
2731 if (new_ext != ext)
2732 return -EINVAL;
2733
2734 iommu->copied_tables = bitmap_zalloc(BIT_ULL(16), GFP_KERNEL);
2735 if (!iommu->copied_tables)
2736 return -ENOMEM;
2737
2738 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2739 if (!old_rt_phys)
2740 return -EINVAL;
2741
2742 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
2743 if (!old_rt)
2744 return -ENOMEM;
2745
2746 /* This is too big for the stack - allocate it from slab */
2747 ctxt_table_entries = ext ? 512 : 256;
2748 ret = -ENOMEM;
2749 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
2750 if (!ctxt_tbls)
2751 goto out_unmap;
2752
2753 for (bus = 0; bus < 256; bus++) {
2754 ret = copy_context_table(iommu, &old_rt[bus],
2755 ctxt_tbls, bus, ext);
2756 if (ret) {
2757 pr_err("%s: Failed to copy context table for bus %d\n",
2758 iommu->name, bus);
2759 continue;
2760 }
2761 }
2762
2763 spin_lock(&iommu->lock);
2764
2765 /* Context tables are copied, now write them to the root_entry table */
2766 for (bus = 0; bus < 256; bus++) {
2767 int idx = ext ? bus * 2 : bus;
2768 u64 val;
2769
2770 if (ctxt_tbls[idx]) {
2771 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2772 iommu->root_entry[bus].lo = val;
2773 }
2774
2775 if (!ext || !ctxt_tbls[idx + 1])
2776 continue;
2777
2778 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2779 iommu->root_entry[bus].hi = val;
2780 }
2781
2782 spin_unlock(&iommu->lock);
2783
2784 kfree(ctxt_tbls);
2785
2786 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
2787
2788 ret = 0;
2789
2790out_unmap:
2791 memunmap(old_rt);
2792
2793 return ret;
2794}
2795
2796#ifdef CONFIG_INTEL_IOMMU_SVM
2797static ioasid_t intel_vcmd_ioasid_alloc(ioasid_t min, ioasid_t max, void *data)
2798{
2799 struct intel_iommu *iommu = data;
2800 ioasid_t ioasid;
2801
2802 if (!iommu)
2803 return INVALID_IOASID;
2804 /*
2805 * VT-d virtual command interface always uses the full 20 bit
2806 * PASID range. Host can partition guest PASID range based on
2807 * policies but it is out of guest's control.
2808 */
2809 if (min < PASID_MIN || max > intel_pasid_max_id)
2810 return INVALID_IOASID;
2811
2812 if (vcmd_alloc_pasid(iommu, &ioasid))
2813 return INVALID_IOASID;
2814
2815 return ioasid;
2816}
2817
2818static void intel_vcmd_ioasid_free(ioasid_t ioasid, void *data)
2819{
2820 struct intel_iommu *iommu = data;
2821
2822 if (!iommu)
2823 return;
2824 /*
2825 * Sanity check the ioasid owner is done at upper layer, e.g. VFIO
2826 * We can only free the PASID when all the devices are unbound.
2827 */
2828 if (ioasid_find(NULL, ioasid, NULL)) {
2829 pr_alert("Cannot free active IOASID %d\n", ioasid);
2830 return;
2831 }
2832 vcmd_free_pasid(iommu, ioasid);
2833}
2834
2835static void register_pasid_allocator(struct intel_iommu *iommu)
2836{
2837 /*
2838 * If we are running in the host, no need for custom allocator
2839 * in that PASIDs are allocated from the host system-wide.
2840 */
2841 if (!cap_caching_mode(iommu->cap))
2842 return;
2843
2844 if (!sm_supported(iommu)) {
2845 pr_warn("VT-d Scalable Mode not enabled, no PASID allocation\n");
2846 return;
2847 }
2848
2849 /*
2850 * Register a custom PASID allocator if we are running in a guest,
2851 * guest PASID must be obtained via virtual command interface.
2852 * There can be multiple vIOMMUs in each guest but only one allocator
2853 * is active. All vIOMMU allocators will eventually be calling the same
2854 * host allocator.
2855 */
2856 if (!vccap_pasid(iommu->vccap))
2857 return;
2858
2859 pr_info("Register custom PASID allocator\n");
2860 iommu->pasid_allocator.alloc = intel_vcmd_ioasid_alloc;
2861 iommu->pasid_allocator.free = intel_vcmd_ioasid_free;
2862 iommu->pasid_allocator.pdata = (void *)iommu;
2863 if (ioasid_register_allocator(&iommu->pasid_allocator)) {
2864 pr_warn("Custom PASID allocator failed, scalable mode disabled\n");
2865 /*
2866 * Disable scalable mode on this IOMMU if there
2867 * is no custom allocator. Mixing SM capable vIOMMU
2868 * and non-SM vIOMMU are not supported.
2869 */
2870 intel_iommu_sm = 0;
2871 }
2872}
2873#endif
2874
2875static int __init init_dmars(void)
2876{
2877 struct dmar_drhd_unit *drhd;
2878 struct intel_iommu *iommu;
2879 int ret;
2880
2881 ret = intel_cap_audit(CAP_AUDIT_STATIC_DMAR, NULL);
2882 if (ret)
2883 goto free_iommu;
2884
2885 for_each_iommu(iommu, drhd) {
2886 if (drhd->ignored) {
2887 iommu_disable_translation(iommu);
2888 continue;
2889 }
2890
2891 /*
2892 * Find the max pasid size of all IOMMU's in the system.
2893 * We need to ensure the system pasid table is no bigger
2894 * than the smallest supported.
2895 */
2896 if (pasid_supported(iommu)) {
2897 u32 temp = 2 << ecap_pss(iommu->ecap);
2898
2899 intel_pasid_max_id = min_t(u32, temp,
2900 intel_pasid_max_id);
2901 }
2902
2903 intel_iommu_init_qi(iommu);
2904
2905 ret = iommu_init_domains(iommu);
2906 if (ret)
2907 goto free_iommu;
2908
2909 init_translation_status(iommu);
2910
2911 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
2912 iommu_disable_translation(iommu);
2913 clear_translation_pre_enabled(iommu);
2914 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
2915 iommu->name);
2916 }
2917
2918 /*
2919 * TBD:
2920 * we could share the same root & context tables
2921 * among all IOMMU's. Need to Split it later.
2922 */
2923 ret = iommu_alloc_root_entry(iommu);
2924 if (ret)
2925 goto free_iommu;
2926
2927 if (translation_pre_enabled(iommu)) {
2928 pr_info("Translation already enabled - trying to copy translation structures\n");
2929
2930 ret = copy_translation_tables(iommu);
2931 if (ret) {
2932 /*
2933 * We found the IOMMU with translation
2934 * enabled - but failed to copy over the
2935 * old root-entry table. Try to proceed
2936 * by disabling translation now and
2937 * allocating a clean root-entry table.
2938 * This might cause DMAR faults, but
2939 * probably the dump will still succeed.
2940 */
2941 pr_err("Failed to copy translation tables from previous kernel for %s\n",
2942 iommu->name);
2943 iommu_disable_translation(iommu);
2944 clear_translation_pre_enabled(iommu);
2945 } else {
2946 pr_info("Copied translation tables from previous kernel for %s\n",
2947 iommu->name);
2948 }
2949 }
2950
2951 if (!ecap_pass_through(iommu->ecap))
2952 hw_pass_through = 0;
2953 intel_svm_check(iommu);
2954 }
2955
2956 /*
2957 * Now that qi is enabled on all iommus, set the root entry and flush
2958 * caches. This is required on some Intel X58 chipsets, otherwise the
2959 * flush_context function will loop forever and the boot hangs.
2960 */
2961 for_each_active_iommu(iommu, drhd) {
2962 iommu_flush_write_buffer(iommu);
2963#ifdef CONFIG_INTEL_IOMMU_SVM
2964 register_pasid_allocator(iommu);
2965#endif
2966 iommu_set_root_entry(iommu);
2967 }
2968
2969#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2970 dmar_map_gfx = 0;
2971#endif
2972
2973 if (!dmar_map_gfx)
2974 iommu_identity_mapping |= IDENTMAP_GFX;
2975
2976 check_tylersburg_isoch();
2977
2978 ret = si_domain_init(hw_pass_through);
2979 if (ret)
2980 goto free_iommu;
2981
2982 /*
2983 * for each drhd
2984 * enable fault log
2985 * global invalidate context cache
2986 * global invalidate iotlb
2987 * enable translation
2988 */
2989 for_each_iommu(iommu, drhd) {
2990 if (drhd->ignored) {
2991 /*
2992 * we always have to disable PMRs or DMA may fail on
2993 * this device
2994 */
2995 if (force_on)
2996 iommu_disable_protect_mem_regions(iommu);
2997 continue;
2998 }
2999
3000 iommu_flush_write_buffer(iommu);
3001
3002#ifdef CONFIG_INTEL_IOMMU_SVM
3003 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
3004 /*
3005 * Call dmar_alloc_hwirq() with dmar_global_lock held,
3006 * could cause possible lock race condition.
3007 */
3008 up_write(&dmar_global_lock);
3009 ret = intel_svm_enable_prq(iommu);
3010 down_write(&dmar_global_lock);
3011 if (ret)
3012 goto free_iommu;
3013 }
3014#endif
3015 ret = dmar_set_interrupt(iommu);
3016 if (ret)
3017 goto free_iommu;
3018 }
3019
3020 return 0;
3021
3022free_iommu:
3023 for_each_active_iommu(iommu, drhd) {
3024 disable_dmar_iommu(iommu);
3025 free_dmar_iommu(iommu);
3026 }
3027 if (si_domain) {
3028 domain_exit(si_domain);
3029 si_domain = NULL;
3030 }
3031
3032 return ret;
3033}
3034
3035static void __init init_no_remapping_devices(void)
3036{
3037 struct dmar_drhd_unit *drhd;
3038 struct device *dev;
3039 int i;
3040
3041 for_each_drhd_unit(drhd) {
3042 if (!drhd->include_all) {
3043 for_each_active_dev_scope(drhd->devices,
3044 drhd->devices_cnt, i, dev)
3045 break;
3046 /* ignore DMAR unit if no devices exist */
3047 if (i == drhd->devices_cnt)
3048 drhd->ignored = 1;
3049 }
3050 }
3051
3052 for_each_active_drhd_unit(drhd) {
3053 if (drhd->include_all)
3054 continue;
3055
3056 for_each_active_dev_scope(drhd->devices,
3057 drhd->devices_cnt, i, dev)
3058 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3059 break;
3060 if (i < drhd->devices_cnt)
3061 continue;
3062
3063 /* This IOMMU has *only* gfx devices. Either bypass it or
3064 set the gfx_mapped flag, as appropriate */
3065 drhd->gfx_dedicated = 1;
3066 if (!dmar_map_gfx)
3067 drhd->ignored = 1;
3068 }
3069}
3070
3071#ifdef CONFIG_SUSPEND
3072static int init_iommu_hw(void)
3073{
3074 struct dmar_drhd_unit *drhd;
3075 struct intel_iommu *iommu = NULL;
3076
3077 for_each_active_iommu(iommu, drhd)
3078 if (iommu->qi)
3079 dmar_reenable_qi(iommu);
3080
3081 for_each_iommu(iommu, drhd) {
3082 if (drhd->ignored) {
3083 /*
3084 * we always have to disable PMRs or DMA may fail on
3085 * this device
3086 */
3087 if (force_on)
3088 iommu_disable_protect_mem_regions(iommu);
3089 continue;
3090 }
3091
3092 iommu_flush_write_buffer(iommu);
3093 iommu_set_root_entry(iommu);
3094 iommu_enable_translation(iommu);
3095 iommu_disable_protect_mem_regions(iommu);
3096 }
3097
3098 return 0;
3099}
3100
3101static void iommu_flush_all(void)
3102{
3103 struct dmar_drhd_unit *drhd;
3104 struct intel_iommu *iommu;
3105
3106 for_each_active_iommu(iommu, drhd) {
3107 iommu->flush.flush_context(iommu, 0, 0, 0,
3108 DMA_CCMD_GLOBAL_INVL);
3109 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3110 DMA_TLB_GLOBAL_FLUSH);
3111 }
3112}
3113
3114static int iommu_suspend(void)
3115{
3116 struct dmar_drhd_unit *drhd;
3117 struct intel_iommu *iommu = NULL;
3118 unsigned long flag;
3119
3120 for_each_active_iommu(iommu, drhd) {
3121 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
3122 GFP_KERNEL);
3123 if (!iommu->iommu_state)
3124 goto nomem;
3125 }
3126
3127 iommu_flush_all();
3128
3129 for_each_active_iommu(iommu, drhd) {
3130 iommu_disable_translation(iommu);
3131
3132 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3133
3134 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3135 readl(iommu->reg + DMAR_FECTL_REG);
3136 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3137 readl(iommu->reg + DMAR_FEDATA_REG);
3138 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3139 readl(iommu->reg + DMAR_FEADDR_REG);
3140 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3141 readl(iommu->reg + DMAR_FEUADDR_REG);
3142
3143 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3144 }
3145 return 0;
3146
3147nomem:
3148 for_each_active_iommu(iommu, drhd)
3149 kfree(iommu->iommu_state);
3150
3151 return -ENOMEM;
3152}
3153
3154static void iommu_resume(void)
3155{
3156 struct dmar_drhd_unit *drhd;
3157 struct intel_iommu *iommu = NULL;
3158 unsigned long flag;
3159
3160 if (init_iommu_hw()) {
3161 if (force_on)
3162 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3163 else
3164 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3165 return;
3166 }
3167
3168 for_each_active_iommu(iommu, drhd) {
3169
3170 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3171
3172 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3173 iommu->reg + DMAR_FECTL_REG);
3174 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3175 iommu->reg + DMAR_FEDATA_REG);
3176 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3177 iommu->reg + DMAR_FEADDR_REG);
3178 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3179 iommu->reg + DMAR_FEUADDR_REG);
3180
3181 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3182 }
3183
3184 for_each_active_iommu(iommu, drhd)
3185 kfree(iommu->iommu_state);
3186}
3187
3188static struct syscore_ops iommu_syscore_ops = {
3189 .resume = iommu_resume,
3190 .suspend = iommu_suspend,
3191};
3192
3193static void __init init_iommu_pm_ops(void)
3194{
3195 register_syscore_ops(&iommu_syscore_ops);
3196}
3197
3198#else
3199static inline void init_iommu_pm_ops(void) {}
3200#endif /* CONFIG_PM */
3201
3202static int __init rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
3203{
3204 if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) ||
3205 !IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) ||
3206 rmrr->end_address <= rmrr->base_address ||
3207 arch_rmrr_sanity_check(rmrr))
3208 return -EINVAL;
3209
3210 return 0;
3211}
3212
3213int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
3214{
3215 struct acpi_dmar_reserved_memory *rmrr;
3216 struct dmar_rmrr_unit *rmrru;
3217
3218 rmrr = (struct acpi_dmar_reserved_memory *)header;
3219 if (rmrr_sanity_check(rmrr)) {
3220 pr_warn(FW_BUG
3221 "Your BIOS is broken; bad RMRR [%#018Lx-%#018Lx]\n"
3222 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3223 rmrr->base_address, rmrr->end_address,
3224 dmi_get_system_info(DMI_BIOS_VENDOR),
3225 dmi_get_system_info(DMI_BIOS_VERSION),
3226 dmi_get_system_info(DMI_PRODUCT_VERSION));
3227 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
3228 }
3229
3230 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3231 if (!rmrru)
3232 goto out;
3233
3234 rmrru->hdr = header;
3235
3236 rmrru->base_address = rmrr->base_address;
3237 rmrru->end_address = rmrr->end_address;
3238
3239 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3240 ((void *)rmrr) + rmrr->header.length,
3241 &rmrru->devices_cnt);
3242 if (rmrru->devices_cnt && rmrru->devices == NULL)
3243 goto free_rmrru;
3244
3245 list_add(&rmrru->list, &dmar_rmrr_units);
3246
3247 return 0;
3248free_rmrru:
3249 kfree(rmrru);
3250out:
3251 return -ENOMEM;
3252}
3253
3254static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3255{
3256 struct dmar_atsr_unit *atsru;
3257 struct acpi_dmar_atsr *tmp;
3258
3259 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list,
3260 dmar_rcu_check()) {
3261 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
3262 if (atsr->segment != tmp->segment)
3263 continue;
3264 if (atsr->header.length != tmp->header.length)
3265 continue;
3266 if (memcmp(atsr, tmp, atsr->header.length) == 0)
3267 return atsru;
3268 }
3269
3270 return NULL;
3271}
3272
3273int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3274{
3275 struct acpi_dmar_atsr *atsr;
3276 struct dmar_atsr_unit *atsru;
3277
3278 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
3279 return 0;
3280
3281 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3282 atsru = dmar_find_atsr(atsr);
3283 if (atsru)
3284 return 0;
3285
3286 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
3287 if (!atsru)
3288 return -ENOMEM;
3289
3290 /*
3291 * If memory is allocated from slab by ACPI _DSM method, we need to
3292 * copy the memory content because the memory buffer will be freed
3293 * on return.
3294 */
3295 atsru->hdr = (void *)(atsru + 1);
3296 memcpy(atsru->hdr, hdr, hdr->length);
3297 atsru->include_all = atsr->flags & 0x1;
3298 if (!atsru->include_all) {
3299 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3300 (void *)atsr + atsr->header.length,
3301 &atsru->devices_cnt);
3302 if (atsru->devices_cnt && atsru->devices == NULL) {
3303 kfree(atsru);
3304 return -ENOMEM;
3305 }
3306 }
3307
3308 list_add_rcu(&atsru->list, &dmar_atsr_units);
3309
3310 return 0;
3311}
3312
3313static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3314{
3315 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3316 kfree(atsru);
3317}
3318
3319int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3320{
3321 struct acpi_dmar_atsr *atsr;
3322 struct dmar_atsr_unit *atsru;
3323
3324 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3325 atsru = dmar_find_atsr(atsr);
3326 if (atsru) {
3327 list_del_rcu(&atsru->list);
3328 synchronize_rcu();
3329 intel_iommu_free_atsr(atsru);
3330 }
3331
3332 return 0;
3333}
3334
3335int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3336{
3337 int i;
3338 struct device *dev;
3339 struct acpi_dmar_atsr *atsr;
3340 struct dmar_atsr_unit *atsru;
3341
3342 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3343 atsru = dmar_find_atsr(atsr);
3344 if (!atsru)
3345 return 0;
3346
3347 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
3348 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
3349 i, dev)
3350 return -EBUSY;
3351 }
3352
3353 return 0;
3354}
3355
3356static struct dmar_satc_unit *dmar_find_satc(struct acpi_dmar_satc *satc)
3357{
3358 struct dmar_satc_unit *satcu;
3359 struct acpi_dmar_satc *tmp;
3360
3361 list_for_each_entry_rcu(satcu, &dmar_satc_units, list,
3362 dmar_rcu_check()) {
3363 tmp = (struct acpi_dmar_satc *)satcu->hdr;
3364 if (satc->segment != tmp->segment)
3365 continue;
3366 if (satc->header.length != tmp->header.length)
3367 continue;
3368 if (memcmp(satc, tmp, satc->header.length) == 0)
3369 return satcu;
3370 }
3371
3372 return NULL;
3373}
3374
3375int dmar_parse_one_satc(struct acpi_dmar_header *hdr, void *arg)
3376{
3377 struct acpi_dmar_satc *satc;
3378 struct dmar_satc_unit *satcu;
3379
3380 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
3381 return 0;
3382
3383 satc = container_of(hdr, struct acpi_dmar_satc, header);
3384 satcu = dmar_find_satc(satc);
3385 if (satcu)
3386 return 0;
3387
3388 satcu = kzalloc(sizeof(*satcu) + hdr->length, GFP_KERNEL);
3389 if (!satcu)
3390 return -ENOMEM;
3391
3392 satcu->hdr = (void *)(satcu + 1);
3393 memcpy(satcu->hdr, hdr, hdr->length);
3394 satcu->atc_required = satc->flags & 0x1;
3395 satcu->devices = dmar_alloc_dev_scope((void *)(satc + 1),
3396 (void *)satc + satc->header.length,
3397 &satcu->devices_cnt);
3398 if (satcu->devices_cnt && !satcu->devices) {
3399 kfree(satcu);
3400 return -ENOMEM;
3401 }
3402 list_add_rcu(&satcu->list, &dmar_satc_units);
3403
3404 return 0;
3405}
3406
3407static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
3408{
3409 int sp, ret;
3410 struct intel_iommu *iommu = dmaru->iommu;
3411
3412 ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_DMAR, iommu);
3413 if (ret)
3414 goto out;
3415
3416 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
3417 pr_warn("%s: Doesn't support hardware pass through.\n",
3418 iommu->name);
3419 return -ENXIO;
3420 }
3421
3422 sp = domain_update_iommu_superpage(NULL, iommu) - 1;
3423 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
3424 pr_warn("%s: Doesn't support large page.\n",
3425 iommu->name);
3426 return -ENXIO;
3427 }
3428
3429 /*
3430 * Disable translation if already enabled prior to OS handover.
3431 */
3432 if (iommu->gcmd & DMA_GCMD_TE)
3433 iommu_disable_translation(iommu);
3434
3435 ret = iommu_init_domains(iommu);
3436 if (ret == 0)
3437 ret = iommu_alloc_root_entry(iommu);
3438 if (ret)
3439 goto out;
3440
3441 intel_svm_check(iommu);
3442
3443 if (dmaru->ignored) {
3444 /*
3445 * we always have to disable PMRs or DMA may fail on this device
3446 */
3447 if (force_on)
3448 iommu_disable_protect_mem_regions(iommu);
3449 return 0;
3450 }
3451
3452 intel_iommu_init_qi(iommu);
3453 iommu_flush_write_buffer(iommu);
3454
3455#ifdef CONFIG_INTEL_IOMMU_SVM
3456 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
3457 ret = intel_svm_enable_prq(iommu);
3458 if (ret)
3459 goto disable_iommu;
3460 }
3461#endif
3462 ret = dmar_set_interrupt(iommu);
3463 if (ret)
3464 goto disable_iommu;
3465
3466 iommu_set_root_entry(iommu);
3467 iommu_enable_translation(iommu);
3468
3469 iommu_disable_protect_mem_regions(iommu);
3470 return 0;
3471
3472disable_iommu:
3473 disable_dmar_iommu(iommu);
3474out:
3475 free_dmar_iommu(iommu);
3476 return ret;
3477}
3478
3479int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
3480{
3481 int ret = 0;
3482 struct intel_iommu *iommu = dmaru->iommu;
3483
3484 if (!intel_iommu_enabled)
3485 return 0;
3486 if (iommu == NULL)
3487 return -EINVAL;
3488
3489 if (insert) {
3490 ret = intel_iommu_add(dmaru);
3491 } else {
3492 disable_dmar_iommu(iommu);
3493 free_dmar_iommu(iommu);
3494 }
3495
3496 return ret;
3497}
3498
3499static void intel_iommu_free_dmars(void)
3500{
3501 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3502 struct dmar_atsr_unit *atsru, *atsr_n;
3503 struct dmar_satc_unit *satcu, *satc_n;
3504
3505 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3506 list_del(&rmrru->list);
3507 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3508 kfree(rmrru);
3509 }
3510
3511 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3512 list_del(&atsru->list);
3513 intel_iommu_free_atsr(atsru);
3514 }
3515 list_for_each_entry_safe(satcu, satc_n, &dmar_satc_units, list) {
3516 list_del(&satcu->list);
3517 dmar_free_dev_scope(&satcu->devices, &satcu->devices_cnt);
3518 kfree(satcu);
3519 }
3520}
3521
3522static struct dmar_satc_unit *dmar_find_matched_satc_unit(struct pci_dev *dev)
3523{
3524 struct dmar_satc_unit *satcu;
3525 struct acpi_dmar_satc *satc;
3526 struct device *tmp;
3527 int i;
3528
3529 dev = pci_physfn(dev);
3530 rcu_read_lock();
3531
3532 list_for_each_entry_rcu(satcu, &dmar_satc_units, list) {
3533 satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
3534 if (satc->segment != pci_domain_nr(dev->bus))
3535 continue;
3536 for_each_dev_scope(satcu->devices, satcu->devices_cnt, i, tmp)
3537 if (to_pci_dev(tmp) == dev)
3538 goto out;
3539 }
3540 satcu = NULL;
3541out:
3542 rcu_read_unlock();
3543 return satcu;
3544}
3545
3546static int dmar_ats_supported(struct pci_dev *dev, struct intel_iommu *iommu)
3547{
3548 int i, ret = 1;
3549 struct pci_bus *bus;
3550 struct pci_dev *bridge = NULL;
3551 struct device *tmp;
3552 struct acpi_dmar_atsr *atsr;
3553 struct dmar_atsr_unit *atsru;
3554 struct dmar_satc_unit *satcu;
3555
3556 dev = pci_physfn(dev);
3557 satcu = dmar_find_matched_satc_unit(dev);
3558 if (satcu)
3559 /*
3560 * This device supports ATS as it is in SATC table.
3561 * When IOMMU is in legacy mode, enabling ATS is done
3562 * automatically by HW for the device that requires
3563 * ATS, hence OS should not enable this device ATS
3564 * to avoid duplicated TLB invalidation.
3565 */
3566 return !(satcu->atc_required && !sm_supported(iommu));
3567
3568 for (bus = dev->bus; bus; bus = bus->parent) {
3569 bridge = bus->self;
3570 /* If it's an integrated device, allow ATS */
3571 if (!bridge)
3572 return 1;
3573 /* Connected via non-PCIe: no ATS */
3574 if (!pci_is_pcie(bridge) ||
3575 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
3576 return 0;
3577 /* If we found the root port, look it up in the ATSR */
3578 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
3579 break;
3580 }
3581
3582 rcu_read_lock();
3583 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3584 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3585 if (atsr->segment != pci_domain_nr(dev->bus))
3586 continue;
3587
3588 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
3589 if (tmp == &bridge->dev)
3590 goto out;
3591
3592 if (atsru->include_all)
3593 goto out;
3594 }
3595 ret = 0;
3596out:
3597 rcu_read_unlock();
3598
3599 return ret;
3600}
3601
3602int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3603{
3604 int ret;
3605 struct dmar_rmrr_unit *rmrru;
3606 struct dmar_atsr_unit *atsru;
3607 struct dmar_satc_unit *satcu;
3608 struct acpi_dmar_atsr *atsr;
3609 struct acpi_dmar_reserved_memory *rmrr;
3610 struct acpi_dmar_satc *satc;
3611
3612 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
3613 return 0;
3614
3615 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3616 rmrr = container_of(rmrru->hdr,
3617 struct acpi_dmar_reserved_memory, header);
3618 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3619 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3620 ((void *)rmrr) + rmrr->header.length,
3621 rmrr->segment, rmrru->devices,
3622 rmrru->devices_cnt);
3623 if (ret < 0)
3624 return ret;
3625 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
3626 dmar_remove_dev_scope(info, rmrr->segment,
3627 rmrru->devices, rmrru->devices_cnt);
3628 }
3629 }
3630
3631 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3632 if (atsru->include_all)
3633 continue;
3634
3635 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3636 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3637 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3638 (void *)atsr + atsr->header.length,
3639 atsr->segment, atsru->devices,
3640 atsru->devices_cnt);
3641 if (ret > 0)
3642 break;
3643 else if (ret < 0)
3644 return ret;
3645 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
3646 if (dmar_remove_dev_scope(info, atsr->segment,
3647 atsru->devices, atsru->devices_cnt))
3648 break;
3649 }
3650 }
3651 list_for_each_entry(satcu, &dmar_satc_units, list) {
3652 satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
3653 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3654 ret = dmar_insert_dev_scope(info, (void *)(satc + 1),
3655 (void *)satc + satc->header.length,
3656 satc->segment, satcu->devices,
3657 satcu->devices_cnt);
3658 if (ret > 0)
3659 break;
3660 else if (ret < 0)
3661 return ret;
3662 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
3663 if (dmar_remove_dev_scope(info, satc->segment,
3664 satcu->devices, satcu->devices_cnt))
3665 break;
3666 }
3667 }
3668
3669 return 0;
3670}
3671
3672static int intel_iommu_memory_notifier(struct notifier_block *nb,
3673 unsigned long val, void *v)
3674{
3675 struct memory_notify *mhp = v;
3676 unsigned long start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
3677 unsigned long last_vpfn = mm_to_dma_pfn(mhp->start_pfn +
3678 mhp->nr_pages - 1);
3679
3680 switch (val) {
3681 case MEM_GOING_ONLINE:
3682 if (iommu_domain_identity_map(si_domain,
3683 start_vpfn, last_vpfn)) {
3684 pr_warn("Failed to build identity map for [%lx-%lx]\n",
3685 start_vpfn, last_vpfn);
3686 return NOTIFY_BAD;
3687 }
3688 break;
3689
3690 case MEM_OFFLINE:
3691 case MEM_CANCEL_ONLINE:
3692 {
3693 struct dmar_drhd_unit *drhd;
3694 struct intel_iommu *iommu;
3695 LIST_HEAD(freelist);
3696
3697 domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist);
3698
3699 rcu_read_lock();
3700 for_each_active_iommu(iommu, drhd)
3701 iommu_flush_iotlb_psi(iommu, si_domain,
3702 start_vpfn, mhp->nr_pages,
3703 list_empty(&freelist), 0);
3704 rcu_read_unlock();
3705 put_pages_list(&freelist);
3706 }
3707 break;
3708 }
3709
3710 return NOTIFY_OK;
3711}
3712
3713static struct notifier_block intel_iommu_memory_nb = {
3714 .notifier_call = intel_iommu_memory_notifier,
3715 .priority = 0
3716};
3717
3718static void intel_disable_iommus(void)
3719{
3720 struct intel_iommu *iommu = NULL;
3721 struct dmar_drhd_unit *drhd;
3722
3723 for_each_iommu(iommu, drhd)
3724 iommu_disable_translation(iommu);
3725}
3726
3727void intel_iommu_shutdown(void)
3728{
3729 struct dmar_drhd_unit *drhd;
3730 struct intel_iommu *iommu = NULL;
3731
3732 if (no_iommu || dmar_disabled)
3733 return;
3734
3735 down_write(&dmar_global_lock);
3736
3737 /* Disable PMRs explicitly here. */
3738 for_each_iommu(iommu, drhd)
3739 iommu_disable_protect_mem_regions(iommu);
3740
3741 /* Make sure the IOMMUs are switched off */
3742 intel_disable_iommus();
3743
3744 up_write(&dmar_global_lock);
3745}
3746
3747static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
3748{
3749 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
3750
3751 return container_of(iommu_dev, struct intel_iommu, iommu);
3752}
3753
3754static ssize_t version_show(struct device *dev,
3755 struct device_attribute *attr, char *buf)
3756{
3757 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3758 u32 ver = readl(iommu->reg + DMAR_VER_REG);
3759 return sprintf(buf, "%d:%d\n",
3760 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
3761}
3762static DEVICE_ATTR_RO(version);
3763
3764static ssize_t address_show(struct device *dev,
3765 struct device_attribute *attr, char *buf)
3766{
3767 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3768 return sprintf(buf, "%llx\n", iommu->reg_phys);
3769}
3770static DEVICE_ATTR_RO(address);
3771
3772static ssize_t cap_show(struct device *dev,
3773 struct device_attribute *attr, char *buf)
3774{
3775 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3776 return sprintf(buf, "%llx\n", iommu->cap);
3777}
3778static DEVICE_ATTR_RO(cap);
3779
3780static ssize_t ecap_show(struct device *dev,
3781 struct device_attribute *attr, char *buf)
3782{
3783 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3784 return sprintf(buf, "%llx\n", iommu->ecap);
3785}
3786static DEVICE_ATTR_RO(ecap);
3787
3788static ssize_t domains_supported_show(struct device *dev,
3789 struct device_attribute *attr, char *buf)
3790{
3791 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3792 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
3793}
3794static DEVICE_ATTR_RO(domains_supported);
3795
3796static ssize_t domains_used_show(struct device *dev,
3797 struct device_attribute *attr, char *buf)
3798{
3799 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3800 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
3801 cap_ndoms(iommu->cap)));
3802}
3803static DEVICE_ATTR_RO(domains_used);
3804
3805static struct attribute *intel_iommu_attrs[] = {
3806 &dev_attr_version.attr,
3807 &dev_attr_address.attr,
3808 &dev_attr_cap.attr,
3809 &dev_attr_ecap.attr,
3810 &dev_attr_domains_supported.attr,
3811 &dev_attr_domains_used.attr,
3812 NULL,
3813};
3814
3815static struct attribute_group intel_iommu_group = {
3816 .name = "intel-iommu",
3817 .attrs = intel_iommu_attrs,
3818};
3819
3820const struct attribute_group *intel_iommu_groups[] = {
3821 &intel_iommu_group,
3822 NULL,
3823};
3824
3825static inline bool has_external_pci(void)
3826{
3827 struct pci_dev *pdev = NULL;
3828
3829 for_each_pci_dev(pdev)
3830 if (pdev->external_facing) {
3831 pci_dev_put(pdev);
3832 return true;
3833 }
3834
3835 return false;
3836}
3837
3838static int __init platform_optin_force_iommu(void)
3839{
3840 if (!dmar_platform_optin() || no_platform_optin || !has_external_pci())
3841 return 0;
3842
3843 if (no_iommu || dmar_disabled)
3844 pr_info("Intel-IOMMU force enabled due to platform opt in\n");
3845
3846 /*
3847 * If Intel-IOMMU is disabled by default, we will apply identity
3848 * map for all devices except those marked as being untrusted.
3849 */
3850 if (dmar_disabled)
3851 iommu_set_default_passthrough(false);
3852
3853 dmar_disabled = 0;
3854 no_iommu = 0;
3855
3856 return 1;
3857}
3858
3859static int __init probe_acpi_namespace_devices(void)
3860{
3861 struct dmar_drhd_unit *drhd;
3862 /* To avoid a -Wunused-but-set-variable warning. */
3863 struct intel_iommu *iommu __maybe_unused;
3864 struct device *dev;
3865 int i, ret = 0;
3866
3867 for_each_active_iommu(iommu, drhd) {
3868 for_each_active_dev_scope(drhd->devices,
3869 drhd->devices_cnt, i, dev) {
3870 struct acpi_device_physical_node *pn;
3871 struct iommu_group *group;
3872 struct acpi_device *adev;
3873
3874 if (dev->bus != &acpi_bus_type)
3875 continue;
3876
3877 adev = to_acpi_device(dev);
3878 mutex_lock(&adev->physical_node_lock);
3879 list_for_each_entry(pn,
3880 &adev->physical_node_list, node) {
3881 group = iommu_group_get(pn->dev);
3882 if (group) {
3883 iommu_group_put(group);
3884 continue;
3885 }
3886
3887 ret = iommu_probe_device(pn->dev);
3888 if (ret)
3889 break;
3890 }
3891 mutex_unlock(&adev->physical_node_lock);
3892
3893 if (ret)
3894 return ret;
3895 }
3896 }
3897
3898 return 0;
3899}
3900
3901static __init int tboot_force_iommu(void)
3902{
3903 if (!tboot_enabled())
3904 return 0;
3905
3906 if (no_iommu || dmar_disabled)
3907 pr_warn("Forcing Intel-IOMMU to enabled\n");
3908
3909 dmar_disabled = 0;
3910 no_iommu = 0;
3911
3912 return 1;
3913}
3914
3915int __init intel_iommu_init(void)
3916{
3917 int ret = -ENODEV;
3918 struct dmar_drhd_unit *drhd;
3919 struct intel_iommu *iommu;
3920
3921 /*
3922 * Intel IOMMU is required for a TXT/tboot launch or platform
3923 * opt in, so enforce that.
3924 */
3925 force_on = (!intel_iommu_tboot_noforce && tboot_force_iommu()) ||
3926 platform_optin_force_iommu();
3927
3928 down_write(&dmar_global_lock);
3929 if (dmar_table_init()) {
3930 if (force_on)
3931 panic("tboot: Failed to initialize DMAR table\n");
3932 goto out_free_dmar;
3933 }
3934
3935 if (dmar_dev_scope_init() < 0) {
3936 if (force_on)
3937 panic("tboot: Failed to initialize DMAR device scope\n");
3938 goto out_free_dmar;
3939 }
3940
3941 up_write(&dmar_global_lock);
3942
3943 /*
3944 * The bus notifier takes the dmar_global_lock, so lockdep will
3945 * complain later when we register it under the lock.
3946 */
3947 dmar_register_bus_notifier();
3948
3949 down_write(&dmar_global_lock);
3950
3951 if (!no_iommu)
3952 intel_iommu_debugfs_init();
3953
3954 if (no_iommu || dmar_disabled) {
3955 /*
3956 * We exit the function here to ensure IOMMU's remapping and
3957 * mempool aren't setup, which means that the IOMMU's PMRs
3958 * won't be disabled via the call to init_dmars(). So disable
3959 * it explicitly here. The PMRs were setup by tboot prior to
3960 * calling SENTER, but the kernel is expected to reset/tear
3961 * down the PMRs.
3962 */
3963 if (intel_iommu_tboot_noforce) {
3964 for_each_iommu(iommu, drhd)
3965 iommu_disable_protect_mem_regions(iommu);
3966 }
3967
3968 /*
3969 * Make sure the IOMMUs are switched off, even when we
3970 * boot into a kexec kernel and the previous kernel left
3971 * them enabled
3972 */
3973 intel_disable_iommus();
3974 goto out_free_dmar;
3975 }
3976
3977 if (list_empty(&dmar_rmrr_units))
3978 pr_info("No RMRR found\n");
3979
3980 if (list_empty(&dmar_atsr_units))
3981 pr_info("No ATSR found\n");
3982
3983 if (list_empty(&dmar_satc_units))
3984 pr_info("No SATC found\n");
3985
3986 init_no_remapping_devices();
3987
3988 ret = init_dmars();
3989 if (ret) {
3990 if (force_on)
3991 panic("tboot: Failed to initialize DMARs\n");
3992 pr_err("Initialization failed\n");
3993 goto out_free_dmar;
3994 }
3995 up_write(&dmar_global_lock);
3996
3997 init_iommu_pm_ops();
3998
3999 down_read(&dmar_global_lock);
4000 for_each_active_iommu(iommu, drhd) {
4001 /*
4002 * The flush queue implementation does not perform
4003 * page-selective invalidations that are required for efficient
4004 * TLB flushes in virtual environments. The benefit of batching
4005 * is likely to be much lower than the overhead of synchronizing
4006 * the virtual and physical IOMMU page-tables.
4007 */
4008 if (cap_caching_mode(iommu->cap)) {
4009 pr_info_once("IOMMU batching disallowed due to virtualization\n");
4010 iommu_set_dma_strict();
4011 }
4012 iommu_device_sysfs_add(&iommu->iommu, NULL,
4013 intel_iommu_groups,
4014 "%s", iommu->name);
4015 iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);
4016 }
4017 up_read(&dmar_global_lock);
4018
4019 if (si_domain && !hw_pass_through)
4020 register_memory_notifier(&intel_iommu_memory_nb);
4021
4022 down_read(&dmar_global_lock);
4023 if (probe_acpi_namespace_devices())
4024 pr_warn("ACPI name space devices didn't probe correctly\n");
4025
4026 /* Finally, we enable the DMA remapping hardware. */
4027 for_each_iommu(iommu, drhd) {
4028 if (!drhd->ignored && !translation_pre_enabled(iommu))
4029 iommu_enable_translation(iommu);
4030
4031 iommu_disable_protect_mem_regions(iommu);
4032 }
4033 up_read(&dmar_global_lock);
4034
4035 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4036
4037 intel_iommu_enabled = 1;
4038
4039 return 0;
4040
4041out_free_dmar:
4042 intel_iommu_free_dmars();
4043 up_write(&dmar_global_lock);
4044 return ret;
4045}
4046
4047static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4048{
4049 struct device_domain_info *info = opaque;
4050
4051 domain_context_clear_one(info, PCI_BUS_NUM(alias), alias & 0xff);
4052 return 0;
4053}
4054
4055/*
4056 * NB - intel-iommu lacks any sort of reference counting for the users of
4057 * dependent devices. If multiple endpoints have intersecting dependent
4058 * devices, unbinding the driver from any one of them will possibly leave
4059 * the others unable to operate.
4060 */
4061static void domain_context_clear(struct device_domain_info *info)
4062{
4063 if (!info->iommu || !info->dev || !dev_is_pci(info->dev))
4064 return;
4065
4066 pci_for_each_dma_alias(to_pci_dev(info->dev),
4067 &domain_context_clear_one_cb, info);
4068}
4069
4070static void dmar_remove_one_dev_info(struct device *dev)
4071{
4072 struct device_domain_info *info = dev_iommu_priv_get(dev);
4073 struct dmar_domain *domain = info->domain;
4074 struct intel_iommu *iommu = info->iommu;
4075 unsigned long flags;
4076
4077 if (!dev_is_real_dma_subdevice(info->dev)) {
4078 if (dev_is_pci(info->dev) && sm_supported(iommu))
4079 intel_pasid_tear_down_entry(iommu, info->dev,
4080 PASID_RID2PASID, false);
4081
4082 iommu_disable_pci_caps(info);
4083 domain_context_clear(info);
4084 }
4085
4086 spin_lock_irqsave(&domain->lock, flags);
4087 list_del(&info->link);
4088 spin_unlock_irqrestore(&domain->lock, flags);
4089
4090 domain_detach_iommu(domain, iommu);
4091 info->domain = NULL;
4092}
4093
4094/*
4095 * Clear the page table pointer in context or pasid table entries so that
4096 * all DMA requests without PASID from the device are blocked. If the page
4097 * table has been set, clean up the data structures.
4098 */
4099static void device_block_translation(struct device *dev)
4100{
4101 struct device_domain_info *info = dev_iommu_priv_get(dev);
4102 struct intel_iommu *iommu = info->iommu;
4103 unsigned long flags;
4104
4105 iommu_disable_pci_caps(info);
4106 if (!dev_is_real_dma_subdevice(dev)) {
4107 if (sm_supported(iommu))
4108 intel_pasid_tear_down_entry(iommu, dev,
4109 PASID_RID2PASID, false);
4110 else
4111 domain_context_clear(info);
4112 }
4113
4114 if (!info->domain)
4115 return;
4116
4117 spin_lock_irqsave(&info->domain->lock, flags);
4118 list_del(&info->link);
4119 spin_unlock_irqrestore(&info->domain->lock, flags);
4120
4121 domain_detach_iommu(info->domain, iommu);
4122 info->domain = NULL;
4123}
4124
4125static int md_domain_init(struct dmar_domain *domain, int guest_width)
4126{
4127 int adjust_width;
4128
4129 /* calculate AGAW */
4130 domain->gaw = guest_width;
4131 adjust_width = guestwidth_to_adjustwidth(guest_width);
4132 domain->agaw = width_to_agaw(adjust_width);
4133
4134 domain->iommu_coherency = false;
4135 domain->iommu_superpage = 0;
4136 domain->max_addr = 0;
4137
4138 /* always allocate the top pgd */
4139 domain->pgd = alloc_pgtable_page(domain->nid);
4140 if (!domain->pgd)
4141 return -ENOMEM;
4142 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4143 return 0;
4144}
4145
4146static int blocking_domain_attach_dev(struct iommu_domain *domain,
4147 struct device *dev)
4148{
4149 device_block_translation(dev);
4150 return 0;
4151}
4152
4153static struct iommu_domain blocking_domain = {
4154 .ops = &(const struct iommu_domain_ops) {
4155 .attach_dev = blocking_domain_attach_dev,
4156 .free = intel_iommu_domain_free
4157 }
4158};
4159
4160static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4161{
4162 struct dmar_domain *dmar_domain;
4163 struct iommu_domain *domain;
4164
4165 switch (type) {
4166 case IOMMU_DOMAIN_BLOCKED:
4167 return &blocking_domain;
4168 case IOMMU_DOMAIN_DMA:
4169 case IOMMU_DOMAIN_DMA_FQ:
4170 case IOMMU_DOMAIN_UNMANAGED:
4171 dmar_domain = alloc_domain(type);
4172 if (!dmar_domain) {
4173 pr_err("Can't allocate dmar_domain\n");
4174 return NULL;
4175 }
4176 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4177 pr_err("Domain initialization failed\n");
4178 domain_exit(dmar_domain);
4179 return NULL;
4180 }
4181
4182 domain = &dmar_domain->domain;
4183 domain->geometry.aperture_start = 0;
4184 domain->geometry.aperture_end =
4185 __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4186 domain->geometry.force_aperture = true;
4187
4188 return domain;
4189 case IOMMU_DOMAIN_IDENTITY:
4190 return &si_domain->domain;
4191 case IOMMU_DOMAIN_SVA:
4192 return intel_svm_domain_alloc();
4193 default:
4194 return NULL;
4195 }
4196
4197 return NULL;
4198}
4199
4200static void intel_iommu_domain_free(struct iommu_domain *domain)
4201{
4202 if (domain != &si_domain->domain && domain != &blocking_domain)
4203 domain_exit(to_dmar_domain(domain));
4204}
4205
4206static int prepare_domain_attach_device(struct iommu_domain *domain,
4207 struct device *dev)
4208{
4209 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4210 struct intel_iommu *iommu;
4211 int addr_width;
4212
4213 iommu = device_to_iommu(dev, NULL, NULL);
4214 if (!iommu)
4215 return -ENODEV;
4216
4217 if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
4218 return -EINVAL;
4219
4220 /* check if this iommu agaw is sufficient for max mapped address */
4221 addr_width = agaw_to_width(iommu->agaw);
4222 if (addr_width > cap_mgaw(iommu->cap))
4223 addr_width = cap_mgaw(iommu->cap);
4224
4225 if (dmar_domain->max_addr > (1LL << addr_width))
4226 return -EINVAL;
4227 dmar_domain->gaw = addr_width;
4228
4229 /*
4230 * Knock out extra levels of page tables if necessary
4231 */
4232 while (iommu->agaw < dmar_domain->agaw) {
4233 struct dma_pte *pte;
4234
4235 pte = dmar_domain->pgd;
4236 if (dma_pte_present(pte)) {
4237 dmar_domain->pgd = phys_to_virt(dma_pte_addr(pte));
4238 free_pgtable_page(pte);
4239 }
4240 dmar_domain->agaw--;
4241 }
4242
4243 return 0;
4244}
4245
4246static int intel_iommu_attach_device(struct iommu_domain *domain,
4247 struct device *dev)
4248{
4249 struct device_domain_info *info = dev_iommu_priv_get(dev);
4250 int ret;
4251
4252 if (domain->type == IOMMU_DOMAIN_UNMANAGED &&
4253 device_is_rmrr_locked(dev)) {
4254 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4255 return -EPERM;
4256 }
4257
4258 if (info->domain)
4259 device_block_translation(dev);
4260
4261 ret = prepare_domain_attach_device(domain, dev);
4262 if (ret)
4263 return ret;
4264
4265 return dmar_domain_attach_device(to_dmar_domain(domain), dev);
4266}
4267
4268static int intel_iommu_map(struct iommu_domain *domain,
4269 unsigned long iova, phys_addr_t hpa,
4270 size_t size, int iommu_prot, gfp_t gfp)
4271{
4272 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4273 u64 max_addr;
4274 int prot = 0;
4275
4276 if (iommu_prot & IOMMU_READ)
4277 prot |= DMA_PTE_READ;
4278 if (iommu_prot & IOMMU_WRITE)
4279 prot |= DMA_PTE_WRITE;
4280 if (dmar_domain->set_pte_snp)
4281 prot |= DMA_PTE_SNP;
4282
4283 max_addr = iova + size;
4284 if (dmar_domain->max_addr < max_addr) {
4285 u64 end;
4286
4287 /* check if minimum agaw is sufficient for mapped address */
4288 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4289 if (end < max_addr) {
4290 pr_err("%s: iommu width (%d) is not "
4291 "sufficient for the mapped address (%llx)\n",
4292 __func__, dmar_domain->gaw, max_addr);
4293 return -EFAULT;
4294 }
4295 dmar_domain->max_addr = max_addr;
4296 }
4297 /* Round up size to next multiple of PAGE_SIZE, if it and
4298 the low bits of hpa would take us onto the next page */
4299 size = aligned_nrpages(hpa, size);
4300 return __domain_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4301 hpa >> VTD_PAGE_SHIFT, size, prot);
4302}
4303
4304static int intel_iommu_map_pages(struct iommu_domain *domain,
4305 unsigned long iova, phys_addr_t paddr,
4306 size_t pgsize, size_t pgcount,
4307 int prot, gfp_t gfp, size_t *mapped)
4308{
4309 unsigned long pgshift = __ffs(pgsize);
4310 size_t size = pgcount << pgshift;
4311 int ret;
4312
4313 if (pgsize != SZ_4K && pgsize != SZ_2M && pgsize != SZ_1G)
4314 return -EINVAL;
4315
4316 if (!IS_ALIGNED(iova | paddr, pgsize))
4317 return -EINVAL;
4318
4319 ret = intel_iommu_map(domain, iova, paddr, size, prot, gfp);
4320 if (!ret && mapped)
4321 *mapped = size;
4322
4323 return ret;
4324}
4325
4326static size_t intel_iommu_unmap(struct iommu_domain *domain,
4327 unsigned long iova, size_t size,
4328 struct iommu_iotlb_gather *gather)
4329{
4330 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4331 unsigned long start_pfn, last_pfn;
4332 int level = 0;
4333
4334 /* Cope with horrid API which requires us to unmap more than the
4335 size argument if it happens to be a large-page mapping. */
4336 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
4337
4338 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4339 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4340
4341 start_pfn = iova >> VTD_PAGE_SHIFT;
4342 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4343
4344 domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist);
4345
4346 if (dmar_domain->max_addr == iova + size)
4347 dmar_domain->max_addr = iova;
4348
4349 iommu_iotlb_gather_add_page(domain, gather, iova, size);
4350
4351 return size;
4352}
4353
4354static size_t intel_iommu_unmap_pages(struct iommu_domain *domain,
4355 unsigned long iova,
4356 size_t pgsize, size_t pgcount,
4357 struct iommu_iotlb_gather *gather)
4358{
4359 unsigned long pgshift = __ffs(pgsize);
4360 size_t size = pgcount << pgshift;
4361
4362 return intel_iommu_unmap(domain, iova, size, gather);
4363}
4364
4365static void intel_iommu_tlb_sync(struct iommu_domain *domain,
4366 struct iommu_iotlb_gather *gather)
4367{
4368 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4369 unsigned long iova_pfn = IOVA_PFN(gather->start);
4370 size_t size = gather->end - gather->start;
4371 struct iommu_domain_info *info;
4372 unsigned long start_pfn;
4373 unsigned long nrpages;
4374 unsigned long i;
4375
4376 nrpages = aligned_nrpages(gather->start, size);
4377 start_pfn = mm_to_dma_pfn(iova_pfn);
4378
4379 xa_for_each(&dmar_domain->iommu_array, i, info)
4380 iommu_flush_iotlb_psi(info->iommu, dmar_domain,
4381 start_pfn, nrpages,
4382 list_empty(&gather->freelist), 0);
4383
4384 put_pages_list(&gather->freelist);
4385}
4386
4387static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4388 dma_addr_t iova)
4389{
4390 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4391 struct dma_pte *pte;
4392 int level = 0;
4393 u64 phys = 0;
4394
4395 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
4396 if (pte && dma_pte_present(pte))
4397 phys = dma_pte_addr(pte) +
4398 (iova & (BIT_MASK(level_to_offset_bits(level) +
4399 VTD_PAGE_SHIFT) - 1));
4400
4401 return phys;
4402}
4403
4404static bool domain_support_force_snooping(struct dmar_domain *domain)
4405{
4406 struct device_domain_info *info;
4407 bool support = true;
4408
4409 assert_spin_locked(&domain->lock);
4410 list_for_each_entry(info, &domain->devices, link) {
4411 if (!ecap_sc_support(info->iommu->ecap)) {
4412 support = false;
4413 break;
4414 }
4415 }
4416
4417 return support;
4418}
4419
4420static void domain_set_force_snooping(struct dmar_domain *domain)
4421{
4422 struct device_domain_info *info;
4423
4424 assert_spin_locked(&domain->lock);
4425 /*
4426 * Second level page table supports per-PTE snoop control. The
4427 * iommu_map() interface will handle this by setting SNP bit.
4428 */
4429 if (!domain->use_first_level) {
4430 domain->set_pte_snp = true;
4431 return;
4432 }
4433
4434 list_for_each_entry(info, &domain->devices, link)
4435 intel_pasid_setup_page_snoop_control(info->iommu, info->dev,
4436 PASID_RID2PASID);
4437}
4438
4439static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain)
4440{
4441 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4442 unsigned long flags;
4443
4444 if (dmar_domain->force_snooping)
4445 return true;
4446
4447 spin_lock_irqsave(&dmar_domain->lock, flags);
4448 if (!domain_support_force_snooping(dmar_domain)) {
4449 spin_unlock_irqrestore(&dmar_domain->lock, flags);
4450 return false;
4451 }
4452
4453 domain_set_force_snooping(dmar_domain);
4454 dmar_domain->force_snooping = true;
4455 spin_unlock_irqrestore(&dmar_domain->lock, flags);
4456
4457 return true;
4458}
4459
4460static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap)
4461{
4462 struct device_domain_info *info = dev_iommu_priv_get(dev);
4463
4464 switch (cap) {
4465 case IOMMU_CAP_CACHE_COHERENCY:
4466 return true;
4467 case IOMMU_CAP_INTR_REMAP:
4468 return irq_remapping_enabled == 1;
4469 case IOMMU_CAP_PRE_BOOT_PROTECTION:
4470 return dmar_platform_optin();
4471 case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
4472 return ecap_sc_support(info->iommu->ecap);
4473 default:
4474 return false;
4475 }
4476}
4477
4478static struct iommu_device *intel_iommu_probe_device(struct device *dev)
4479{
4480 struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;
4481 struct device_domain_info *info;
4482 struct intel_iommu *iommu;
4483 u8 bus, devfn;
4484 int ret;
4485
4486 iommu = device_to_iommu(dev, &bus, &devfn);
4487 if (!iommu || !iommu->iommu.ops)
4488 return ERR_PTR(-ENODEV);
4489
4490 info = kzalloc(sizeof(*info), GFP_KERNEL);
4491 if (!info)
4492 return ERR_PTR(-ENOMEM);
4493
4494 if (dev_is_real_dma_subdevice(dev)) {
4495 info->bus = pdev->bus->number;
4496 info->devfn = pdev->devfn;
4497 info->segment = pci_domain_nr(pdev->bus);
4498 } else {
4499 info->bus = bus;
4500 info->devfn = devfn;
4501 info->segment = iommu->segment;
4502 }
4503
4504 info->dev = dev;
4505 info->iommu = iommu;
4506 if (dev_is_pci(dev)) {
4507 if (ecap_dev_iotlb_support(iommu->ecap) &&
4508 pci_ats_supported(pdev) &&
4509 dmar_ats_supported(pdev, iommu)) {
4510 info->ats_supported = 1;
4511 info->dtlb_extra_inval = dev_needs_extra_dtlb_flush(pdev);
4512 }
4513 if (sm_supported(iommu)) {
4514 if (pasid_supported(iommu)) {
4515 int features = pci_pasid_features(pdev);
4516
4517 if (features >= 0)
4518 info->pasid_supported = features | 1;
4519 }
4520
4521 if (info->ats_supported && ecap_prs(iommu->ecap) &&
4522 pci_pri_supported(pdev))
4523 info->pri_supported = 1;
4524 }
4525 }
4526
4527 dev_iommu_priv_set(dev, info);
4528
4529 if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
4530 ret = intel_pasid_alloc_table(dev);
4531 if (ret) {
4532 dev_err(dev, "PASID table allocation failed\n");
4533 dev_iommu_priv_set(dev, NULL);
4534 kfree(info);
4535 return ERR_PTR(ret);
4536 }
4537 }
4538
4539 return &iommu->iommu;
4540}
4541
4542static void intel_iommu_release_device(struct device *dev)
4543{
4544 struct device_domain_info *info = dev_iommu_priv_get(dev);
4545
4546 dmar_remove_one_dev_info(dev);
4547 intel_pasid_free_table(dev);
4548 dev_iommu_priv_set(dev, NULL);
4549 kfree(info);
4550 set_dma_ops(dev, NULL);
4551}
4552
4553static void intel_iommu_probe_finalize(struct device *dev)
4554{
4555 set_dma_ops(dev, NULL);
4556 iommu_setup_dma_ops(dev, 0, U64_MAX);
4557}
4558
4559static void intel_iommu_get_resv_regions(struct device *device,
4560 struct list_head *head)
4561{
4562 int prot = DMA_PTE_READ | DMA_PTE_WRITE;
4563 struct iommu_resv_region *reg;
4564 struct dmar_rmrr_unit *rmrr;
4565 struct device *i_dev;
4566 int i;
4567
4568 rcu_read_lock();
4569 for_each_rmrr_units(rmrr) {
4570 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
4571 i, i_dev) {
4572 struct iommu_resv_region *resv;
4573 enum iommu_resv_type type;
4574 size_t length;
4575
4576 if (i_dev != device &&
4577 !is_downstream_to_pci_bridge(device, i_dev))
4578 continue;
4579
4580 length = rmrr->end_address - rmrr->base_address + 1;
4581
4582 type = device_rmrr_is_relaxable(device) ?
4583 IOMMU_RESV_DIRECT_RELAXABLE : IOMMU_RESV_DIRECT;
4584
4585 resv = iommu_alloc_resv_region(rmrr->base_address,
4586 length, prot, type,
4587 GFP_ATOMIC);
4588 if (!resv)
4589 break;
4590
4591 list_add_tail(&resv->list, head);
4592 }
4593 }
4594 rcu_read_unlock();
4595
4596#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
4597 if (dev_is_pci(device)) {
4598 struct pci_dev *pdev = to_pci_dev(device);
4599
4600 if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) {
4601 reg = iommu_alloc_resv_region(0, 1UL << 24, prot,
4602 IOMMU_RESV_DIRECT_RELAXABLE,
4603 GFP_KERNEL);
4604 if (reg)
4605 list_add_tail(®->list, head);
4606 }
4607 }
4608#endif /* CONFIG_INTEL_IOMMU_FLOPPY_WA */
4609
4610 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
4611 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
4612 0, IOMMU_RESV_MSI, GFP_KERNEL);
4613 if (!reg)
4614 return;
4615 list_add_tail(®->list, head);
4616}
4617
4618static struct iommu_group *intel_iommu_device_group(struct device *dev)
4619{
4620 if (dev_is_pci(dev))
4621 return pci_device_group(dev);
4622 return generic_device_group(dev);
4623}
4624
4625static int intel_iommu_enable_sva(struct device *dev)
4626{
4627 struct device_domain_info *info = dev_iommu_priv_get(dev);
4628 struct intel_iommu *iommu;
4629 int ret;
4630
4631 if (!info || dmar_disabled)
4632 return -EINVAL;
4633
4634 iommu = info->iommu;
4635 if (!iommu)
4636 return -EINVAL;
4637
4638 if (!(iommu->flags & VTD_FLAG_SVM_CAPABLE))
4639 return -ENODEV;
4640
4641 if (!info->pasid_enabled || !info->pri_enabled || !info->ats_enabled)
4642 return -EINVAL;
4643
4644 ret = iopf_queue_add_device(iommu->iopf_queue, dev);
4645 if (!ret)
4646 ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev);
4647
4648 return ret;
4649}
4650
4651static int intel_iommu_disable_sva(struct device *dev)
4652{
4653 struct device_domain_info *info = dev_iommu_priv_get(dev);
4654 struct intel_iommu *iommu = info->iommu;
4655 int ret;
4656
4657 ret = iommu_unregister_device_fault_handler(dev);
4658 if (!ret)
4659 ret = iopf_queue_remove_device(iommu->iopf_queue, dev);
4660
4661 return ret;
4662}
4663
4664static int intel_iommu_enable_iopf(struct device *dev)
4665{
4666 struct device_domain_info *info = dev_iommu_priv_get(dev);
4667
4668 if (info && info->pri_supported)
4669 return 0;
4670
4671 return -ENODEV;
4672}
4673
4674static int
4675intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
4676{
4677 switch (feat) {
4678 case IOMMU_DEV_FEAT_IOPF:
4679 return intel_iommu_enable_iopf(dev);
4680
4681 case IOMMU_DEV_FEAT_SVA:
4682 return intel_iommu_enable_sva(dev);
4683
4684 default:
4685 return -ENODEV;
4686 }
4687}
4688
4689static int
4690intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
4691{
4692 switch (feat) {
4693 case IOMMU_DEV_FEAT_IOPF:
4694 return 0;
4695
4696 case IOMMU_DEV_FEAT_SVA:
4697 return intel_iommu_disable_sva(dev);
4698
4699 default:
4700 return -ENODEV;
4701 }
4702}
4703
4704static bool intel_iommu_is_attach_deferred(struct device *dev)
4705{
4706 struct device_domain_info *info = dev_iommu_priv_get(dev);
4707
4708 return translation_pre_enabled(info->iommu) && !info->domain;
4709}
4710
4711/*
4712 * Check that the device does not live on an external facing PCI port that is
4713 * marked as untrusted. Such devices should not be able to apply quirks and
4714 * thus not be able to bypass the IOMMU restrictions.
4715 */
4716static bool risky_device(struct pci_dev *pdev)
4717{
4718 if (pdev->untrusted) {
4719 pci_info(pdev,
4720 "Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n",
4721 pdev->vendor, pdev->device);
4722 pci_info(pdev, "Please check with your BIOS/Platform vendor about this\n");
4723 return true;
4724 }
4725 return false;
4726}
4727
4728static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
4729 unsigned long iova, size_t size)
4730{
4731 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4732 unsigned long pages = aligned_nrpages(iova, size);
4733 unsigned long pfn = iova >> VTD_PAGE_SHIFT;
4734 struct iommu_domain_info *info;
4735 unsigned long i;
4736
4737 xa_for_each(&dmar_domain->iommu_array, i, info)
4738 __mapping_notify_one(info->iommu, dmar_domain, pfn, pages);
4739}
4740
4741static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
4742{
4743 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
4744 struct iommu_domain *domain;
4745
4746 /* Domain type specific cleanup: */
4747 domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0);
4748 if (domain) {
4749 switch (domain->type) {
4750 case IOMMU_DOMAIN_SVA:
4751 intel_svm_remove_dev_pasid(dev, pasid);
4752 break;
4753 default:
4754 /* should never reach here */
4755 WARN_ON(1);
4756 break;
4757 }
4758 }
4759
4760 intel_pasid_tear_down_entry(iommu, dev, pasid, false);
4761}
4762
4763const struct iommu_ops intel_iommu_ops = {
4764 .capable = intel_iommu_capable,
4765 .domain_alloc = intel_iommu_domain_alloc,
4766 .probe_device = intel_iommu_probe_device,
4767 .probe_finalize = intel_iommu_probe_finalize,
4768 .release_device = intel_iommu_release_device,
4769 .get_resv_regions = intel_iommu_get_resv_regions,
4770 .device_group = intel_iommu_device_group,
4771 .dev_enable_feat = intel_iommu_dev_enable_feat,
4772 .dev_disable_feat = intel_iommu_dev_disable_feat,
4773 .is_attach_deferred = intel_iommu_is_attach_deferred,
4774 .def_domain_type = device_def_domain_type,
4775 .remove_dev_pasid = intel_iommu_remove_dev_pasid,
4776 .pgsize_bitmap = SZ_4K,
4777#ifdef CONFIG_INTEL_IOMMU_SVM
4778 .page_response = intel_svm_page_response,
4779#endif
4780 .default_domain_ops = &(const struct iommu_domain_ops) {
4781 .attach_dev = intel_iommu_attach_device,
4782 .map_pages = intel_iommu_map_pages,
4783 .unmap_pages = intel_iommu_unmap_pages,
4784 .iotlb_sync_map = intel_iommu_iotlb_sync_map,
4785 .flush_iotlb_all = intel_flush_iotlb_all,
4786 .iotlb_sync = intel_iommu_tlb_sync,
4787 .iova_to_phys = intel_iommu_iova_to_phys,
4788 .free = intel_iommu_domain_free,
4789 .enforce_cache_coherency = intel_iommu_enforce_cache_coherency,
4790 }
4791};
4792
4793static void quirk_iommu_igfx(struct pci_dev *dev)
4794{
4795 if (risky_device(dev))
4796 return;
4797
4798 pci_info(dev, "Disabling IOMMU for graphics on this chipset\n");
4799 dmar_map_gfx = 0;
4800}
4801
4802/* G4x/GM45 integrated gfx dmar support is totally busted. */
4803DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx);
4804DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx);
4805DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_igfx);
4806DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_igfx);
4807DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_igfx);
4808DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_igfx);
4809DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_igfx);
4810
4811/* Broadwell igfx malfunctions with dmar */
4812DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1606, quirk_iommu_igfx);
4813DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160B, quirk_iommu_igfx);
4814DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160E, quirk_iommu_igfx);
4815DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1602, quirk_iommu_igfx);
4816DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160A, quirk_iommu_igfx);
4817DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160D, quirk_iommu_igfx);
4818DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1616, quirk_iommu_igfx);
4819DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161B, quirk_iommu_igfx);
4820DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161E, quirk_iommu_igfx);
4821DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1612, quirk_iommu_igfx);
4822DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161A, quirk_iommu_igfx);
4823DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161D, quirk_iommu_igfx);
4824DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1626, quirk_iommu_igfx);
4825DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162B, quirk_iommu_igfx);
4826DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162E, quirk_iommu_igfx);
4827DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1622, quirk_iommu_igfx);
4828DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162A, quirk_iommu_igfx);
4829DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162D, quirk_iommu_igfx);
4830DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1636, quirk_iommu_igfx);
4831DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163B, quirk_iommu_igfx);
4832DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163E, quirk_iommu_igfx);
4833DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1632, quirk_iommu_igfx);
4834DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163A, quirk_iommu_igfx);
4835DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163D, quirk_iommu_igfx);
4836
4837static void quirk_iommu_rwbf(struct pci_dev *dev)
4838{
4839 if (risky_device(dev))
4840 return;
4841
4842 /*
4843 * Mobile 4 Series Chipset neglects to set RWBF capability,
4844 * but needs it. Same seems to hold for the desktop versions.
4845 */
4846 pci_info(dev, "Forcing write-buffer flush capability\n");
4847 rwbf_quirk = 1;
4848}
4849
4850DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4851DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4852DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4853DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4854DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4855DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4856DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4857
4858#define GGC 0x52
4859#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4860#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4861#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4862#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4863#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4864#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4865#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4866#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4867
4868static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4869{
4870 unsigned short ggc;
4871
4872 if (risky_device(dev))
4873 return;
4874
4875 if (pci_read_config_word(dev, GGC, &ggc))
4876 return;
4877
4878 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4879 pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4880 dmar_map_gfx = 0;
4881 } else if (dmar_map_gfx) {
4882 /* we have to ensure the gfx device is idle before we flush */
4883 pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n");
4884 iommu_set_dma_strict();
4885 }
4886}
4887DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4888DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4889DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4890DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4891
4892static void quirk_igfx_skip_te_disable(struct pci_dev *dev)
4893{
4894 unsigned short ver;
4895
4896 if (!IS_GFX_DEVICE(dev))
4897 return;
4898
4899 ver = (dev->device >> 8) & 0xff;
4900 if (ver != 0x45 && ver != 0x46 && ver != 0x4c &&
4901 ver != 0x4e && ver != 0x8a && ver != 0x98 &&
4902 ver != 0x9a && ver != 0xa7)
4903 return;
4904
4905 if (risky_device(dev))
4906 return;
4907
4908 pci_info(dev, "Skip IOMMU disabling for graphics\n");
4909 iommu_skip_te_disable = 1;
4910}
4911DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, quirk_igfx_skip_te_disable);
4912
4913/* On Tylersburg chipsets, some BIOSes have been known to enable the
4914 ISOCH DMAR unit for the Azalia sound device, but not give it any
4915 TLB entries, which causes it to deadlock. Check for that. We do
4916 this in a function called from init_dmars(), instead of in a PCI
4917 quirk, because we don't want to print the obnoxious "BIOS broken"
4918 message if VT-d is actually disabled.
4919*/
4920static void __init check_tylersburg_isoch(void)
4921{
4922 struct pci_dev *pdev;
4923 uint32_t vtisochctrl;
4924
4925 /* If there's no Azalia in the system anyway, forget it. */
4926 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4927 if (!pdev)
4928 return;
4929
4930 if (risky_device(pdev)) {
4931 pci_dev_put(pdev);
4932 return;
4933 }
4934
4935 pci_dev_put(pdev);
4936
4937 /* System Management Registers. Might be hidden, in which case
4938 we can't do the sanity check. But that's OK, because the
4939 known-broken BIOSes _don't_ actually hide it, so far. */
4940 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4941 if (!pdev)
4942 return;
4943
4944 if (risky_device(pdev)) {
4945 pci_dev_put(pdev);
4946 return;
4947 }
4948
4949 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4950 pci_dev_put(pdev);
4951 return;
4952 }
4953
4954 pci_dev_put(pdev);
4955
4956 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4957 if (vtisochctrl & 1)
4958 return;
4959
4960 /* Drop all bits other than the number of TLB entries */
4961 vtisochctrl &= 0x1c;
4962
4963 /* If we have the recommended number of TLB entries (16), fine. */
4964 if (vtisochctrl == 0x10)
4965 return;
4966
4967 /* Zero TLB entries? You get to ride the short bus to school. */
4968 if (!vtisochctrl) {
4969 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4970 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4971 dmi_get_system_info(DMI_BIOS_VENDOR),
4972 dmi_get_system_info(DMI_BIOS_VERSION),
4973 dmi_get_system_info(DMI_PRODUCT_VERSION));
4974 iommu_identity_mapping |= IDENTMAP_AZALIA;
4975 return;
4976 }
4977
4978 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4979 vtisochctrl);
4980}
4981
4982/*
4983 * Here we deal with a device TLB defect where device may inadvertently issue ATS
4984 * invalidation completion before posted writes initiated with translated address
4985 * that utilized translations matching the invalidation address range, violating
4986 * the invalidation completion ordering.
4987 * Therefore, any use cases that cannot guarantee DMA is stopped before unmap is
4988 * vulnerable to this defect. In other words, any dTLB invalidation initiated not
4989 * under the control of the trusted/privileged host device driver must use this
4990 * quirk.
4991 * Device TLBs are invalidated under the following six conditions:
4992 * 1. Device driver does DMA API unmap IOVA
4993 * 2. Device driver unbind a PASID from a process, sva_unbind_device()
4994 * 3. PASID is torn down, after PASID cache is flushed. e.g. process
4995 * exit_mmap() due to crash
4996 * 4. Under SVA usage, called by mmu_notifier.invalidate_range() where
4997 * VM has to free pages that were unmapped
4998 * 5. Userspace driver unmaps a DMA buffer
4999 * 6. Cache invalidation in vSVA usage (upcoming)
5000 *
5001 * For #1 and #2, device drivers are responsible for stopping DMA traffic
5002 * before unmap/unbind. For #3, iommu driver gets mmu_notifier to
5003 * invalidate TLB the same way as normal user unmap which will use this quirk.
5004 * The dTLB invalidation after PASID cache flush does not need this quirk.
5005 *
5006 * As a reminder, #6 will *NEED* this quirk as we enable nested translation.
5007 */
5008void quirk_extra_dev_tlb_flush(struct device_domain_info *info,
5009 unsigned long address, unsigned long mask,
5010 u32 pasid, u16 qdep)
5011{
5012 u16 sid;
5013
5014 if (likely(!info->dtlb_extra_inval))
5015 return;
5016
5017 sid = PCI_DEVID(info->bus, info->devfn);
5018 if (pasid == PASID_RID2PASID) {
5019 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
5020 qdep, address, mask);
5021 } else {
5022 qi_flush_dev_iotlb_pasid(info->iommu, sid, info->pfsid,
5023 pasid, qdep, address, mask);
5024 }
5025}