Loading...
Note: File does not exist in v5.4.
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * Copyright © 2006-2014 Intel Corporation.
4 *
5 * Authors: David Woodhouse <dwmw2@infradead.org>,
6 * Ashok Raj <ashok.raj@intel.com>,
7 * Shaohua Li <shaohua.li@intel.com>,
8 * Anil S Keshavamurthy <anil.s.keshavamurthy@intel.com>,
9 * Fenghua Yu <fenghua.yu@intel.com>
10 * Joerg Roedel <jroedel@suse.de>
11 */
12
13#define pr_fmt(fmt) "DMAR: " fmt
14#define dev_fmt(fmt) pr_fmt(fmt)
15
16#include <linux/crash_dump.h>
17#include <linux/dma-direct.h>
18#include <linux/dmi.h>
19#include <linux/intel-svm.h>
20#include <linux/memory.h>
21#include <linux/pci.h>
22#include <linux/pci-ats.h>
23#include <linux/spinlock.h>
24#include <linux/syscore_ops.h>
25#include <linux/tboot.h>
26
27#include "iommu.h"
28#include "../dma-iommu.h"
29#include "../irq_remapping.h"
30#include "../iommu-sva.h"
31#include "pasid.h"
32#include "cap_audit.h"
33
34#define ROOT_SIZE VTD_PAGE_SIZE
35#define CONTEXT_SIZE VTD_PAGE_SIZE
36
37#define IS_GFX_DEVICE(pdev) ((pdev->class >> 16) == PCI_BASE_CLASS_DISPLAY)
38#define IS_USB_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_SERIAL_USB)
39#define IS_ISA_DEVICE(pdev) ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA)
40#define IS_AZALIA(pdev) ((pdev)->vendor == 0x8086 && (pdev)->device == 0x3a3e)
41
42#define IOAPIC_RANGE_START (0xfee00000)
43#define IOAPIC_RANGE_END (0xfeefffff)
44#define IOVA_START_ADDR (0x1000)
45
46#define DEFAULT_DOMAIN_ADDRESS_WIDTH 57
47
48#define MAX_AGAW_WIDTH 64
49#define MAX_AGAW_PFN_WIDTH (MAX_AGAW_WIDTH - VTD_PAGE_SHIFT)
50
51#define __DOMAIN_MAX_PFN(gaw) ((((uint64_t)1) << ((gaw) - VTD_PAGE_SHIFT)) - 1)
52#define __DOMAIN_MAX_ADDR(gaw) ((((uint64_t)1) << (gaw)) - 1)
53
54/* We limit DOMAIN_MAX_PFN to fit in an unsigned long, and DOMAIN_MAX_ADDR
55 to match. That way, we can use 'unsigned long' for PFNs with impunity. */
56#define DOMAIN_MAX_PFN(gaw) ((unsigned long) min_t(uint64_t, \
57 __DOMAIN_MAX_PFN(gaw), (unsigned long)-1))
58#define DOMAIN_MAX_ADDR(gaw) (((uint64_t)__DOMAIN_MAX_PFN(gaw)) << VTD_PAGE_SHIFT)
59
60/* IO virtual address start page frame number */
61#define IOVA_START_PFN (1)
62
63#define IOVA_PFN(addr) ((addr) >> PAGE_SHIFT)
64
65/* page table handling */
66#define LEVEL_STRIDE (9)
67#define LEVEL_MASK (((u64)1 << LEVEL_STRIDE) - 1)
68
69static inline int agaw_to_level(int agaw)
70{
71 return agaw + 2;
72}
73
74static inline int agaw_to_width(int agaw)
75{
76 return min_t(int, 30 + agaw * LEVEL_STRIDE, MAX_AGAW_WIDTH);
77}
78
79static inline int width_to_agaw(int width)
80{
81 return DIV_ROUND_UP(width - 30, LEVEL_STRIDE);
82}
83
84static inline unsigned int level_to_offset_bits(int level)
85{
86 return (level - 1) * LEVEL_STRIDE;
87}
88
89static inline int pfn_level_offset(u64 pfn, int level)
90{
91 return (pfn >> level_to_offset_bits(level)) & LEVEL_MASK;
92}
93
94static inline u64 level_mask(int level)
95{
96 return -1ULL << level_to_offset_bits(level);
97}
98
99static inline u64 level_size(int level)
100{
101 return 1ULL << level_to_offset_bits(level);
102}
103
104static inline u64 align_to_level(u64 pfn, int level)
105{
106 return (pfn + level_size(level) - 1) & level_mask(level);
107}
108
109static inline unsigned long lvl_to_nr_pages(unsigned int lvl)
110{
111 return 1UL << min_t(int, (lvl - 1) * LEVEL_STRIDE, MAX_AGAW_PFN_WIDTH);
112}
113
114/* VT-d pages must always be _smaller_ than MM pages. Otherwise things
115 are never going to work. */
116static inline unsigned long mm_to_dma_pfn(unsigned long mm_pfn)
117{
118 return mm_pfn << (PAGE_SHIFT - VTD_PAGE_SHIFT);
119}
120static inline unsigned long page_to_dma_pfn(struct page *pg)
121{
122 return mm_to_dma_pfn(page_to_pfn(pg));
123}
124static inline unsigned long virt_to_dma_pfn(void *p)
125{
126 return page_to_dma_pfn(virt_to_page(p));
127}
128
129static void __init check_tylersburg_isoch(void);
130static int rwbf_quirk;
131
132/*
133 * set to 1 to panic kernel if can't successfully enable VT-d
134 * (used when kernel is launched w/ TXT)
135 */
136static int force_on = 0;
137static int intel_iommu_tboot_noforce;
138static int no_platform_optin;
139
140#define ROOT_ENTRY_NR (VTD_PAGE_SIZE/sizeof(struct root_entry))
141
142/*
143 * Take a root_entry and return the Lower Context Table Pointer (LCTP)
144 * if marked present.
145 */
146static phys_addr_t root_entry_lctp(struct root_entry *re)
147{
148 if (!(re->lo & 1))
149 return 0;
150
151 return re->lo & VTD_PAGE_MASK;
152}
153
154/*
155 * Take a root_entry and return the Upper Context Table Pointer (UCTP)
156 * if marked present.
157 */
158static phys_addr_t root_entry_uctp(struct root_entry *re)
159{
160 if (!(re->hi & 1))
161 return 0;
162
163 return re->hi & VTD_PAGE_MASK;
164}
165
166static inline void context_set_present(struct context_entry *context)
167{
168 context->lo |= 1;
169}
170
171static inline void context_set_fault_enable(struct context_entry *context)
172{
173 context->lo &= (((u64)-1) << 2) | 1;
174}
175
176static inline void context_set_translation_type(struct context_entry *context,
177 unsigned long value)
178{
179 context->lo &= (((u64)-1) << 4) | 3;
180 context->lo |= (value & 3) << 2;
181}
182
183static inline void context_set_address_root(struct context_entry *context,
184 unsigned long value)
185{
186 context->lo &= ~VTD_PAGE_MASK;
187 context->lo |= value & VTD_PAGE_MASK;
188}
189
190static inline void context_set_address_width(struct context_entry *context,
191 unsigned long value)
192{
193 context->hi |= value & 7;
194}
195
196static inline void context_set_domain_id(struct context_entry *context,
197 unsigned long value)
198{
199 context->hi |= (value & ((1 << 16) - 1)) << 8;
200}
201
202static inline void context_set_pasid(struct context_entry *context)
203{
204 context->lo |= CONTEXT_PASIDE;
205}
206
207static inline int context_domain_id(struct context_entry *c)
208{
209 return((c->hi >> 8) & 0xffff);
210}
211
212static inline void context_clear_entry(struct context_entry *context)
213{
214 context->lo = 0;
215 context->hi = 0;
216}
217
218static inline bool context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
219{
220 if (!iommu->copied_tables)
221 return false;
222
223 return test_bit(((long)bus << 8) | devfn, iommu->copied_tables);
224}
225
226static inline void
227set_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
228{
229 set_bit(((long)bus << 8) | devfn, iommu->copied_tables);
230}
231
232static inline void
233clear_context_copied(struct intel_iommu *iommu, u8 bus, u8 devfn)
234{
235 clear_bit(((long)bus << 8) | devfn, iommu->copied_tables);
236}
237
238/*
239 * This domain is a statically identity mapping domain.
240 * 1. This domain creats a static 1:1 mapping to all usable memory.
241 * 2. It maps to each iommu if successful.
242 * 3. Each iommu mapps to this domain if successful.
243 */
244static struct dmar_domain *si_domain;
245static int hw_pass_through = 1;
246
247struct dmar_rmrr_unit {
248 struct list_head list; /* list of rmrr units */
249 struct acpi_dmar_header *hdr; /* ACPI header */
250 u64 base_address; /* reserved base address*/
251 u64 end_address; /* reserved end address */
252 struct dmar_dev_scope *devices; /* target devices */
253 int devices_cnt; /* target device count */
254};
255
256struct dmar_atsr_unit {
257 struct list_head list; /* list of ATSR units */
258 struct acpi_dmar_header *hdr; /* ACPI header */
259 struct dmar_dev_scope *devices; /* target devices */
260 int devices_cnt; /* target device count */
261 u8 include_all:1; /* include all ports */
262};
263
264struct dmar_satc_unit {
265 struct list_head list; /* list of SATC units */
266 struct acpi_dmar_header *hdr; /* ACPI header */
267 struct dmar_dev_scope *devices; /* target devices */
268 struct intel_iommu *iommu; /* the corresponding iommu */
269 int devices_cnt; /* target device count */
270 u8 atc_required:1; /* ATS is required */
271};
272
273static LIST_HEAD(dmar_atsr_units);
274static LIST_HEAD(dmar_rmrr_units);
275static LIST_HEAD(dmar_satc_units);
276
277#define for_each_rmrr_units(rmrr) \
278 list_for_each_entry(rmrr, &dmar_rmrr_units, list)
279
280static void device_block_translation(struct device *dev);
281static void intel_iommu_domain_free(struct iommu_domain *domain);
282
283int dmar_disabled = !IS_ENABLED(CONFIG_INTEL_IOMMU_DEFAULT_ON);
284int intel_iommu_sm = IS_ENABLED(CONFIG_INTEL_IOMMU_SCALABLE_MODE_DEFAULT_ON);
285
286int intel_iommu_enabled = 0;
287EXPORT_SYMBOL_GPL(intel_iommu_enabled);
288
289static int dmar_map_gfx = 1;
290static int intel_iommu_superpage = 1;
291static int iommu_identity_mapping;
292static int iommu_skip_te_disable;
293
294#define IDENTMAP_GFX 2
295#define IDENTMAP_AZALIA 4
296
297const struct iommu_ops intel_iommu_ops;
298
299static bool translation_pre_enabled(struct intel_iommu *iommu)
300{
301 return (iommu->flags & VTD_FLAG_TRANS_PRE_ENABLED);
302}
303
304static void clear_translation_pre_enabled(struct intel_iommu *iommu)
305{
306 iommu->flags &= ~VTD_FLAG_TRANS_PRE_ENABLED;
307}
308
309static void init_translation_status(struct intel_iommu *iommu)
310{
311 u32 gsts;
312
313 gsts = readl(iommu->reg + DMAR_GSTS_REG);
314 if (gsts & DMA_GSTS_TES)
315 iommu->flags |= VTD_FLAG_TRANS_PRE_ENABLED;
316}
317
318static int __init intel_iommu_setup(char *str)
319{
320 if (!str)
321 return -EINVAL;
322
323 while (*str) {
324 if (!strncmp(str, "on", 2)) {
325 dmar_disabled = 0;
326 pr_info("IOMMU enabled\n");
327 } else if (!strncmp(str, "off", 3)) {
328 dmar_disabled = 1;
329 no_platform_optin = 1;
330 pr_info("IOMMU disabled\n");
331 } else if (!strncmp(str, "igfx_off", 8)) {
332 dmar_map_gfx = 0;
333 pr_info("Disable GFX device mapping\n");
334 } else if (!strncmp(str, "forcedac", 8)) {
335 pr_warn("intel_iommu=forcedac deprecated; use iommu.forcedac instead\n");
336 iommu_dma_forcedac = true;
337 } else if (!strncmp(str, "strict", 6)) {
338 pr_warn("intel_iommu=strict deprecated; use iommu.strict=1 instead\n");
339 iommu_set_dma_strict();
340 } else if (!strncmp(str, "sp_off", 6)) {
341 pr_info("Disable supported super page\n");
342 intel_iommu_superpage = 0;
343 } else if (!strncmp(str, "sm_on", 5)) {
344 pr_info("Enable scalable mode if hardware supports\n");
345 intel_iommu_sm = 1;
346 } else if (!strncmp(str, "sm_off", 6)) {
347 pr_info("Scalable mode is disallowed\n");
348 intel_iommu_sm = 0;
349 } else if (!strncmp(str, "tboot_noforce", 13)) {
350 pr_info("Intel-IOMMU: not forcing on after tboot. This could expose security risk for tboot\n");
351 intel_iommu_tboot_noforce = 1;
352 } else {
353 pr_notice("Unknown option - '%s'\n", str);
354 }
355
356 str += strcspn(str, ",");
357 while (*str == ',')
358 str++;
359 }
360
361 return 1;
362}
363__setup("intel_iommu=", intel_iommu_setup);
364
365void *alloc_pgtable_page(int node)
366{
367 struct page *page;
368 void *vaddr = NULL;
369
370 page = alloc_pages_node(node, GFP_ATOMIC | __GFP_ZERO, 0);
371 if (page)
372 vaddr = page_address(page);
373 return vaddr;
374}
375
376void free_pgtable_page(void *vaddr)
377{
378 free_page((unsigned long)vaddr);
379}
380
381static inline int domain_type_is_si(struct dmar_domain *domain)
382{
383 return domain->domain.type == IOMMU_DOMAIN_IDENTITY;
384}
385
386static inline int domain_pfn_supported(struct dmar_domain *domain,
387 unsigned long pfn)
388{
389 int addr_width = agaw_to_width(domain->agaw) - VTD_PAGE_SHIFT;
390
391 return !(addr_width < BITS_PER_LONG && pfn >> addr_width);
392}
393
394/*
395 * Calculate the Supported Adjusted Guest Address Widths of an IOMMU.
396 * Refer to 11.4.2 of the VT-d spec for the encoding of each bit of
397 * the returned SAGAW.
398 */
399static unsigned long __iommu_calculate_sagaw(struct intel_iommu *iommu)
400{
401 unsigned long fl_sagaw, sl_sagaw;
402
403 fl_sagaw = BIT(2) | (cap_fl5lp_support(iommu->cap) ? BIT(3) : 0);
404 sl_sagaw = cap_sagaw(iommu->cap);
405
406 /* Second level only. */
407 if (!sm_supported(iommu) || !ecap_flts(iommu->ecap))
408 return sl_sagaw;
409
410 /* First level only. */
411 if (!ecap_slts(iommu->ecap))
412 return fl_sagaw;
413
414 return fl_sagaw & sl_sagaw;
415}
416
417static int __iommu_calculate_agaw(struct intel_iommu *iommu, int max_gaw)
418{
419 unsigned long sagaw;
420 int agaw;
421
422 sagaw = __iommu_calculate_sagaw(iommu);
423 for (agaw = width_to_agaw(max_gaw); agaw >= 0; agaw--) {
424 if (test_bit(agaw, &sagaw))
425 break;
426 }
427
428 return agaw;
429}
430
431/*
432 * Calculate max SAGAW for each iommu.
433 */
434int iommu_calculate_max_sagaw(struct intel_iommu *iommu)
435{
436 return __iommu_calculate_agaw(iommu, MAX_AGAW_WIDTH);
437}
438
439/*
440 * calculate agaw for each iommu.
441 * "SAGAW" may be different across iommus, use a default agaw, and
442 * get a supported less agaw for iommus that don't support the default agaw.
443 */
444int iommu_calculate_agaw(struct intel_iommu *iommu)
445{
446 return __iommu_calculate_agaw(iommu, DEFAULT_DOMAIN_ADDRESS_WIDTH);
447}
448
449static inline bool iommu_paging_structure_coherency(struct intel_iommu *iommu)
450{
451 return sm_supported(iommu) ?
452 ecap_smpwc(iommu->ecap) : ecap_coherent(iommu->ecap);
453}
454
455static void domain_update_iommu_coherency(struct dmar_domain *domain)
456{
457 struct iommu_domain_info *info;
458 struct dmar_drhd_unit *drhd;
459 struct intel_iommu *iommu;
460 bool found = false;
461 unsigned long i;
462
463 domain->iommu_coherency = true;
464 xa_for_each(&domain->iommu_array, i, info) {
465 found = true;
466 if (!iommu_paging_structure_coherency(info->iommu)) {
467 domain->iommu_coherency = false;
468 break;
469 }
470 }
471 if (found)
472 return;
473
474 /* No hardware attached; use lowest common denominator */
475 rcu_read_lock();
476 for_each_active_iommu(iommu, drhd) {
477 if (!iommu_paging_structure_coherency(iommu)) {
478 domain->iommu_coherency = false;
479 break;
480 }
481 }
482 rcu_read_unlock();
483}
484
485static int domain_update_iommu_superpage(struct dmar_domain *domain,
486 struct intel_iommu *skip)
487{
488 struct dmar_drhd_unit *drhd;
489 struct intel_iommu *iommu;
490 int mask = 0x3;
491
492 if (!intel_iommu_superpage)
493 return 0;
494
495 /* set iommu_superpage to the smallest common denominator */
496 rcu_read_lock();
497 for_each_active_iommu(iommu, drhd) {
498 if (iommu != skip) {
499 if (domain && domain->use_first_level) {
500 if (!cap_fl1gp_support(iommu->cap))
501 mask = 0x1;
502 } else {
503 mask &= cap_super_page_val(iommu->cap);
504 }
505
506 if (!mask)
507 break;
508 }
509 }
510 rcu_read_unlock();
511
512 return fls(mask);
513}
514
515static int domain_update_device_node(struct dmar_domain *domain)
516{
517 struct device_domain_info *info;
518 int nid = NUMA_NO_NODE;
519 unsigned long flags;
520
521 spin_lock_irqsave(&domain->lock, flags);
522 list_for_each_entry(info, &domain->devices, link) {
523 /*
524 * There could possibly be multiple device numa nodes as devices
525 * within the same domain may sit behind different IOMMUs. There
526 * isn't perfect answer in such situation, so we select first
527 * come first served policy.
528 */
529 nid = dev_to_node(info->dev);
530 if (nid != NUMA_NO_NODE)
531 break;
532 }
533 spin_unlock_irqrestore(&domain->lock, flags);
534
535 return nid;
536}
537
538static void domain_update_iotlb(struct dmar_domain *domain);
539
540/* Return the super pagesize bitmap if supported. */
541static unsigned long domain_super_pgsize_bitmap(struct dmar_domain *domain)
542{
543 unsigned long bitmap = 0;
544
545 /*
546 * 1-level super page supports page size of 2MiB, 2-level super page
547 * supports page size of both 2MiB and 1GiB.
548 */
549 if (domain->iommu_superpage == 1)
550 bitmap |= SZ_2M;
551 else if (domain->iommu_superpage == 2)
552 bitmap |= SZ_2M | SZ_1G;
553
554 return bitmap;
555}
556
557/* Some capabilities may be different across iommus */
558static void domain_update_iommu_cap(struct dmar_domain *domain)
559{
560 domain_update_iommu_coherency(domain);
561 domain->iommu_superpage = domain_update_iommu_superpage(domain, NULL);
562
563 /*
564 * If RHSA is missing, we should default to the device numa domain
565 * as fall back.
566 */
567 if (domain->nid == NUMA_NO_NODE)
568 domain->nid = domain_update_device_node(domain);
569
570 /*
571 * First-level translation restricts the input-address to a
572 * canonical address (i.e., address bits 63:N have the same
573 * value as address bit [N-1], where N is 48-bits with 4-level
574 * paging and 57-bits with 5-level paging). Hence, skip bit
575 * [N-1].
576 */
577 if (domain->use_first_level)
578 domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw - 1);
579 else
580 domain->domain.geometry.aperture_end = __DOMAIN_MAX_ADDR(domain->gaw);
581
582 domain->domain.pgsize_bitmap |= domain_super_pgsize_bitmap(domain);
583 domain_update_iotlb(domain);
584}
585
586struct context_entry *iommu_context_addr(struct intel_iommu *iommu, u8 bus,
587 u8 devfn, int alloc)
588{
589 struct root_entry *root = &iommu->root_entry[bus];
590 struct context_entry *context;
591 u64 *entry;
592
593 /*
594 * Except that the caller requested to allocate a new entry,
595 * returning a copied context entry makes no sense.
596 */
597 if (!alloc && context_copied(iommu, bus, devfn))
598 return NULL;
599
600 entry = &root->lo;
601 if (sm_supported(iommu)) {
602 if (devfn >= 0x80) {
603 devfn -= 0x80;
604 entry = &root->hi;
605 }
606 devfn *= 2;
607 }
608 if (*entry & 1)
609 context = phys_to_virt(*entry & VTD_PAGE_MASK);
610 else {
611 unsigned long phy_addr;
612 if (!alloc)
613 return NULL;
614
615 context = alloc_pgtable_page(iommu->node);
616 if (!context)
617 return NULL;
618
619 __iommu_flush_cache(iommu, (void *)context, CONTEXT_SIZE);
620 phy_addr = virt_to_phys((void *)context);
621 *entry = phy_addr | 1;
622 __iommu_flush_cache(iommu, entry, sizeof(*entry));
623 }
624 return &context[devfn];
625}
626
627/**
628 * is_downstream_to_pci_bridge - test if a device belongs to the PCI
629 * sub-hierarchy of a candidate PCI-PCI bridge
630 * @dev: candidate PCI device belonging to @bridge PCI sub-hierarchy
631 * @bridge: the candidate PCI-PCI bridge
632 *
633 * Return: true if @dev belongs to @bridge PCI sub-hierarchy, else false.
634 */
635static bool
636is_downstream_to_pci_bridge(struct device *dev, struct device *bridge)
637{
638 struct pci_dev *pdev, *pbridge;
639
640 if (!dev_is_pci(dev) || !dev_is_pci(bridge))
641 return false;
642
643 pdev = to_pci_dev(dev);
644 pbridge = to_pci_dev(bridge);
645
646 if (pbridge->subordinate &&
647 pbridge->subordinate->number <= pdev->bus->number &&
648 pbridge->subordinate->busn_res.end >= pdev->bus->number)
649 return true;
650
651 return false;
652}
653
654static bool quirk_ioat_snb_local_iommu(struct pci_dev *pdev)
655{
656 struct dmar_drhd_unit *drhd;
657 u32 vtbar;
658 int rc;
659
660 /* We know that this device on this chipset has its own IOMMU.
661 * If we find it under a different IOMMU, then the BIOS is lying
662 * to us. Hope that the IOMMU for this device is actually
663 * disabled, and it needs no translation...
664 */
665 rc = pci_bus_read_config_dword(pdev->bus, PCI_DEVFN(0, 0), 0xb0, &vtbar);
666 if (rc) {
667 /* "can't" happen */
668 dev_info(&pdev->dev, "failed to run vt-d quirk\n");
669 return false;
670 }
671 vtbar &= 0xffff0000;
672
673 /* we know that the this iommu should be at offset 0xa000 from vtbar */
674 drhd = dmar_find_matched_drhd_unit(pdev);
675 if (!drhd || drhd->reg_base_addr - vtbar != 0xa000) {
676 pr_warn_once(FW_BUG "BIOS assigned incorrect VT-d unit for Intel(R) QuickData Technology device\n");
677 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
678 return true;
679 }
680
681 return false;
682}
683
684static bool iommu_is_dummy(struct intel_iommu *iommu, struct device *dev)
685{
686 if (!iommu || iommu->drhd->ignored)
687 return true;
688
689 if (dev_is_pci(dev)) {
690 struct pci_dev *pdev = to_pci_dev(dev);
691
692 if (pdev->vendor == PCI_VENDOR_ID_INTEL &&
693 pdev->device == PCI_DEVICE_ID_INTEL_IOAT_SNB &&
694 quirk_ioat_snb_local_iommu(pdev))
695 return true;
696 }
697
698 return false;
699}
700
701struct intel_iommu *device_to_iommu(struct device *dev, u8 *bus, u8 *devfn)
702{
703 struct dmar_drhd_unit *drhd = NULL;
704 struct pci_dev *pdev = NULL;
705 struct intel_iommu *iommu;
706 struct device *tmp;
707 u16 segment = 0;
708 int i;
709
710 if (!dev)
711 return NULL;
712
713 if (dev_is_pci(dev)) {
714 struct pci_dev *pf_pdev;
715
716 pdev = pci_real_dma_dev(to_pci_dev(dev));
717
718 /* VFs aren't listed in scope tables; we need to look up
719 * the PF instead to find the IOMMU. */
720 pf_pdev = pci_physfn(pdev);
721 dev = &pf_pdev->dev;
722 segment = pci_domain_nr(pdev->bus);
723 } else if (has_acpi_companion(dev))
724 dev = &ACPI_COMPANION(dev)->dev;
725
726 rcu_read_lock();
727 for_each_iommu(iommu, drhd) {
728 if (pdev && segment != drhd->segment)
729 continue;
730
731 for_each_active_dev_scope(drhd->devices,
732 drhd->devices_cnt, i, tmp) {
733 if (tmp == dev) {
734 /* For a VF use its original BDF# not that of the PF
735 * which we used for the IOMMU lookup. Strictly speaking
736 * we could do this for all PCI devices; we only need to
737 * get the BDF# from the scope table for ACPI matches. */
738 if (pdev && pdev->is_virtfn)
739 goto got_pdev;
740
741 if (bus && devfn) {
742 *bus = drhd->devices[i].bus;
743 *devfn = drhd->devices[i].devfn;
744 }
745 goto out;
746 }
747
748 if (is_downstream_to_pci_bridge(dev, tmp))
749 goto got_pdev;
750 }
751
752 if (pdev && drhd->include_all) {
753got_pdev:
754 if (bus && devfn) {
755 *bus = pdev->bus->number;
756 *devfn = pdev->devfn;
757 }
758 goto out;
759 }
760 }
761 iommu = NULL;
762out:
763 if (iommu_is_dummy(iommu, dev))
764 iommu = NULL;
765
766 rcu_read_unlock();
767
768 return iommu;
769}
770
771static void domain_flush_cache(struct dmar_domain *domain,
772 void *addr, int size)
773{
774 if (!domain->iommu_coherency)
775 clflush_cache_range(addr, size);
776}
777
778static void free_context_table(struct intel_iommu *iommu)
779{
780 struct context_entry *context;
781 int i;
782
783 if (!iommu->root_entry)
784 return;
785
786 for (i = 0; i < ROOT_ENTRY_NR; i++) {
787 context = iommu_context_addr(iommu, i, 0, 0);
788 if (context)
789 free_pgtable_page(context);
790
791 if (!sm_supported(iommu))
792 continue;
793
794 context = iommu_context_addr(iommu, i, 0x80, 0);
795 if (context)
796 free_pgtable_page(context);
797 }
798
799 free_pgtable_page(iommu->root_entry);
800 iommu->root_entry = NULL;
801}
802
803#ifdef CONFIG_DMAR_DEBUG
804static void pgtable_walk(struct intel_iommu *iommu, unsigned long pfn,
805 u8 bus, u8 devfn, struct dma_pte *parent, int level)
806{
807 struct dma_pte *pte;
808 int offset;
809
810 while (1) {
811 offset = pfn_level_offset(pfn, level);
812 pte = &parent[offset];
813 if (!pte || (dma_pte_superpage(pte) || !dma_pte_present(pte))) {
814 pr_info("PTE not present at level %d\n", level);
815 break;
816 }
817
818 pr_info("pte level: %d, pte value: 0x%016llx\n", level, pte->val);
819
820 if (level == 1)
821 break;
822
823 parent = phys_to_virt(dma_pte_addr(pte));
824 level--;
825 }
826}
827
828void dmar_fault_dump_ptes(struct intel_iommu *iommu, u16 source_id,
829 unsigned long long addr, u32 pasid)
830{
831 struct pasid_dir_entry *dir, *pde;
832 struct pasid_entry *entries, *pte;
833 struct context_entry *ctx_entry;
834 struct root_entry *rt_entry;
835 int i, dir_index, index, level;
836 u8 devfn = source_id & 0xff;
837 u8 bus = source_id >> 8;
838 struct dma_pte *pgtable;
839
840 pr_info("Dump %s table entries for IOVA 0x%llx\n", iommu->name, addr);
841
842 /* root entry dump */
843 rt_entry = &iommu->root_entry[bus];
844 if (!rt_entry) {
845 pr_info("root table entry is not present\n");
846 return;
847 }
848
849 if (sm_supported(iommu))
850 pr_info("scalable mode root entry: hi 0x%016llx, low 0x%016llx\n",
851 rt_entry->hi, rt_entry->lo);
852 else
853 pr_info("root entry: 0x%016llx", rt_entry->lo);
854
855 /* context entry dump */
856 ctx_entry = iommu_context_addr(iommu, bus, devfn, 0);
857 if (!ctx_entry) {
858 pr_info("context table entry is not present\n");
859 return;
860 }
861
862 pr_info("context entry: hi 0x%016llx, low 0x%016llx\n",
863 ctx_entry->hi, ctx_entry->lo);
864
865 /* legacy mode does not require PASID entries */
866 if (!sm_supported(iommu)) {
867 level = agaw_to_level(ctx_entry->hi & 7);
868 pgtable = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
869 goto pgtable_walk;
870 }
871
872 /* get the pointer to pasid directory entry */
873 dir = phys_to_virt(ctx_entry->lo & VTD_PAGE_MASK);
874 if (!dir) {
875 pr_info("pasid directory entry is not present\n");
876 return;
877 }
878 /* For request-without-pasid, get the pasid from context entry */
879 if (intel_iommu_sm && pasid == INVALID_IOASID)
880 pasid = PASID_RID2PASID;
881
882 dir_index = pasid >> PASID_PDE_SHIFT;
883 pde = &dir[dir_index];
884 pr_info("pasid dir entry: 0x%016llx\n", pde->val);
885
886 /* get the pointer to the pasid table entry */
887 entries = get_pasid_table_from_pde(pde);
888 if (!entries) {
889 pr_info("pasid table entry is not present\n");
890 return;
891 }
892 index = pasid & PASID_PTE_MASK;
893 pte = &entries[index];
894 for (i = 0; i < ARRAY_SIZE(pte->val); i++)
895 pr_info("pasid table entry[%d]: 0x%016llx\n", i, pte->val[i]);
896
897 if (pasid_pte_get_pgtt(pte) == PASID_ENTRY_PGTT_FL_ONLY) {
898 level = pte->val[2] & BIT_ULL(2) ? 5 : 4;
899 pgtable = phys_to_virt(pte->val[2] & VTD_PAGE_MASK);
900 } else {
901 level = agaw_to_level((pte->val[0] >> 2) & 0x7);
902 pgtable = phys_to_virt(pte->val[0] & VTD_PAGE_MASK);
903 }
904
905pgtable_walk:
906 pgtable_walk(iommu, addr >> VTD_PAGE_SHIFT, bus, devfn, pgtable, level);
907}
908#endif
909
910static struct dma_pte *pfn_to_dma_pte(struct dmar_domain *domain,
911 unsigned long pfn, int *target_level)
912{
913 struct dma_pte *parent, *pte;
914 int level = agaw_to_level(domain->agaw);
915 int offset;
916
917 BUG_ON(!domain->pgd);
918
919 if (!domain_pfn_supported(domain, pfn))
920 /* Address beyond IOMMU's addressing capabilities. */
921 return NULL;
922
923 parent = domain->pgd;
924
925 while (1) {
926 void *tmp_page;
927
928 offset = pfn_level_offset(pfn, level);
929 pte = &parent[offset];
930 if (!*target_level && (dma_pte_superpage(pte) || !dma_pte_present(pte)))
931 break;
932 if (level == *target_level)
933 break;
934
935 if (!dma_pte_present(pte)) {
936 uint64_t pteval;
937
938 tmp_page = alloc_pgtable_page(domain->nid);
939
940 if (!tmp_page)
941 return NULL;
942
943 domain_flush_cache(domain, tmp_page, VTD_PAGE_SIZE);
944 pteval = ((uint64_t)virt_to_dma_pfn(tmp_page) << VTD_PAGE_SHIFT) | DMA_PTE_READ | DMA_PTE_WRITE;
945 if (domain->use_first_level)
946 pteval |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
947
948 if (cmpxchg64(&pte->val, 0ULL, pteval))
949 /* Someone else set it while we were thinking; use theirs. */
950 free_pgtable_page(tmp_page);
951 else
952 domain_flush_cache(domain, pte, sizeof(*pte));
953 }
954 if (level == 1)
955 break;
956
957 parent = phys_to_virt(dma_pte_addr(pte));
958 level--;
959 }
960
961 if (!*target_level)
962 *target_level = level;
963
964 return pte;
965}
966
967/* return address's pte at specific level */
968static struct dma_pte *dma_pfn_level_pte(struct dmar_domain *domain,
969 unsigned long pfn,
970 int level, int *large_page)
971{
972 struct dma_pte *parent, *pte;
973 int total = agaw_to_level(domain->agaw);
974 int offset;
975
976 parent = domain->pgd;
977 while (level <= total) {
978 offset = pfn_level_offset(pfn, total);
979 pte = &parent[offset];
980 if (level == total)
981 return pte;
982
983 if (!dma_pte_present(pte)) {
984 *large_page = total;
985 break;
986 }
987
988 if (dma_pte_superpage(pte)) {
989 *large_page = total;
990 return pte;
991 }
992
993 parent = phys_to_virt(dma_pte_addr(pte));
994 total--;
995 }
996 return NULL;
997}
998
999/* clear last level pte, a tlb flush should be followed */
1000static void dma_pte_clear_range(struct dmar_domain *domain,
1001 unsigned long start_pfn,
1002 unsigned long last_pfn)
1003{
1004 unsigned int large_page;
1005 struct dma_pte *first_pte, *pte;
1006
1007 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1008 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1009 BUG_ON(start_pfn > last_pfn);
1010
1011 /* we don't need lock here; nobody else touches the iova range */
1012 do {
1013 large_page = 1;
1014 first_pte = pte = dma_pfn_level_pte(domain, start_pfn, 1, &large_page);
1015 if (!pte) {
1016 start_pfn = align_to_level(start_pfn + 1, large_page + 1);
1017 continue;
1018 }
1019 do {
1020 dma_clear_pte(pte);
1021 start_pfn += lvl_to_nr_pages(large_page);
1022 pte++;
1023 } while (start_pfn <= last_pfn && !first_pte_in_page(pte));
1024
1025 domain_flush_cache(domain, first_pte,
1026 (void *)pte - (void *)first_pte);
1027
1028 } while (start_pfn && start_pfn <= last_pfn);
1029}
1030
1031static void dma_pte_free_level(struct dmar_domain *domain, int level,
1032 int retain_level, struct dma_pte *pte,
1033 unsigned long pfn, unsigned long start_pfn,
1034 unsigned long last_pfn)
1035{
1036 pfn = max(start_pfn, pfn);
1037 pte = &pte[pfn_level_offset(pfn, level)];
1038
1039 do {
1040 unsigned long level_pfn;
1041 struct dma_pte *level_pte;
1042
1043 if (!dma_pte_present(pte) || dma_pte_superpage(pte))
1044 goto next;
1045
1046 level_pfn = pfn & level_mask(level);
1047 level_pte = phys_to_virt(dma_pte_addr(pte));
1048
1049 if (level > 2) {
1050 dma_pte_free_level(domain, level - 1, retain_level,
1051 level_pte, level_pfn, start_pfn,
1052 last_pfn);
1053 }
1054
1055 /*
1056 * Free the page table if we're below the level we want to
1057 * retain and the range covers the entire table.
1058 */
1059 if (level < retain_level && !(start_pfn > level_pfn ||
1060 last_pfn < level_pfn + level_size(level) - 1)) {
1061 dma_clear_pte(pte);
1062 domain_flush_cache(domain, pte, sizeof(*pte));
1063 free_pgtable_page(level_pte);
1064 }
1065next:
1066 pfn += level_size(level);
1067 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1068}
1069
1070/*
1071 * clear last level (leaf) ptes and free page table pages below the
1072 * level we wish to keep intact.
1073 */
1074static void dma_pte_free_pagetable(struct dmar_domain *domain,
1075 unsigned long start_pfn,
1076 unsigned long last_pfn,
1077 int retain_level)
1078{
1079 dma_pte_clear_range(domain, start_pfn, last_pfn);
1080
1081 /* We don't need lock here; nobody else touches the iova range */
1082 dma_pte_free_level(domain, agaw_to_level(domain->agaw), retain_level,
1083 domain->pgd, 0, start_pfn, last_pfn);
1084
1085 /* free pgd */
1086 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1087 free_pgtable_page(domain->pgd);
1088 domain->pgd = NULL;
1089 }
1090}
1091
1092/* When a page at a given level is being unlinked from its parent, we don't
1093 need to *modify* it at all. All we need to do is make a list of all the
1094 pages which can be freed just as soon as we've flushed the IOTLB and we
1095 know the hardware page-walk will no longer touch them.
1096 The 'pte' argument is the *parent* PTE, pointing to the page that is to
1097 be freed. */
1098static void dma_pte_list_pagetables(struct dmar_domain *domain,
1099 int level, struct dma_pte *pte,
1100 struct list_head *freelist)
1101{
1102 struct page *pg;
1103
1104 pg = pfn_to_page(dma_pte_addr(pte) >> PAGE_SHIFT);
1105 list_add_tail(&pg->lru, freelist);
1106
1107 if (level == 1)
1108 return;
1109
1110 pte = page_address(pg);
1111 do {
1112 if (dma_pte_present(pte) && !dma_pte_superpage(pte))
1113 dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1114 pte++;
1115 } while (!first_pte_in_page(pte));
1116}
1117
1118static void dma_pte_clear_level(struct dmar_domain *domain, int level,
1119 struct dma_pte *pte, unsigned long pfn,
1120 unsigned long start_pfn, unsigned long last_pfn,
1121 struct list_head *freelist)
1122{
1123 struct dma_pte *first_pte = NULL, *last_pte = NULL;
1124
1125 pfn = max(start_pfn, pfn);
1126 pte = &pte[pfn_level_offset(pfn, level)];
1127
1128 do {
1129 unsigned long level_pfn = pfn & level_mask(level);
1130
1131 if (!dma_pte_present(pte))
1132 goto next;
1133
1134 /* If range covers entire pagetable, free it */
1135 if (start_pfn <= level_pfn &&
1136 last_pfn >= level_pfn + level_size(level) - 1) {
1137 /* These suborbinate page tables are going away entirely. Don't
1138 bother to clear them; we're just going to *free* them. */
1139 if (level > 1 && !dma_pte_superpage(pte))
1140 dma_pte_list_pagetables(domain, level - 1, pte, freelist);
1141
1142 dma_clear_pte(pte);
1143 if (!first_pte)
1144 first_pte = pte;
1145 last_pte = pte;
1146 } else if (level > 1) {
1147 /* Recurse down into a level that isn't *entirely* obsolete */
1148 dma_pte_clear_level(domain, level - 1,
1149 phys_to_virt(dma_pte_addr(pte)),
1150 level_pfn, start_pfn, last_pfn,
1151 freelist);
1152 }
1153next:
1154 pfn = level_pfn + level_size(level);
1155 } while (!first_pte_in_page(++pte) && pfn <= last_pfn);
1156
1157 if (first_pte)
1158 domain_flush_cache(domain, first_pte,
1159 (void *)++last_pte - (void *)first_pte);
1160}
1161
1162/* We can't just free the pages because the IOMMU may still be walking
1163 the page tables, and may have cached the intermediate levels. The
1164 pages can only be freed after the IOTLB flush has been done. */
1165static void domain_unmap(struct dmar_domain *domain, unsigned long start_pfn,
1166 unsigned long last_pfn, struct list_head *freelist)
1167{
1168 BUG_ON(!domain_pfn_supported(domain, start_pfn));
1169 BUG_ON(!domain_pfn_supported(domain, last_pfn));
1170 BUG_ON(start_pfn > last_pfn);
1171
1172 /* we don't need lock here; nobody else touches the iova range */
1173 dma_pte_clear_level(domain, agaw_to_level(domain->agaw),
1174 domain->pgd, 0, start_pfn, last_pfn, freelist);
1175
1176 /* free pgd */
1177 if (start_pfn == 0 && last_pfn == DOMAIN_MAX_PFN(domain->gaw)) {
1178 struct page *pgd_page = virt_to_page(domain->pgd);
1179 list_add_tail(&pgd_page->lru, freelist);
1180 domain->pgd = NULL;
1181 }
1182}
1183
1184/* iommu handling */
1185static int iommu_alloc_root_entry(struct intel_iommu *iommu)
1186{
1187 struct root_entry *root;
1188
1189 root = (struct root_entry *)alloc_pgtable_page(iommu->node);
1190 if (!root) {
1191 pr_err("Allocating root entry for %s failed\n",
1192 iommu->name);
1193 return -ENOMEM;
1194 }
1195
1196 __iommu_flush_cache(iommu, root, ROOT_SIZE);
1197 iommu->root_entry = root;
1198
1199 return 0;
1200}
1201
1202static void iommu_set_root_entry(struct intel_iommu *iommu)
1203{
1204 u64 addr;
1205 u32 sts;
1206 unsigned long flag;
1207
1208 addr = virt_to_phys(iommu->root_entry);
1209 if (sm_supported(iommu))
1210 addr |= DMA_RTADDR_SMT;
1211
1212 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1213 dmar_writeq(iommu->reg + DMAR_RTADDR_REG, addr);
1214
1215 writel(iommu->gcmd | DMA_GCMD_SRTP, iommu->reg + DMAR_GCMD_REG);
1216
1217 /* Make sure hardware complete it */
1218 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1219 readl, (sts & DMA_GSTS_RTPS), sts);
1220
1221 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1222
1223 /*
1224 * Hardware invalidates all DMA remapping hardware translation
1225 * caches as part of SRTP flow.
1226 */
1227 if (cap_esrtps(iommu->cap))
1228 return;
1229
1230 iommu->flush.flush_context(iommu, 0, 0, 0, DMA_CCMD_GLOBAL_INVL);
1231 if (sm_supported(iommu))
1232 qi_flush_pasid_cache(iommu, 0, QI_PC_GLOBAL, 0);
1233 iommu->flush.flush_iotlb(iommu, 0, 0, 0, DMA_TLB_GLOBAL_FLUSH);
1234}
1235
1236void iommu_flush_write_buffer(struct intel_iommu *iommu)
1237{
1238 u32 val;
1239 unsigned long flag;
1240
1241 if (!rwbf_quirk && !cap_rwbf(iommu->cap))
1242 return;
1243
1244 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1245 writel(iommu->gcmd | DMA_GCMD_WBF, iommu->reg + DMAR_GCMD_REG);
1246
1247 /* Make sure hardware complete it */
1248 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1249 readl, (!(val & DMA_GSTS_WBFS)), val);
1250
1251 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1252}
1253
1254/* return value determine if we need a write buffer flush */
1255static void __iommu_flush_context(struct intel_iommu *iommu,
1256 u16 did, u16 source_id, u8 function_mask,
1257 u64 type)
1258{
1259 u64 val = 0;
1260 unsigned long flag;
1261
1262 switch (type) {
1263 case DMA_CCMD_GLOBAL_INVL:
1264 val = DMA_CCMD_GLOBAL_INVL;
1265 break;
1266 case DMA_CCMD_DOMAIN_INVL:
1267 val = DMA_CCMD_DOMAIN_INVL|DMA_CCMD_DID(did);
1268 break;
1269 case DMA_CCMD_DEVICE_INVL:
1270 val = DMA_CCMD_DEVICE_INVL|DMA_CCMD_DID(did)
1271 | DMA_CCMD_SID(source_id) | DMA_CCMD_FM(function_mask);
1272 break;
1273 default:
1274 BUG();
1275 }
1276 val |= DMA_CCMD_ICC;
1277
1278 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1279 dmar_writeq(iommu->reg + DMAR_CCMD_REG, val);
1280
1281 /* Make sure hardware complete it */
1282 IOMMU_WAIT_OP(iommu, DMAR_CCMD_REG,
1283 dmar_readq, (!(val & DMA_CCMD_ICC)), val);
1284
1285 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1286}
1287
1288/* return value determine if we need a write buffer flush */
1289static void __iommu_flush_iotlb(struct intel_iommu *iommu, u16 did,
1290 u64 addr, unsigned int size_order, u64 type)
1291{
1292 int tlb_offset = ecap_iotlb_offset(iommu->ecap);
1293 u64 val = 0, val_iva = 0;
1294 unsigned long flag;
1295
1296 switch (type) {
1297 case DMA_TLB_GLOBAL_FLUSH:
1298 /* global flush doesn't need set IVA_REG */
1299 val = DMA_TLB_GLOBAL_FLUSH|DMA_TLB_IVT;
1300 break;
1301 case DMA_TLB_DSI_FLUSH:
1302 val = DMA_TLB_DSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1303 break;
1304 case DMA_TLB_PSI_FLUSH:
1305 val = DMA_TLB_PSI_FLUSH|DMA_TLB_IVT|DMA_TLB_DID(did);
1306 /* IH bit is passed in as part of address */
1307 val_iva = size_order | addr;
1308 break;
1309 default:
1310 BUG();
1311 }
1312 /* Note: set drain read/write */
1313#if 0
1314 /*
1315 * This is probably to be super secure.. Looks like we can
1316 * ignore it without any impact.
1317 */
1318 if (cap_read_drain(iommu->cap))
1319 val |= DMA_TLB_READ_DRAIN;
1320#endif
1321 if (cap_write_drain(iommu->cap))
1322 val |= DMA_TLB_WRITE_DRAIN;
1323
1324 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1325 /* Note: Only uses first TLB reg currently */
1326 if (val_iva)
1327 dmar_writeq(iommu->reg + tlb_offset, val_iva);
1328 dmar_writeq(iommu->reg + tlb_offset + 8, val);
1329
1330 /* Make sure hardware complete it */
1331 IOMMU_WAIT_OP(iommu, tlb_offset + 8,
1332 dmar_readq, (!(val & DMA_TLB_IVT)), val);
1333
1334 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1335
1336 /* check IOTLB invalidation granularity */
1337 if (DMA_TLB_IAIG(val) == 0)
1338 pr_err("Flush IOTLB failed\n");
1339 if (DMA_TLB_IAIG(val) != DMA_TLB_IIRG(type))
1340 pr_debug("TLB flush request %Lx, actual %Lx\n",
1341 (unsigned long long)DMA_TLB_IIRG(type),
1342 (unsigned long long)DMA_TLB_IAIG(val));
1343}
1344
1345static struct device_domain_info *
1346domain_lookup_dev_info(struct dmar_domain *domain,
1347 struct intel_iommu *iommu, u8 bus, u8 devfn)
1348{
1349 struct device_domain_info *info;
1350 unsigned long flags;
1351
1352 spin_lock_irqsave(&domain->lock, flags);
1353 list_for_each_entry(info, &domain->devices, link) {
1354 if (info->iommu == iommu && info->bus == bus &&
1355 info->devfn == devfn) {
1356 spin_unlock_irqrestore(&domain->lock, flags);
1357 return info;
1358 }
1359 }
1360 spin_unlock_irqrestore(&domain->lock, flags);
1361
1362 return NULL;
1363}
1364
1365static void domain_update_iotlb(struct dmar_domain *domain)
1366{
1367 struct device_domain_info *info;
1368 bool has_iotlb_device = false;
1369 unsigned long flags;
1370
1371 spin_lock_irqsave(&domain->lock, flags);
1372 list_for_each_entry(info, &domain->devices, link) {
1373 if (info->ats_enabled) {
1374 has_iotlb_device = true;
1375 break;
1376 }
1377 }
1378 domain->has_iotlb_device = has_iotlb_device;
1379 spin_unlock_irqrestore(&domain->lock, flags);
1380}
1381
1382/*
1383 * The extra devTLB flush quirk impacts those QAT devices with PCI device
1384 * IDs ranging from 0x4940 to 0x4943. It is exempted from risky_device()
1385 * check because it applies only to the built-in QAT devices and it doesn't
1386 * grant additional privileges.
1387 */
1388#define BUGGY_QAT_DEVID_MASK 0x4940
1389static bool dev_needs_extra_dtlb_flush(struct pci_dev *pdev)
1390{
1391 if (pdev->vendor != PCI_VENDOR_ID_INTEL)
1392 return false;
1393
1394 if ((pdev->device & 0xfffc) != BUGGY_QAT_DEVID_MASK)
1395 return false;
1396
1397 return true;
1398}
1399
1400static void iommu_enable_pci_caps(struct device_domain_info *info)
1401{
1402 struct pci_dev *pdev;
1403
1404 if (!dev_is_pci(info->dev))
1405 return;
1406
1407 pdev = to_pci_dev(info->dev);
1408 /* For IOMMU that supports device IOTLB throttling (DIT), we assign
1409 * PFSID to the invalidation desc of a VF such that IOMMU HW can gauge
1410 * queue depth at PF level. If DIT is not set, PFSID will be treated as
1411 * reserved, which should be set to 0.
1412 */
1413 if (!ecap_dit(info->iommu->ecap))
1414 info->pfsid = 0;
1415 else {
1416 struct pci_dev *pf_pdev;
1417
1418 /* pdev will be returned if device is not a vf */
1419 pf_pdev = pci_physfn(pdev);
1420 info->pfsid = pci_dev_id(pf_pdev);
1421 }
1422
1423 /* The PCIe spec, in its wisdom, declares that the behaviour of
1424 the device if you enable PASID support after ATS support is
1425 undefined. So always enable PASID support on devices which
1426 have it, even if we can't yet know if we're ever going to
1427 use it. */
1428 if (info->pasid_supported && !pci_enable_pasid(pdev, info->pasid_supported & ~1))
1429 info->pasid_enabled = 1;
1430
1431 if (info->pri_supported &&
1432 (info->pasid_enabled ? pci_prg_resp_pasid_required(pdev) : 1) &&
1433 !pci_reset_pri(pdev) && !pci_enable_pri(pdev, PRQ_DEPTH))
1434 info->pri_enabled = 1;
1435
1436 if (info->ats_supported && pci_ats_page_aligned(pdev) &&
1437 !pci_enable_ats(pdev, VTD_PAGE_SHIFT)) {
1438 info->ats_enabled = 1;
1439 domain_update_iotlb(info->domain);
1440 info->ats_qdep = pci_ats_queue_depth(pdev);
1441 }
1442}
1443
1444static void iommu_disable_pci_caps(struct device_domain_info *info)
1445{
1446 struct pci_dev *pdev;
1447
1448 if (!dev_is_pci(info->dev))
1449 return;
1450
1451 pdev = to_pci_dev(info->dev);
1452
1453 if (info->ats_enabled) {
1454 pci_disable_ats(pdev);
1455 info->ats_enabled = 0;
1456 domain_update_iotlb(info->domain);
1457 }
1458
1459 if (info->pri_enabled) {
1460 pci_disable_pri(pdev);
1461 info->pri_enabled = 0;
1462 }
1463
1464 if (info->pasid_enabled) {
1465 pci_disable_pasid(pdev);
1466 info->pasid_enabled = 0;
1467 }
1468}
1469
1470static void __iommu_flush_dev_iotlb(struct device_domain_info *info,
1471 u64 addr, unsigned int mask)
1472{
1473 u16 sid, qdep;
1474
1475 if (!info || !info->ats_enabled)
1476 return;
1477
1478 sid = info->bus << 8 | info->devfn;
1479 qdep = info->ats_qdep;
1480 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
1481 qdep, addr, mask);
1482 quirk_extra_dev_tlb_flush(info, addr, mask, PASID_RID2PASID, qdep);
1483}
1484
1485static void iommu_flush_dev_iotlb(struct dmar_domain *domain,
1486 u64 addr, unsigned mask)
1487{
1488 struct device_domain_info *info;
1489 unsigned long flags;
1490
1491 if (!domain->has_iotlb_device)
1492 return;
1493
1494 spin_lock_irqsave(&domain->lock, flags);
1495 list_for_each_entry(info, &domain->devices, link)
1496 __iommu_flush_dev_iotlb(info, addr, mask);
1497 spin_unlock_irqrestore(&domain->lock, flags);
1498}
1499
1500static void iommu_flush_iotlb_psi(struct intel_iommu *iommu,
1501 struct dmar_domain *domain,
1502 unsigned long pfn, unsigned int pages,
1503 int ih, int map)
1504{
1505 unsigned int aligned_pages = __roundup_pow_of_two(pages);
1506 unsigned int mask = ilog2(aligned_pages);
1507 uint64_t addr = (uint64_t)pfn << VTD_PAGE_SHIFT;
1508 u16 did = domain_id_iommu(domain, iommu);
1509
1510 BUG_ON(pages == 0);
1511
1512 if (ih)
1513 ih = 1 << 6;
1514
1515 if (domain->use_first_level) {
1516 qi_flush_piotlb(iommu, did, PASID_RID2PASID, addr, pages, ih);
1517 } else {
1518 unsigned long bitmask = aligned_pages - 1;
1519
1520 /*
1521 * PSI masks the low order bits of the base address. If the
1522 * address isn't aligned to the mask, then compute a mask value
1523 * needed to ensure the target range is flushed.
1524 */
1525 if (unlikely(bitmask & pfn)) {
1526 unsigned long end_pfn = pfn + pages - 1, shared_bits;
1527
1528 /*
1529 * Since end_pfn <= pfn + bitmask, the only way bits
1530 * higher than bitmask can differ in pfn and end_pfn is
1531 * by carrying. This means after masking out bitmask,
1532 * high bits starting with the first set bit in
1533 * shared_bits are all equal in both pfn and end_pfn.
1534 */
1535 shared_bits = ~(pfn ^ end_pfn) & ~bitmask;
1536 mask = shared_bits ? __ffs(shared_bits) : BITS_PER_LONG;
1537 }
1538
1539 /*
1540 * Fallback to domain selective flush if no PSI support or
1541 * the size is too big.
1542 */
1543 if (!cap_pgsel_inv(iommu->cap) ||
1544 mask > cap_max_amask_val(iommu->cap))
1545 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1546 DMA_TLB_DSI_FLUSH);
1547 else
1548 iommu->flush.flush_iotlb(iommu, did, addr | ih, mask,
1549 DMA_TLB_PSI_FLUSH);
1550 }
1551
1552 /*
1553 * In caching mode, changes of pages from non-present to present require
1554 * flush. However, device IOTLB doesn't need to be flushed in this case.
1555 */
1556 if (!cap_caching_mode(iommu->cap) || !map)
1557 iommu_flush_dev_iotlb(domain, addr, mask);
1558}
1559
1560/* Notification for newly created mappings */
1561static inline void __mapping_notify_one(struct intel_iommu *iommu,
1562 struct dmar_domain *domain,
1563 unsigned long pfn, unsigned int pages)
1564{
1565 /*
1566 * It's a non-present to present mapping. Only flush if caching mode
1567 * and second level.
1568 */
1569 if (cap_caching_mode(iommu->cap) && !domain->use_first_level)
1570 iommu_flush_iotlb_psi(iommu, domain, pfn, pages, 0, 1);
1571 else
1572 iommu_flush_write_buffer(iommu);
1573}
1574
1575static void intel_flush_iotlb_all(struct iommu_domain *domain)
1576{
1577 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
1578 struct iommu_domain_info *info;
1579 unsigned long idx;
1580
1581 xa_for_each(&dmar_domain->iommu_array, idx, info) {
1582 struct intel_iommu *iommu = info->iommu;
1583 u16 did = domain_id_iommu(dmar_domain, iommu);
1584
1585 if (dmar_domain->use_first_level)
1586 qi_flush_piotlb(iommu, did, PASID_RID2PASID, 0, -1, 0);
1587 else
1588 iommu->flush.flush_iotlb(iommu, did, 0, 0,
1589 DMA_TLB_DSI_FLUSH);
1590
1591 if (!cap_caching_mode(iommu->cap))
1592 iommu_flush_dev_iotlb(dmar_domain, 0, MAX_AGAW_PFN_WIDTH);
1593 }
1594}
1595
1596static void iommu_disable_protect_mem_regions(struct intel_iommu *iommu)
1597{
1598 u32 pmen;
1599 unsigned long flags;
1600
1601 if (!cap_plmr(iommu->cap) && !cap_phmr(iommu->cap))
1602 return;
1603
1604 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1605 pmen = readl(iommu->reg + DMAR_PMEN_REG);
1606 pmen &= ~DMA_PMEN_EPM;
1607 writel(pmen, iommu->reg + DMAR_PMEN_REG);
1608
1609 /* wait for the protected region status bit to clear */
1610 IOMMU_WAIT_OP(iommu, DMAR_PMEN_REG,
1611 readl, !(pmen & DMA_PMEN_PRS), pmen);
1612
1613 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1614}
1615
1616static void iommu_enable_translation(struct intel_iommu *iommu)
1617{
1618 u32 sts;
1619 unsigned long flags;
1620
1621 raw_spin_lock_irqsave(&iommu->register_lock, flags);
1622 iommu->gcmd |= DMA_GCMD_TE;
1623 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1624
1625 /* Make sure hardware complete it */
1626 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1627 readl, (sts & DMA_GSTS_TES), sts);
1628
1629 raw_spin_unlock_irqrestore(&iommu->register_lock, flags);
1630}
1631
1632static void iommu_disable_translation(struct intel_iommu *iommu)
1633{
1634 u32 sts;
1635 unsigned long flag;
1636
1637 if (iommu_skip_te_disable && iommu->drhd->gfx_dedicated &&
1638 (cap_read_drain(iommu->cap) || cap_write_drain(iommu->cap)))
1639 return;
1640
1641 raw_spin_lock_irqsave(&iommu->register_lock, flag);
1642 iommu->gcmd &= ~DMA_GCMD_TE;
1643 writel(iommu->gcmd, iommu->reg + DMAR_GCMD_REG);
1644
1645 /* Make sure hardware complete it */
1646 IOMMU_WAIT_OP(iommu, DMAR_GSTS_REG,
1647 readl, (!(sts & DMA_GSTS_TES)), sts);
1648
1649 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
1650}
1651
1652static int iommu_init_domains(struct intel_iommu *iommu)
1653{
1654 u32 ndomains;
1655
1656 ndomains = cap_ndoms(iommu->cap);
1657 pr_debug("%s: Number of Domains supported <%d>\n",
1658 iommu->name, ndomains);
1659
1660 spin_lock_init(&iommu->lock);
1661
1662 iommu->domain_ids = bitmap_zalloc(ndomains, GFP_KERNEL);
1663 if (!iommu->domain_ids)
1664 return -ENOMEM;
1665
1666 /*
1667 * If Caching mode is set, then invalid translations are tagged
1668 * with domain-id 0, hence we need to pre-allocate it. We also
1669 * use domain-id 0 as a marker for non-allocated domain-id, so
1670 * make sure it is not used for a real domain.
1671 */
1672 set_bit(0, iommu->domain_ids);
1673
1674 /*
1675 * Vt-d spec rev3.0 (section 6.2.3.1) requires that each pasid
1676 * entry for first-level or pass-through translation modes should
1677 * be programmed with a domain id different from those used for
1678 * second-level or nested translation. We reserve a domain id for
1679 * this purpose.
1680 */
1681 if (sm_supported(iommu))
1682 set_bit(FLPT_DEFAULT_DID, iommu->domain_ids);
1683
1684 return 0;
1685}
1686
1687static void disable_dmar_iommu(struct intel_iommu *iommu)
1688{
1689 if (!iommu->domain_ids)
1690 return;
1691
1692 /*
1693 * All iommu domains must have been detached from the devices,
1694 * hence there should be no domain IDs in use.
1695 */
1696 if (WARN_ON(bitmap_weight(iommu->domain_ids, cap_ndoms(iommu->cap))
1697 > NUM_RESERVED_DID))
1698 return;
1699
1700 if (iommu->gcmd & DMA_GCMD_TE)
1701 iommu_disable_translation(iommu);
1702}
1703
1704static void free_dmar_iommu(struct intel_iommu *iommu)
1705{
1706 if (iommu->domain_ids) {
1707 bitmap_free(iommu->domain_ids);
1708 iommu->domain_ids = NULL;
1709 }
1710
1711 if (iommu->copied_tables) {
1712 bitmap_free(iommu->copied_tables);
1713 iommu->copied_tables = NULL;
1714 }
1715
1716 /* free context mapping */
1717 free_context_table(iommu);
1718
1719#ifdef CONFIG_INTEL_IOMMU_SVM
1720 if (pasid_supported(iommu)) {
1721 if (ecap_prs(iommu->ecap))
1722 intel_svm_finish_prq(iommu);
1723 }
1724 if (vccap_pasid(iommu->vccap))
1725 ioasid_unregister_allocator(&iommu->pasid_allocator);
1726
1727#endif
1728}
1729
1730/*
1731 * Check and return whether first level is used by default for
1732 * DMA translation.
1733 */
1734static bool first_level_by_default(unsigned int type)
1735{
1736 /* Only SL is available in legacy mode */
1737 if (!scalable_mode_support())
1738 return false;
1739
1740 /* Only level (either FL or SL) is available, just use it */
1741 if (intel_cap_flts_sanity() ^ intel_cap_slts_sanity())
1742 return intel_cap_flts_sanity();
1743
1744 /* Both levels are available, decide it based on domain type */
1745 return type != IOMMU_DOMAIN_UNMANAGED;
1746}
1747
1748static struct dmar_domain *alloc_domain(unsigned int type)
1749{
1750 struct dmar_domain *domain;
1751
1752 domain = kzalloc(sizeof(*domain), GFP_KERNEL);
1753 if (!domain)
1754 return NULL;
1755
1756 domain->nid = NUMA_NO_NODE;
1757 if (first_level_by_default(type))
1758 domain->use_first_level = true;
1759 domain->has_iotlb_device = false;
1760 INIT_LIST_HEAD(&domain->devices);
1761 spin_lock_init(&domain->lock);
1762 xa_init(&domain->iommu_array);
1763
1764 return domain;
1765}
1766
1767static int domain_attach_iommu(struct dmar_domain *domain,
1768 struct intel_iommu *iommu)
1769{
1770 struct iommu_domain_info *info, *curr;
1771 unsigned long ndomains;
1772 int num, ret = -ENOSPC;
1773
1774 info = kzalloc(sizeof(*info), GFP_KERNEL);
1775 if (!info)
1776 return -ENOMEM;
1777
1778 spin_lock(&iommu->lock);
1779 curr = xa_load(&domain->iommu_array, iommu->seq_id);
1780 if (curr) {
1781 curr->refcnt++;
1782 spin_unlock(&iommu->lock);
1783 kfree(info);
1784 return 0;
1785 }
1786
1787 ndomains = cap_ndoms(iommu->cap);
1788 num = find_first_zero_bit(iommu->domain_ids, ndomains);
1789 if (num >= ndomains) {
1790 pr_err("%s: No free domain ids\n", iommu->name);
1791 goto err_unlock;
1792 }
1793
1794 set_bit(num, iommu->domain_ids);
1795 info->refcnt = 1;
1796 info->did = num;
1797 info->iommu = iommu;
1798 curr = xa_cmpxchg(&domain->iommu_array, iommu->seq_id,
1799 NULL, info, GFP_ATOMIC);
1800 if (curr) {
1801 ret = xa_err(curr) ? : -EBUSY;
1802 goto err_clear;
1803 }
1804 domain_update_iommu_cap(domain);
1805
1806 spin_unlock(&iommu->lock);
1807 return 0;
1808
1809err_clear:
1810 clear_bit(info->did, iommu->domain_ids);
1811err_unlock:
1812 spin_unlock(&iommu->lock);
1813 kfree(info);
1814 return ret;
1815}
1816
1817static void domain_detach_iommu(struct dmar_domain *domain,
1818 struct intel_iommu *iommu)
1819{
1820 struct iommu_domain_info *info;
1821
1822 spin_lock(&iommu->lock);
1823 info = xa_load(&domain->iommu_array, iommu->seq_id);
1824 if (--info->refcnt == 0) {
1825 clear_bit(info->did, iommu->domain_ids);
1826 xa_erase(&domain->iommu_array, iommu->seq_id);
1827 domain->nid = NUMA_NO_NODE;
1828 domain_update_iommu_cap(domain);
1829 kfree(info);
1830 }
1831 spin_unlock(&iommu->lock);
1832}
1833
1834static inline int guestwidth_to_adjustwidth(int gaw)
1835{
1836 int agaw;
1837 int r = (gaw - 12) % 9;
1838
1839 if (r == 0)
1840 agaw = gaw;
1841 else
1842 agaw = gaw + 9 - r;
1843 if (agaw > 64)
1844 agaw = 64;
1845 return agaw;
1846}
1847
1848static void domain_exit(struct dmar_domain *domain)
1849{
1850 if (domain->pgd) {
1851 LIST_HEAD(freelist);
1852
1853 domain_unmap(domain, 0, DOMAIN_MAX_PFN(domain->gaw), &freelist);
1854 put_pages_list(&freelist);
1855 }
1856
1857 if (WARN_ON(!list_empty(&domain->devices)))
1858 return;
1859
1860 kfree(domain);
1861}
1862
1863/*
1864 * Get the PASID directory size for scalable mode context entry.
1865 * Value of X in the PDTS field of a scalable mode context entry
1866 * indicates PASID directory with 2^(X + 7) entries.
1867 */
1868static inline unsigned long context_get_sm_pds(struct pasid_table *table)
1869{
1870 unsigned long pds, max_pde;
1871
1872 max_pde = table->max_pasid >> PASID_PDE_SHIFT;
1873 pds = find_first_bit(&max_pde, MAX_NR_PASID_BITS);
1874 if (pds < 7)
1875 return 0;
1876
1877 return pds - 7;
1878}
1879
1880/*
1881 * Set the RID_PASID field of a scalable mode context entry. The
1882 * IOMMU hardware will use the PASID value set in this field for
1883 * DMA translations of DMA requests without PASID.
1884 */
1885static inline void
1886context_set_sm_rid2pasid(struct context_entry *context, unsigned long pasid)
1887{
1888 context->hi |= pasid & ((1 << 20) - 1);
1889}
1890
1891/*
1892 * Set the DTE(Device-TLB Enable) field of a scalable mode context
1893 * entry.
1894 */
1895static inline void context_set_sm_dte(struct context_entry *context)
1896{
1897 context->lo |= (1 << 2);
1898}
1899
1900/*
1901 * Set the PRE(Page Request Enable) field of a scalable mode context
1902 * entry.
1903 */
1904static inline void context_set_sm_pre(struct context_entry *context)
1905{
1906 context->lo |= (1 << 4);
1907}
1908
1909/* Convert value to context PASID directory size field coding. */
1910#define context_pdts(pds) (((pds) & 0x7) << 9)
1911
1912static int domain_context_mapping_one(struct dmar_domain *domain,
1913 struct intel_iommu *iommu,
1914 struct pasid_table *table,
1915 u8 bus, u8 devfn)
1916{
1917 struct device_domain_info *info =
1918 domain_lookup_dev_info(domain, iommu, bus, devfn);
1919 u16 did = domain_id_iommu(domain, iommu);
1920 int translation = CONTEXT_TT_MULTI_LEVEL;
1921 struct context_entry *context;
1922 int ret;
1923
1924 WARN_ON(did == 0);
1925
1926 if (hw_pass_through && domain_type_is_si(domain))
1927 translation = CONTEXT_TT_PASS_THROUGH;
1928
1929 pr_debug("Set context mapping for %02x:%02x.%d\n",
1930 bus, PCI_SLOT(devfn), PCI_FUNC(devfn));
1931
1932 BUG_ON(!domain->pgd);
1933
1934 spin_lock(&iommu->lock);
1935 ret = -ENOMEM;
1936 context = iommu_context_addr(iommu, bus, devfn, 1);
1937 if (!context)
1938 goto out_unlock;
1939
1940 ret = 0;
1941 if (context_present(context) && !context_copied(iommu, bus, devfn))
1942 goto out_unlock;
1943
1944 /*
1945 * For kdump cases, old valid entries may be cached due to the
1946 * in-flight DMA and copied pgtable, but there is no unmapping
1947 * behaviour for them, thus we need an explicit cache flush for
1948 * the newly-mapped device. For kdump, at this point, the device
1949 * is supposed to finish reset at its driver probe stage, so no
1950 * in-flight DMA will exist, and we don't need to worry anymore
1951 * hereafter.
1952 */
1953 if (context_copied(iommu, bus, devfn)) {
1954 u16 did_old = context_domain_id(context);
1955
1956 if (did_old < cap_ndoms(iommu->cap)) {
1957 iommu->flush.flush_context(iommu, did_old,
1958 (((u16)bus) << 8) | devfn,
1959 DMA_CCMD_MASK_NOBIT,
1960 DMA_CCMD_DEVICE_INVL);
1961 iommu->flush.flush_iotlb(iommu, did_old, 0, 0,
1962 DMA_TLB_DSI_FLUSH);
1963 }
1964
1965 clear_context_copied(iommu, bus, devfn);
1966 }
1967
1968 context_clear_entry(context);
1969
1970 if (sm_supported(iommu)) {
1971 unsigned long pds;
1972
1973 WARN_ON(!table);
1974
1975 /* Setup the PASID DIR pointer: */
1976 pds = context_get_sm_pds(table);
1977 context->lo = (u64)virt_to_phys(table->table) |
1978 context_pdts(pds);
1979
1980 /* Setup the RID_PASID field: */
1981 context_set_sm_rid2pasid(context, PASID_RID2PASID);
1982
1983 /*
1984 * Setup the Device-TLB enable bit and Page request
1985 * Enable bit:
1986 */
1987 if (info && info->ats_supported)
1988 context_set_sm_dte(context);
1989 if (info && info->pri_supported)
1990 context_set_sm_pre(context);
1991 if (info && info->pasid_supported)
1992 context_set_pasid(context);
1993 } else {
1994 struct dma_pte *pgd = domain->pgd;
1995 int agaw;
1996
1997 context_set_domain_id(context, did);
1998
1999 if (translation != CONTEXT_TT_PASS_THROUGH) {
2000 /*
2001 * Skip top levels of page tables for iommu which has
2002 * less agaw than default. Unnecessary for PT mode.
2003 */
2004 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2005 ret = -ENOMEM;
2006 pgd = phys_to_virt(dma_pte_addr(pgd));
2007 if (!dma_pte_present(pgd))
2008 goto out_unlock;
2009 }
2010
2011 if (info && info->ats_supported)
2012 translation = CONTEXT_TT_DEV_IOTLB;
2013 else
2014 translation = CONTEXT_TT_MULTI_LEVEL;
2015
2016 context_set_address_root(context, virt_to_phys(pgd));
2017 context_set_address_width(context, agaw);
2018 } else {
2019 /*
2020 * In pass through mode, AW must be programmed to
2021 * indicate the largest AGAW value supported by
2022 * hardware. And ASR is ignored by hardware.
2023 */
2024 context_set_address_width(context, iommu->msagaw);
2025 }
2026
2027 context_set_translation_type(context, translation);
2028 }
2029
2030 context_set_fault_enable(context);
2031 context_set_present(context);
2032 if (!ecap_coherent(iommu->ecap))
2033 clflush_cache_range(context, sizeof(*context));
2034
2035 /*
2036 * It's a non-present to present mapping. If hardware doesn't cache
2037 * non-present entry we only need to flush the write-buffer. If the
2038 * _does_ cache non-present entries, then it does so in the special
2039 * domain #0, which we have to flush:
2040 */
2041 if (cap_caching_mode(iommu->cap)) {
2042 iommu->flush.flush_context(iommu, 0,
2043 (((u16)bus) << 8) | devfn,
2044 DMA_CCMD_MASK_NOBIT,
2045 DMA_CCMD_DEVICE_INVL);
2046 iommu->flush.flush_iotlb(iommu, did, 0, 0, DMA_TLB_DSI_FLUSH);
2047 } else {
2048 iommu_flush_write_buffer(iommu);
2049 }
2050
2051 ret = 0;
2052
2053out_unlock:
2054 spin_unlock(&iommu->lock);
2055
2056 return ret;
2057}
2058
2059struct domain_context_mapping_data {
2060 struct dmar_domain *domain;
2061 struct intel_iommu *iommu;
2062 struct pasid_table *table;
2063};
2064
2065static int domain_context_mapping_cb(struct pci_dev *pdev,
2066 u16 alias, void *opaque)
2067{
2068 struct domain_context_mapping_data *data = opaque;
2069
2070 return domain_context_mapping_one(data->domain, data->iommu,
2071 data->table, PCI_BUS_NUM(alias),
2072 alias & 0xff);
2073}
2074
2075static int
2076domain_context_mapping(struct dmar_domain *domain, struct device *dev)
2077{
2078 struct domain_context_mapping_data data;
2079 struct pasid_table *table;
2080 struct intel_iommu *iommu;
2081 u8 bus, devfn;
2082
2083 iommu = device_to_iommu(dev, &bus, &devfn);
2084 if (!iommu)
2085 return -ENODEV;
2086
2087 table = intel_pasid_get_table(dev);
2088
2089 if (!dev_is_pci(dev))
2090 return domain_context_mapping_one(domain, iommu, table,
2091 bus, devfn);
2092
2093 data.domain = domain;
2094 data.iommu = iommu;
2095 data.table = table;
2096
2097 return pci_for_each_dma_alias(to_pci_dev(dev),
2098 &domain_context_mapping_cb, &data);
2099}
2100
2101/* Returns a number of VTD pages, but aligned to MM page size */
2102static inline unsigned long aligned_nrpages(unsigned long host_addr,
2103 size_t size)
2104{
2105 host_addr &= ~PAGE_MASK;
2106 return PAGE_ALIGN(host_addr + size) >> VTD_PAGE_SHIFT;
2107}
2108
2109/* Return largest possible superpage level for a given mapping */
2110static inline int hardware_largepage_caps(struct dmar_domain *domain,
2111 unsigned long iov_pfn,
2112 unsigned long phy_pfn,
2113 unsigned long pages)
2114{
2115 int support, level = 1;
2116 unsigned long pfnmerge;
2117
2118 support = domain->iommu_superpage;
2119
2120 /* To use a large page, the virtual *and* physical addresses
2121 must be aligned to 2MiB/1GiB/etc. Lower bits set in either
2122 of them will mean we have to use smaller pages. So just
2123 merge them and check both at once. */
2124 pfnmerge = iov_pfn | phy_pfn;
2125
2126 while (support && !(pfnmerge & ~VTD_STRIDE_MASK)) {
2127 pages >>= VTD_STRIDE_SHIFT;
2128 if (!pages)
2129 break;
2130 pfnmerge >>= VTD_STRIDE_SHIFT;
2131 level++;
2132 support--;
2133 }
2134 return level;
2135}
2136
2137/*
2138 * Ensure that old small page tables are removed to make room for superpage(s).
2139 * We're going to add new large pages, so make sure we don't remove their parent
2140 * tables. The IOTLB/devTLBs should be flushed if any PDE/PTEs are cleared.
2141 */
2142static void switch_to_super_page(struct dmar_domain *domain,
2143 unsigned long start_pfn,
2144 unsigned long end_pfn, int level)
2145{
2146 unsigned long lvl_pages = lvl_to_nr_pages(level);
2147 struct iommu_domain_info *info;
2148 struct dma_pte *pte = NULL;
2149 unsigned long i;
2150
2151 while (start_pfn <= end_pfn) {
2152 if (!pte)
2153 pte = pfn_to_dma_pte(domain, start_pfn, &level);
2154
2155 if (dma_pte_present(pte)) {
2156 dma_pte_free_pagetable(domain, start_pfn,
2157 start_pfn + lvl_pages - 1,
2158 level + 1);
2159
2160 xa_for_each(&domain->iommu_array, i, info)
2161 iommu_flush_iotlb_psi(info->iommu, domain,
2162 start_pfn, lvl_pages,
2163 0, 0);
2164 }
2165
2166 pte++;
2167 start_pfn += lvl_pages;
2168 if (first_pte_in_page(pte))
2169 pte = NULL;
2170 }
2171}
2172
2173static int
2174__domain_mapping(struct dmar_domain *domain, unsigned long iov_pfn,
2175 unsigned long phys_pfn, unsigned long nr_pages, int prot)
2176{
2177 struct dma_pte *first_pte = NULL, *pte = NULL;
2178 unsigned int largepage_lvl = 0;
2179 unsigned long lvl_pages = 0;
2180 phys_addr_t pteval;
2181 u64 attr;
2182
2183 BUG_ON(!domain_pfn_supported(domain, iov_pfn + nr_pages - 1));
2184
2185 if ((prot & (DMA_PTE_READ|DMA_PTE_WRITE)) == 0)
2186 return -EINVAL;
2187
2188 attr = prot & (DMA_PTE_READ | DMA_PTE_WRITE | DMA_PTE_SNP);
2189 attr |= DMA_FL_PTE_PRESENT;
2190 if (domain->use_first_level) {
2191 attr |= DMA_FL_PTE_XD | DMA_FL_PTE_US | DMA_FL_PTE_ACCESS;
2192 if (prot & DMA_PTE_WRITE)
2193 attr |= DMA_FL_PTE_DIRTY;
2194 }
2195
2196 pteval = ((phys_addr_t)phys_pfn << VTD_PAGE_SHIFT) | attr;
2197
2198 while (nr_pages > 0) {
2199 uint64_t tmp;
2200
2201 if (!pte) {
2202 largepage_lvl = hardware_largepage_caps(domain, iov_pfn,
2203 phys_pfn, nr_pages);
2204
2205 pte = pfn_to_dma_pte(domain, iov_pfn, &largepage_lvl);
2206 if (!pte)
2207 return -ENOMEM;
2208 first_pte = pte;
2209
2210 lvl_pages = lvl_to_nr_pages(largepage_lvl);
2211
2212 /* It is large page*/
2213 if (largepage_lvl > 1) {
2214 unsigned long end_pfn;
2215 unsigned long pages_to_remove;
2216
2217 pteval |= DMA_PTE_LARGE_PAGE;
2218 pages_to_remove = min_t(unsigned long, nr_pages,
2219 nr_pte_to_next_page(pte) * lvl_pages);
2220 end_pfn = iov_pfn + pages_to_remove - 1;
2221 switch_to_super_page(domain, iov_pfn, end_pfn, largepage_lvl);
2222 } else {
2223 pteval &= ~(uint64_t)DMA_PTE_LARGE_PAGE;
2224 }
2225
2226 }
2227 /* We don't need lock here, nobody else
2228 * touches the iova range
2229 */
2230 tmp = cmpxchg64_local(&pte->val, 0ULL, pteval);
2231 if (tmp) {
2232 static int dumps = 5;
2233 pr_crit("ERROR: DMA PTE for vPFN 0x%lx already set (to %llx not %llx)\n",
2234 iov_pfn, tmp, (unsigned long long)pteval);
2235 if (dumps) {
2236 dumps--;
2237 debug_dma_dump_mappings(NULL);
2238 }
2239 WARN_ON(1);
2240 }
2241
2242 nr_pages -= lvl_pages;
2243 iov_pfn += lvl_pages;
2244 phys_pfn += lvl_pages;
2245 pteval += lvl_pages * VTD_PAGE_SIZE;
2246
2247 /* If the next PTE would be the first in a new page, then we
2248 * need to flush the cache on the entries we've just written.
2249 * And then we'll need to recalculate 'pte', so clear it and
2250 * let it get set again in the if (!pte) block above.
2251 *
2252 * If we're done (!nr_pages) we need to flush the cache too.
2253 *
2254 * Also if we've been setting superpages, we may need to
2255 * recalculate 'pte' and switch back to smaller pages for the
2256 * end of the mapping, if the trailing size is not enough to
2257 * use another superpage (i.e. nr_pages < lvl_pages).
2258 */
2259 pte++;
2260 if (!nr_pages || first_pte_in_page(pte) ||
2261 (largepage_lvl > 1 && nr_pages < lvl_pages)) {
2262 domain_flush_cache(domain, first_pte,
2263 (void *)pte - (void *)first_pte);
2264 pte = NULL;
2265 }
2266 }
2267
2268 return 0;
2269}
2270
2271static void domain_context_clear_one(struct device_domain_info *info, u8 bus, u8 devfn)
2272{
2273 struct intel_iommu *iommu = info->iommu;
2274 struct context_entry *context;
2275 u16 did_old;
2276
2277 if (!iommu)
2278 return;
2279
2280 spin_lock(&iommu->lock);
2281 context = iommu_context_addr(iommu, bus, devfn, 0);
2282 if (!context) {
2283 spin_unlock(&iommu->lock);
2284 return;
2285 }
2286
2287 if (sm_supported(iommu)) {
2288 if (hw_pass_through && domain_type_is_si(info->domain))
2289 did_old = FLPT_DEFAULT_DID;
2290 else
2291 did_old = domain_id_iommu(info->domain, iommu);
2292 } else {
2293 did_old = context_domain_id(context);
2294 }
2295
2296 context_clear_entry(context);
2297 __iommu_flush_cache(iommu, context, sizeof(*context));
2298 spin_unlock(&iommu->lock);
2299 iommu->flush.flush_context(iommu,
2300 did_old,
2301 (((u16)bus) << 8) | devfn,
2302 DMA_CCMD_MASK_NOBIT,
2303 DMA_CCMD_DEVICE_INVL);
2304
2305 if (sm_supported(iommu))
2306 qi_flush_pasid_cache(iommu, did_old, QI_PC_ALL_PASIDS, 0);
2307
2308 iommu->flush.flush_iotlb(iommu,
2309 did_old,
2310 0,
2311 0,
2312 DMA_TLB_DSI_FLUSH);
2313
2314 __iommu_flush_dev_iotlb(info, 0, MAX_AGAW_PFN_WIDTH);
2315}
2316
2317static int domain_setup_first_level(struct intel_iommu *iommu,
2318 struct dmar_domain *domain,
2319 struct device *dev,
2320 u32 pasid)
2321{
2322 struct dma_pte *pgd = domain->pgd;
2323 int agaw, level;
2324 int flags = 0;
2325
2326 /*
2327 * Skip top levels of page tables for iommu which has
2328 * less agaw than default. Unnecessary for PT mode.
2329 */
2330 for (agaw = domain->agaw; agaw > iommu->agaw; agaw--) {
2331 pgd = phys_to_virt(dma_pte_addr(pgd));
2332 if (!dma_pte_present(pgd))
2333 return -ENOMEM;
2334 }
2335
2336 level = agaw_to_level(agaw);
2337 if (level != 4 && level != 5)
2338 return -EINVAL;
2339
2340 if (pasid != PASID_RID2PASID)
2341 flags |= PASID_FLAG_SUPERVISOR_MODE;
2342 if (level == 5)
2343 flags |= PASID_FLAG_FL5LP;
2344
2345 if (domain->force_snooping)
2346 flags |= PASID_FLAG_PAGE_SNOOP;
2347
2348 return intel_pasid_setup_first_level(iommu, dev, (pgd_t *)pgd, pasid,
2349 domain_id_iommu(domain, iommu),
2350 flags);
2351}
2352
2353static bool dev_is_real_dma_subdevice(struct device *dev)
2354{
2355 return dev && dev_is_pci(dev) &&
2356 pci_real_dma_dev(to_pci_dev(dev)) != to_pci_dev(dev);
2357}
2358
2359static int iommu_domain_identity_map(struct dmar_domain *domain,
2360 unsigned long first_vpfn,
2361 unsigned long last_vpfn)
2362{
2363 /*
2364 * RMRR range might have overlap with physical memory range,
2365 * clear it first
2366 */
2367 dma_pte_clear_range(domain, first_vpfn, last_vpfn);
2368
2369 return __domain_mapping(domain, first_vpfn,
2370 first_vpfn, last_vpfn - first_vpfn + 1,
2371 DMA_PTE_READ|DMA_PTE_WRITE);
2372}
2373
2374static int md_domain_init(struct dmar_domain *domain, int guest_width);
2375
2376static int __init si_domain_init(int hw)
2377{
2378 struct dmar_rmrr_unit *rmrr;
2379 struct device *dev;
2380 int i, nid, ret;
2381
2382 si_domain = alloc_domain(IOMMU_DOMAIN_IDENTITY);
2383 if (!si_domain)
2384 return -EFAULT;
2385
2386 if (md_domain_init(si_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
2387 domain_exit(si_domain);
2388 si_domain = NULL;
2389 return -EFAULT;
2390 }
2391
2392 if (hw)
2393 return 0;
2394
2395 for_each_online_node(nid) {
2396 unsigned long start_pfn, end_pfn;
2397 int i;
2398
2399 for_each_mem_pfn_range(i, nid, &start_pfn, &end_pfn, NULL) {
2400 ret = iommu_domain_identity_map(si_domain,
2401 mm_to_dma_pfn(start_pfn),
2402 mm_to_dma_pfn(end_pfn));
2403 if (ret)
2404 return ret;
2405 }
2406 }
2407
2408 /*
2409 * Identity map the RMRRs so that devices with RMRRs could also use
2410 * the si_domain.
2411 */
2412 for_each_rmrr_units(rmrr) {
2413 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
2414 i, dev) {
2415 unsigned long long start = rmrr->base_address;
2416 unsigned long long end = rmrr->end_address;
2417
2418 if (WARN_ON(end < start ||
2419 end >> agaw_to_width(si_domain->agaw)))
2420 continue;
2421
2422 ret = iommu_domain_identity_map(si_domain,
2423 mm_to_dma_pfn(start >> PAGE_SHIFT),
2424 mm_to_dma_pfn(end >> PAGE_SHIFT));
2425 if (ret)
2426 return ret;
2427 }
2428 }
2429
2430 return 0;
2431}
2432
2433static int dmar_domain_attach_device(struct dmar_domain *domain,
2434 struct device *dev)
2435{
2436 struct device_domain_info *info = dev_iommu_priv_get(dev);
2437 struct intel_iommu *iommu;
2438 unsigned long flags;
2439 u8 bus, devfn;
2440 int ret;
2441
2442 iommu = device_to_iommu(dev, &bus, &devfn);
2443 if (!iommu)
2444 return -ENODEV;
2445
2446 ret = domain_attach_iommu(domain, iommu);
2447 if (ret)
2448 return ret;
2449 info->domain = domain;
2450 spin_lock_irqsave(&domain->lock, flags);
2451 list_add(&info->link, &domain->devices);
2452 spin_unlock_irqrestore(&domain->lock, flags);
2453
2454 /* PASID table is mandatory for a PCI device in scalable mode. */
2455 if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
2456 /* Setup the PASID entry for requests without PASID: */
2457 if (hw_pass_through && domain_type_is_si(domain))
2458 ret = intel_pasid_setup_pass_through(iommu, domain,
2459 dev, PASID_RID2PASID);
2460 else if (domain->use_first_level)
2461 ret = domain_setup_first_level(iommu, domain, dev,
2462 PASID_RID2PASID);
2463 else
2464 ret = intel_pasid_setup_second_level(iommu, domain,
2465 dev, PASID_RID2PASID);
2466 if (ret) {
2467 dev_err(dev, "Setup RID2PASID failed\n");
2468 device_block_translation(dev);
2469 return ret;
2470 }
2471 }
2472
2473 ret = domain_context_mapping(domain, dev);
2474 if (ret) {
2475 dev_err(dev, "Domain context map failed\n");
2476 device_block_translation(dev);
2477 return ret;
2478 }
2479
2480 iommu_enable_pci_caps(info);
2481
2482 return 0;
2483}
2484
2485static bool device_has_rmrr(struct device *dev)
2486{
2487 struct dmar_rmrr_unit *rmrr;
2488 struct device *tmp;
2489 int i;
2490
2491 rcu_read_lock();
2492 for_each_rmrr_units(rmrr) {
2493 /*
2494 * Return TRUE if this RMRR contains the device that
2495 * is passed in.
2496 */
2497 for_each_active_dev_scope(rmrr->devices,
2498 rmrr->devices_cnt, i, tmp)
2499 if (tmp == dev ||
2500 is_downstream_to_pci_bridge(dev, tmp)) {
2501 rcu_read_unlock();
2502 return true;
2503 }
2504 }
2505 rcu_read_unlock();
2506 return false;
2507}
2508
2509/**
2510 * device_rmrr_is_relaxable - Test whether the RMRR of this device
2511 * is relaxable (ie. is allowed to be not enforced under some conditions)
2512 * @dev: device handle
2513 *
2514 * We assume that PCI USB devices with RMRRs have them largely
2515 * for historical reasons and that the RMRR space is not actively used post
2516 * boot. This exclusion may change if vendors begin to abuse it.
2517 *
2518 * The same exception is made for graphics devices, with the requirement that
2519 * any use of the RMRR regions will be torn down before assigning the device
2520 * to a guest.
2521 *
2522 * Return: true if the RMRR is relaxable, false otherwise
2523 */
2524static bool device_rmrr_is_relaxable(struct device *dev)
2525{
2526 struct pci_dev *pdev;
2527
2528 if (!dev_is_pci(dev))
2529 return false;
2530
2531 pdev = to_pci_dev(dev);
2532 if (IS_USB_DEVICE(pdev) || IS_GFX_DEVICE(pdev))
2533 return true;
2534 else
2535 return false;
2536}
2537
2538/*
2539 * There are a couple cases where we need to restrict the functionality of
2540 * devices associated with RMRRs. The first is when evaluating a device for
2541 * identity mapping because problems exist when devices are moved in and out
2542 * of domains and their respective RMRR information is lost. This means that
2543 * a device with associated RMRRs will never be in a "passthrough" domain.
2544 * The second is use of the device through the IOMMU API. This interface
2545 * expects to have full control of the IOVA space for the device. We cannot
2546 * satisfy both the requirement that RMRR access is maintained and have an
2547 * unencumbered IOVA space. We also have no ability to quiesce the device's
2548 * use of the RMRR space or even inform the IOMMU API user of the restriction.
2549 * We therefore prevent devices associated with an RMRR from participating in
2550 * the IOMMU API, which eliminates them from device assignment.
2551 *
2552 * In both cases, devices which have relaxable RMRRs are not concerned by this
2553 * restriction. See device_rmrr_is_relaxable comment.
2554 */
2555static bool device_is_rmrr_locked(struct device *dev)
2556{
2557 if (!device_has_rmrr(dev))
2558 return false;
2559
2560 if (device_rmrr_is_relaxable(dev))
2561 return false;
2562
2563 return true;
2564}
2565
2566/*
2567 * Return the required default domain type for a specific device.
2568 *
2569 * @dev: the device in query
2570 * @startup: true if this is during early boot
2571 *
2572 * Returns:
2573 * - IOMMU_DOMAIN_DMA: device requires a dynamic mapping domain
2574 * - IOMMU_DOMAIN_IDENTITY: device requires an identical mapping domain
2575 * - 0: both identity and dynamic domains work for this device
2576 */
2577static int device_def_domain_type(struct device *dev)
2578{
2579 if (dev_is_pci(dev)) {
2580 struct pci_dev *pdev = to_pci_dev(dev);
2581
2582 if ((iommu_identity_mapping & IDENTMAP_AZALIA) && IS_AZALIA(pdev))
2583 return IOMMU_DOMAIN_IDENTITY;
2584
2585 if ((iommu_identity_mapping & IDENTMAP_GFX) && IS_GFX_DEVICE(pdev))
2586 return IOMMU_DOMAIN_IDENTITY;
2587 }
2588
2589 return 0;
2590}
2591
2592static void intel_iommu_init_qi(struct intel_iommu *iommu)
2593{
2594 /*
2595 * Start from the sane iommu hardware state.
2596 * If the queued invalidation is already initialized by us
2597 * (for example, while enabling interrupt-remapping) then
2598 * we got the things already rolling from a sane state.
2599 */
2600 if (!iommu->qi) {
2601 /*
2602 * Clear any previous faults.
2603 */
2604 dmar_fault(-1, iommu);
2605 /*
2606 * Disable queued invalidation if supported and already enabled
2607 * before OS handover.
2608 */
2609 dmar_disable_qi(iommu);
2610 }
2611
2612 if (dmar_enable_qi(iommu)) {
2613 /*
2614 * Queued Invalidate not enabled, use Register Based Invalidate
2615 */
2616 iommu->flush.flush_context = __iommu_flush_context;
2617 iommu->flush.flush_iotlb = __iommu_flush_iotlb;
2618 pr_info("%s: Using Register based invalidation\n",
2619 iommu->name);
2620 } else {
2621 iommu->flush.flush_context = qi_flush_context;
2622 iommu->flush.flush_iotlb = qi_flush_iotlb;
2623 pr_info("%s: Using Queued invalidation\n", iommu->name);
2624 }
2625}
2626
2627static int copy_context_table(struct intel_iommu *iommu,
2628 struct root_entry *old_re,
2629 struct context_entry **tbl,
2630 int bus, bool ext)
2631{
2632 int tbl_idx, pos = 0, idx, devfn, ret = 0, did;
2633 struct context_entry *new_ce = NULL, ce;
2634 struct context_entry *old_ce = NULL;
2635 struct root_entry re;
2636 phys_addr_t old_ce_phys;
2637
2638 tbl_idx = ext ? bus * 2 : bus;
2639 memcpy(&re, old_re, sizeof(re));
2640
2641 for (devfn = 0; devfn < 256; devfn++) {
2642 /* First calculate the correct index */
2643 idx = (ext ? devfn * 2 : devfn) % 256;
2644
2645 if (idx == 0) {
2646 /* First save what we may have and clean up */
2647 if (new_ce) {
2648 tbl[tbl_idx] = new_ce;
2649 __iommu_flush_cache(iommu, new_ce,
2650 VTD_PAGE_SIZE);
2651 pos = 1;
2652 }
2653
2654 if (old_ce)
2655 memunmap(old_ce);
2656
2657 ret = 0;
2658 if (devfn < 0x80)
2659 old_ce_phys = root_entry_lctp(&re);
2660 else
2661 old_ce_phys = root_entry_uctp(&re);
2662
2663 if (!old_ce_phys) {
2664 if (ext && devfn == 0) {
2665 /* No LCTP, try UCTP */
2666 devfn = 0x7f;
2667 continue;
2668 } else {
2669 goto out;
2670 }
2671 }
2672
2673 ret = -ENOMEM;
2674 old_ce = memremap(old_ce_phys, PAGE_SIZE,
2675 MEMREMAP_WB);
2676 if (!old_ce)
2677 goto out;
2678
2679 new_ce = alloc_pgtable_page(iommu->node);
2680 if (!new_ce)
2681 goto out_unmap;
2682
2683 ret = 0;
2684 }
2685
2686 /* Now copy the context entry */
2687 memcpy(&ce, old_ce + idx, sizeof(ce));
2688
2689 if (!context_present(&ce))
2690 continue;
2691
2692 did = context_domain_id(&ce);
2693 if (did >= 0 && did < cap_ndoms(iommu->cap))
2694 set_bit(did, iommu->domain_ids);
2695
2696 set_context_copied(iommu, bus, devfn);
2697 new_ce[idx] = ce;
2698 }
2699
2700 tbl[tbl_idx + pos] = new_ce;
2701
2702 __iommu_flush_cache(iommu, new_ce, VTD_PAGE_SIZE);
2703
2704out_unmap:
2705 memunmap(old_ce);
2706
2707out:
2708 return ret;
2709}
2710
2711static int copy_translation_tables(struct intel_iommu *iommu)
2712{
2713 struct context_entry **ctxt_tbls;
2714 struct root_entry *old_rt;
2715 phys_addr_t old_rt_phys;
2716 int ctxt_table_entries;
2717 u64 rtaddr_reg;
2718 int bus, ret;
2719 bool new_ext, ext;
2720
2721 rtaddr_reg = dmar_readq(iommu->reg + DMAR_RTADDR_REG);
2722 ext = !!(rtaddr_reg & DMA_RTADDR_SMT);
2723 new_ext = !!sm_supported(iommu);
2724
2725 /*
2726 * The RTT bit can only be changed when translation is disabled,
2727 * but disabling translation means to open a window for data
2728 * corruption. So bail out and don't copy anything if we would
2729 * have to change the bit.
2730 */
2731 if (new_ext != ext)
2732 return -EINVAL;
2733
2734 iommu->copied_tables = bitmap_zalloc(BIT_ULL(16), GFP_KERNEL);
2735 if (!iommu->copied_tables)
2736 return -ENOMEM;
2737
2738 old_rt_phys = rtaddr_reg & VTD_PAGE_MASK;
2739 if (!old_rt_phys)
2740 return -EINVAL;
2741
2742 old_rt = memremap(old_rt_phys, PAGE_SIZE, MEMREMAP_WB);
2743 if (!old_rt)
2744 return -ENOMEM;
2745
2746 /* This is too big for the stack - allocate it from slab */
2747 ctxt_table_entries = ext ? 512 : 256;
2748 ret = -ENOMEM;
2749 ctxt_tbls = kcalloc(ctxt_table_entries, sizeof(void *), GFP_KERNEL);
2750 if (!ctxt_tbls)
2751 goto out_unmap;
2752
2753 for (bus = 0; bus < 256; bus++) {
2754 ret = copy_context_table(iommu, &old_rt[bus],
2755 ctxt_tbls, bus, ext);
2756 if (ret) {
2757 pr_err("%s: Failed to copy context table for bus %d\n",
2758 iommu->name, bus);
2759 continue;
2760 }
2761 }
2762
2763 spin_lock(&iommu->lock);
2764
2765 /* Context tables are copied, now write them to the root_entry table */
2766 for (bus = 0; bus < 256; bus++) {
2767 int idx = ext ? bus * 2 : bus;
2768 u64 val;
2769
2770 if (ctxt_tbls[idx]) {
2771 val = virt_to_phys(ctxt_tbls[idx]) | 1;
2772 iommu->root_entry[bus].lo = val;
2773 }
2774
2775 if (!ext || !ctxt_tbls[idx + 1])
2776 continue;
2777
2778 val = virt_to_phys(ctxt_tbls[idx + 1]) | 1;
2779 iommu->root_entry[bus].hi = val;
2780 }
2781
2782 spin_unlock(&iommu->lock);
2783
2784 kfree(ctxt_tbls);
2785
2786 __iommu_flush_cache(iommu, iommu->root_entry, PAGE_SIZE);
2787
2788 ret = 0;
2789
2790out_unmap:
2791 memunmap(old_rt);
2792
2793 return ret;
2794}
2795
2796#ifdef CONFIG_INTEL_IOMMU_SVM
2797static ioasid_t intel_vcmd_ioasid_alloc(ioasid_t min, ioasid_t max, void *data)
2798{
2799 struct intel_iommu *iommu = data;
2800 ioasid_t ioasid;
2801
2802 if (!iommu)
2803 return INVALID_IOASID;
2804 /*
2805 * VT-d virtual command interface always uses the full 20 bit
2806 * PASID range. Host can partition guest PASID range based on
2807 * policies but it is out of guest's control.
2808 */
2809 if (min < PASID_MIN || max > intel_pasid_max_id)
2810 return INVALID_IOASID;
2811
2812 if (vcmd_alloc_pasid(iommu, &ioasid))
2813 return INVALID_IOASID;
2814
2815 return ioasid;
2816}
2817
2818static void intel_vcmd_ioasid_free(ioasid_t ioasid, void *data)
2819{
2820 struct intel_iommu *iommu = data;
2821
2822 if (!iommu)
2823 return;
2824 /*
2825 * Sanity check the ioasid owner is done at upper layer, e.g. VFIO
2826 * We can only free the PASID when all the devices are unbound.
2827 */
2828 if (ioasid_find(NULL, ioasid, NULL)) {
2829 pr_alert("Cannot free active IOASID %d\n", ioasid);
2830 return;
2831 }
2832 vcmd_free_pasid(iommu, ioasid);
2833}
2834
2835static void register_pasid_allocator(struct intel_iommu *iommu)
2836{
2837 /*
2838 * If we are running in the host, no need for custom allocator
2839 * in that PASIDs are allocated from the host system-wide.
2840 */
2841 if (!cap_caching_mode(iommu->cap))
2842 return;
2843
2844 if (!sm_supported(iommu)) {
2845 pr_warn("VT-d Scalable Mode not enabled, no PASID allocation\n");
2846 return;
2847 }
2848
2849 /*
2850 * Register a custom PASID allocator if we are running in a guest,
2851 * guest PASID must be obtained via virtual command interface.
2852 * There can be multiple vIOMMUs in each guest but only one allocator
2853 * is active. All vIOMMU allocators will eventually be calling the same
2854 * host allocator.
2855 */
2856 if (!vccap_pasid(iommu->vccap))
2857 return;
2858
2859 pr_info("Register custom PASID allocator\n");
2860 iommu->pasid_allocator.alloc = intel_vcmd_ioasid_alloc;
2861 iommu->pasid_allocator.free = intel_vcmd_ioasid_free;
2862 iommu->pasid_allocator.pdata = (void *)iommu;
2863 if (ioasid_register_allocator(&iommu->pasid_allocator)) {
2864 pr_warn("Custom PASID allocator failed, scalable mode disabled\n");
2865 /*
2866 * Disable scalable mode on this IOMMU if there
2867 * is no custom allocator. Mixing SM capable vIOMMU
2868 * and non-SM vIOMMU are not supported.
2869 */
2870 intel_iommu_sm = 0;
2871 }
2872}
2873#endif
2874
2875static int __init init_dmars(void)
2876{
2877 struct dmar_drhd_unit *drhd;
2878 struct intel_iommu *iommu;
2879 int ret;
2880
2881 ret = intel_cap_audit(CAP_AUDIT_STATIC_DMAR, NULL);
2882 if (ret)
2883 goto free_iommu;
2884
2885 for_each_iommu(iommu, drhd) {
2886 if (drhd->ignored) {
2887 iommu_disable_translation(iommu);
2888 continue;
2889 }
2890
2891 /*
2892 * Find the max pasid size of all IOMMU's in the system.
2893 * We need to ensure the system pasid table is no bigger
2894 * than the smallest supported.
2895 */
2896 if (pasid_supported(iommu)) {
2897 u32 temp = 2 << ecap_pss(iommu->ecap);
2898
2899 intel_pasid_max_id = min_t(u32, temp,
2900 intel_pasid_max_id);
2901 }
2902
2903 intel_iommu_init_qi(iommu);
2904
2905 ret = iommu_init_domains(iommu);
2906 if (ret)
2907 goto free_iommu;
2908
2909 init_translation_status(iommu);
2910
2911 if (translation_pre_enabled(iommu) && !is_kdump_kernel()) {
2912 iommu_disable_translation(iommu);
2913 clear_translation_pre_enabled(iommu);
2914 pr_warn("Translation was enabled for %s but we are not in kdump mode\n",
2915 iommu->name);
2916 }
2917
2918 /*
2919 * TBD:
2920 * we could share the same root & context tables
2921 * among all IOMMU's. Need to Split it later.
2922 */
2923 ret = iommu_alloc_root_entry(iommu);
2924 if (ret)
2925 goto free_iommu;
2926
2927 if (translation_pre_enabled(iommu)) {
2928 pr_info("Translation already enabled - trying to copy translation structures\n");
2929
2930 ret = copy_translation_tables(iommu);
2931 if (ret) {
2932 /*
2933 * We found the IOMMU with translation
2934 * enabled - but failed to copy over the
2935 * old root-entry table. Try to proceed
2936 * by disabling translation now and
2937 * allocating a clean root-entry table.
2938 * This might cause DMAR faults, but
2939 * probably the dump will still succeed.
2940 */
2941 pr_err("Failed to copy translation tables from previous kernel for %s\n",
2942 iommu->name);
2943 iommu_disable_translation(iommu);
2944 clear_translation_pre_enabled(iommu);
2945 } else {
2946 pr_info("Copied translation tables from previous kernel for %s\n",
2947 iommu->name);
2948 }
2949 }
2950
2951 if (!ecap_pass_through(iommu->ecap))
2952 hw_pass_through = 0;
2953 intel_svm_check(iommu);
2954 }
2955
2956 /*
2957 * Now that qi is enabled on all iommus, set the root entry and flush
2958 * caches. This is required on some Intel X58 chipsets, otherwise the
2959 * flush_context function will loop forever and the boot hangs.
2960 */
2961 for_each_active_iommu(iommu, drhd) {
2962 iommu_flush_write_buffer(iommu);
2963#ifdef CONFIG_INTEL_IOMMU_SVM
2964 register_pasid_allocator(iommu);
2965#endif
2966 iommu_set_root_entry(iommu);
2967 }
2968
2969#ifdef CONFIG_INTEL_IOMMU_BROKEN_GFX_WA
2970 dmar_map_gfx = 0;
2971#endif
2972
2973 if (!dmar_map_gfx)
2974 iommu_identity_mapping |= IDENTMAP_GFX;
2975
2976 check_tylersburg_isoch();
2977
2978 ret = si_domain_init(hw_pass_through);
2979 if (ret)
2980 goto free_iommu;
2981
2982 /*
2983 * for each drhd
2984 * enable fault log
2985 * global invalidate context cache
2986 * global invalidate iotlb
2987 * enable translation
2988 */
2989 for_each_iommu(iommu, drhd) {
2990 if (drhd->ignored) {
2991 /*
2992 * we always have to disable PMRs or DMA may fail on
2993 * this device
2994 */
2995 if (force_on)
2996 iommu_disable_protect_mem_regions(iommu);
2997 continue;
2998 }
2999
3000 iommu_flush_write_buffer(iommu);
3001
3002#ifdef CONFIG_INTEL_IOMMU_SVM
3003 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
3004 /*
3005 * Call dmar_alloc_hwirq() with dmar_global_lock held,
3006 * could cause possible lock race condition.
3007 */
3008 up_write(&dmar_global_lock);
3009 ret = intel_svm_enable_prq(iommu);
3010 down_write(&dmar_global_lock);
3011 if (ret)
3012 goto free_iommu;
3013 }
3014#endif
3015 ret = dmar_set_interrupt(iommu);
3016 if (ret)
3017 goto free_iommu;
3018 }
3019
3020 return 0;
3021
3022free_iommu:
3023 for_each_active_iommu(iommu, drhd) {
3024 disable_dmar_iommu(iommu);
3025 free_dmar_iommu(iommu);
3026 }
3027 if (si_domain) {
3028 domain_exit(si_domain);
3029 si_domain = NULL;
3030 }
3031
3032 return ret;
3033}
3034
3035static void __init init_no_remapping_devices(void)
3036{
3037 struct dmar_drhd_unit *drhd;
3038 struct device *dev;
3039 int i;
3040
3041 for_each_drhd_unit(drhd) {
3042 if (!drhd->include_all) {
3043 for_each_active_dev_scope(drhd->devices,
3044 drhd->devices_cnt, i, dev)
3045 break;
3046 /* ignore DMAR unit if no devices exist */
3047 if (i == drhd->devices_cnt)
3048 drhd->ignored = 1;
3049 }
3050 }
3051
3052 for_each_active_drhd_unit(drhd) {
3053 if (drhd->include_all)
3054 continue;
3055
3056 for_each_active_dev_scope(drhd->devices,
3057 drhd->devices_cnt, i, dev)
3058 if (!dev_is_pci(dev) || !IS_GFX_DEVICE(to_pci_dev(dev)))
3059 break;
3060 if (i < drhd->devices_cnt)
3061 continue;
3062
3063 /* This IOMMU has *only* gfx devices. Either bypass it or
3064 set the gfx_mapped flag, as appropriate */
3065 drhd->gfx_dedicated = 1;
3066 if (!dmar_map_gfx)
3067 drhd->ignored = 1;
3068 }
3069}
3070
3071#ifdef CONFIG_SUSPEND
3072static int init_iommu_hw(void)
3073{
3074 struct dmar_drhd_unit *drhd;
3075 struct intel_iommu *iommu = NULL;
3076
3077 for_each_active_iommu(iommu, drhd)
3078 if (iommu->qi)
3079 dmar_reenable_qi(iommu);
3080
3081 for_each_iommu(iommu, drhd) {
3082 if (drhd->ignored) {
3083 /*
3084 * we always have to disable PMRs or DMA may fail on
3085 * this device
3086 */
3087 if (force_on)
3088 iommu_disable_protect_mem_regions(iommu);
3089 continue;
3090 }
3091
3092 iommu_flush_write_buffer(iommu);
3093 iommu_set_root_entry(iommu);
3094 iommu_enable_translation(iommu);
3095 iommu_disable_protect_mem_regions(iommu);
3096 }
3097
3098 return 0;
3099}
3100
3101static void iommu_flush_all(void)
3102{
3103 struct dmar_drhd_unit *drhd;
3104 struct intel_iommu *iommu;
3105
3106 for_each_active_iommu(iommu, drhd) {
3107 iommu->flush.flush_context(iommu, 0, 0, 0,
3108 DMA_CCMD_GLOBAL_INVL);
3109 iommu->flush.flush_iotlb(iommu, 0, 0, 0,
3110 DMA_TLB_GLOBAL_FLUSH);
3111 }
3112}
3113
3114static int iommu_suspend(void)
3115{
3116 struct dmar_drhd_unit *drhd;
3117 struct intel_iommu *iommu = NULL;
3118 unsigned long flag;
3119
3120 for_each_active_iommu(iommu, drhd) {
3121 iommu->iommu_state = kcalloc(MAX_SR_DMAR_REGS, sizeof(u32),
3122 GFP_KERNEL);
3123 if (!iommu->iommu_state)
3124 goto nomem;
3125 }
3126
3127 iommu_flush_all();
3128
3129 for_each_active_iommu(iommu, drhd) {
3130 iommu_disable_translation(iommu);
3131
3132 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3133
3134 iommu->iommu_state[SR_DMAR_FECTL_REG] =
3135 readl(iommu->reg + DMAR_FECTL_REG);
3136 iommu->iommu_state[SR_DMAR_FEDATA_REG] =
3137 readl(iommu->reg + DMAR_FEDATA_REG);
3138 iommu->iommu_state[SR_DMAR_FEADDR_REG] =
3139 readl(iommu->reg + DMAR_FEADDR_REG);
3140 iommu->iommu_state[SR_DMAR_FEUADDR_REG] =
3141 readl(iommu->reg + DMAR_FEUADDR_REG);
3142
3143 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3144 }
3145 return 0;
3146
3147nomem:
3148 for_each_active_iommu(iommu, drhd)
3149 kfree(iommu->iommu_state);
3150
3151 return -ENOMEM;
3152}
3153
3154static void iommu_resume(void)
3155{
3156 struct dmar_drhd_unit *drhd;
3157 struct intel_iommu *iommu = NULL;
3158 unsigned long flag;
3159
3160 if (init_iommu_hw()) {
3161 if (force_on)
3162 panic("tboot: IOMMU setup failed, DMAR can not resume!\n");
3163 else
3164 WARN(1, "IOMMU setup failed, DMAR can not resume!\n");
3165 return;
3166 }
3167
3168 for_each_active_iommu(iommu, drhd) {
3169
3170 raw_spin_lock_irqsave(&iommu->register_lock, flag);
3171
3172 writel(iommu->iommu_state[SR_DMAR_FECTL_REG],
3173 iommu->reg + DMAR_FECTL_REG);
3174 writel(iommu->iommu_state[SR_DMAR_FEDATA_REG],
3175 iommu->reg + DMAR_FEDATA_REG);
3176 writel(iommu->iommu_state[SR_DMAR_FEADDR_REG],
3177 iommu->reg + DMAR_FEADDR_REG);
3178 writel(iommu->iommu_state[SR_DMAR_FEUADDR_REG],
3179 iommu->reg + DMAR_FEUADDR_REG);
3180
3181 raw_spin_unlock_irqrestore(&iommu->register_lock, flag);
3182 }
3183
3184 for_each_active_iommu(iommu, drhd)
3185 kfree(iommu->iommu_state);
3186}
3187
3188static struct syscore_ops iommu_syscore_ops = {
3189 .resume = iommu_resume,
3190 .suspend = iommu_suspend,
3191};
3192
3193static void __init init_iommu_pm_ops(void)
3194{
3195 register_syscore_ops(&iommu_syscore_ops);
3196}
3197
3198#else
3199static inline void init_iommu_pm_ops(void) {}
3200#endif /* CONFIG_PM */
3201
3202static int __init rmrr_sanity_check(struct acpi_dmar_reserved_memory *rmrr)
3203{
3204 if (!IS_ALIGNED(rmrr->base_address, PAGE_SIZE) ||
3205 !IS_ALIGNED(rmrr->end_address + 1, PAGE_SIZE) ||
3206 rmrr->end_address <= rmrr->base_address ||
3207 arch_rmrr_sanity_check(rmrr))
3208 return -EINVAL;
3209
3210 return 0;
3211}
3212
3213int __init dmar_parse_one_rmrr(struct acpi_dmar_header *header, void *arg)
3214{
3215 struct acpi_dmar_reserved_memory *rmrr;
3216 struct dmar_rmrr_unit *rmrru;
3217
3218 rmrr = (struct acpi_dmar_reserved_memory *)header;
3219 if (rmrr_sanity_check(rmrr)) {
3220 pr_warn(FW_BUG
3221 "Your BIOS is broken; bad RMRR [%#018Lx-%#018Lx]\n"
3222 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
3223 rmrr->base_address, rmrr->end_address,
3224 dmi_get_system_info(DMI_BIOS_VENDOR),
3225 dmi_get_system_info(DMI_BIOS_VERSION),
3226 dmi_get_system_info(DMI_PRODUCT_VERSION));
3227 add_taint(TAINT_FIRMWARE_WORKAROUND, LOCKDEP_STILL_OK);
3228 }
3229
3230 rmrru = kzalloc(sizeof(*rmrru), GFP_KERNEL);
3231 if (!rmrru)
3232 goto out;
3233
3234 rmrru->hdr = header;
3235
3236 rmrru->base_address = rmrr->base_address;
3237 rmrru->end_address = rmrr->end_address;
3238
3239 rmrru->devices = dmar_alloc_dev_scope((void *)(rmrr + 1),
3240 ((void *)rmrr) + rmrr->header.length,
3241 &rmrru->devices_cnt);
3242 if (rmrru->devices_cnt && rmrru->devices == NULL)
3243 goto free_rmrru;
3244
3245 list_add(&rmrru->list, &dmar_rmrr_units);
3246
3247 return 0;
3248free_rmrru:
3249 kfree(rmrru);
3250out:
3251 return -ENOMEM;
3252}
3253
3254static struct dmar_atsr_unit *dmar_find_atsr(struct acpi_dmar_atsr *atsr)
3255{
3256 struct dmar_atsr_unit *atsru;
3257 struct acpi_dmar_atsr *tmp;
3258
3259 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list,
3260 dmar_rcu_check()) {
3261 tmp = (struct acpi_dmar_atsr *)atsru->hdr;
3262 if (atsr->segment != tmp->segment)
3263 continue;
3264 if (atsr->header.length != tmp->header.length)
3265 continue;
3266 if (memcmp(atsr, tmp, atsr->header.length) == 0)
3267 return atsru;
3268 }
3269
3270 return NULL;
3271}
3272
3273int dmar_parse_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3274{
3275 struct acpi_dmar_atsr *atsr;
3276 struct dmar_atsr_unit *atsru;
3277
3278 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
3279 return 0;
3280
3281 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3282 atsru = dmar_find_atsr(atsr);
3283 if (atsru)
3284 return 0;
3285
3286 atsru = kzalloc(sizeof(*atsru) + hdr->length, GFP_KERNEL);
3287 if (!atsru)
3288 return -ENOMEM;
3289
3290 /*
3291 * If memory is allocated from slab by ACPI _DSM method, we need to
3292 * copy the memory content because the memory buffer will be freed
3293 * on return.
3294 */
3295 atsru->hdr = (void *)(atsru + 1);
3296 memcpy(atsru->hdr, hdr, hdr->length);
3297 atsru->include_all = atsr->flags & 0x1;
3298 if (!atsru->include_all) {
3299 atsru->devices = dmar_alloc_dev_scope((void *)(atsr + 1),
3300 (void *)atsr + atsr->header.length,
3301 &atsru->devices_cnt);
3302 if (atsru->devices_cnt && atsru->devices == NULL) {
3303 kfree(atsru);
3304 return -ENOMEM;
3305 }
3306 }
3307
3308 list_add_rcu(&atsru->list, &dmar_atsr_units);
3309
3310 return 0;
3311}
3312
3313static void intel_iommu_free_atsr(struct dmar_atsr_unit *atsru)
3314{
3315 dmar_free_dev_scope(&atsru->devices, &atsru->devices_cnt);
3316 kfree(atsru);
3317}
3318
3319int dmar_release_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3320{
3321 struct acpi_dmar_atsr *atsr;
3322 struct dmar_atsr_unit *atsru;
3323
3324 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3325 atsru = dmar_find_atsr(atsr);
3326 if (atsru) {
3327 list_del_rcu(&atsru->list);
3328 synchronize_rcu();
3329 intel_iommu_free_atsr(atsru);
3330 }
3331
3332 return 0;
3333}
3334
3335int dmar_check_one_atsr(struct acpi_dmar_header *hdr, void *arg)
3336{
3337 int i;
3338 struct device *dev;
3339 struct acpi_dmar_atsr *atsr;
3340 struct dmar_atsr_unit *atsru;
3341
3342 atsr = container_of(hdr, struct acpi_dmar_atsr, header);
3343 atsru = dmar_find_atsr(atsr);
3344 if (!atsru)
3345 return 0;
3346
3347 if (!atsru->include_all && atsru->devices && atsru->devices_cnt) {
3348 for_each_active_dev_scope(atsru->devices, atsru->devices_cnt,
3349 i, dev)
3350 return -EBUSY;
3351 }
3352
3353 return 0;
3354}
3355
3356static struct dmar_satc_unit *dmar_find_satc(struct acpi_dmar_satc *satc)
3357{
3358 struct dmar_satc_unit *satcu;
3359 struct acpi_dmar_satc *tmp;
3360
3361 list_for_each_entry_rcu(satcu, &dmar_satc_units, list,
3362 dmar_rcu_check()) {
3363 tmp = (struct acpi_dmar_satc *)satcu->hdr;
3364 if (satc->segment != tmp->segment)
3365 continue;
3366 if (satc->header.length != tmp->header.length)
3367 continue;
3368 if (memcmp(satc, tmp, satc->header.length) == 0)
3369 return satcu;
3370 }
3371
3372 return NULL;
3373}
3374
3375int dmar_parse_one_satc(struct acpi_dmar_header *hdr, void *arg)
3376{
3377 struct acpi_dmar_satc *satc;
3378 struct dmar_satc_unit *satcu;
3379
3380 if (system_state >= SYSTEM_RUNNING && !intel_iommu_enabled)
3381 return 0;
3382
3383 satc = container_of(hdr, struct acpi_dmar_satc, header);
3384 satcu = dmar_find_satc(satc);
3385 if (satcu)
3386 return 0;
3387
3388 satcu = kzalloc(sizeof(*satcu) + hdr->length, GFP_KERNEL);
3389 if (!satcu)
3390 return -ENOMEM;
3391
3392 satcu->hdr = (void *)(satcu + 1);
3393 memcpy(satcu->hdr, hdr, hdr->length);
3394 satcu->atc_required = satc->flags & 0x1;
3395 satcu->devices = dmar_alloc_dev_scope((void *)(satc + 1),
3396 (void *)satc + satc->header.length,
3397 &satcu->devices_cnt);
3398 if (satcu->devices_cnt && !satcu->devices) {
3399 kfree(satcu);
3400 return -ENOMEM;
3401 }
3402 list_add_rcu(&satcu->list, &dmar_satc_units);
3403
3404 return 0;
3405}
3406
3407static int intel_iommu_add(struct dmar_drhd_unit *dmaru)
3408{
3409 int sp, ret;
3410 struct intel_iommu *iommu = dmaru->iommu;
3411
3412 ret = intel_cap_audit(CAP_AUDIT_HOTPLUG_DMAR, iommu);
3413 if (ret)
3414 goto out;
3415
3416 if (hw_pass_through && !ecap_pass_through(iommu->ecap)) {
3417 pr_warn("%s: Doesn't support hardware pass through.\n",
3418 iommu->name);
3419 return -ENXIO;
3420 }
3421
3422 sp = domain_update_iommu_superpage(NULL, iommu) - 1;
3423 if (sp >= 0 && !(cap_super_page_val(iommu->cap) & (1 << sp))) {
3424 pr_warn("%s: Doesn't support large page.\n",
3425 iommu->name);
3426 return -ENXIO;
3427 }
3428
3429 /*
3430 * Disable translation if already enabled prior to OS handover.
3431 */
3432 if (iommu->gcmd & DMA_GCMD_TE)
3433 iommu_disable_translation(iommu);
3434
3435 ret = iommu_init_domains(iommu);
3436 if (ret == 0)
3437 ret = iommu_alloc_root_entry(iommu);
3438 if (ret)
3439 goto out;
3440
3441 intel_svm_check(iommu);
3442
3443 if (dmaru->ignored) {
3444 /*
3445 * we always have to disable PMRs or DMA may fail on this device
3446 */
3447 if (force_on)
3448 iommu_disable_protect_mem_regions(iommu);
3449 return 0;
3450 }
3451
3452 intel_iommu_init_qi(iommu);
3453 iommu_flush_write_buffer(iommu);
3454
3455#ifdef CONFIG_INTEL_IOMMU_SVM
3456 if (pasid_supported(iommu) && ecap_prs(iommu->ecap)) {
3457 ret = intel_svm_enable_prq(iommu);
3458 if (ret)
3459 goto disable_iommu;
3460 }
3461#endif
3462 ret = dmar_set_interrupt(iommu);
3463 if (ret)
3464 goto disable_iommu;
3465
3466 iommu_set_root_entry(iommu);
3467 iommu_enable_translation(iommu);
3468
3469 iommu_disable_protect_mem_regions(iommu);
3470 return 0;
3471
3472disable_iommu:
3473 disable_dmar_iommu(iommu);
3474out:
3475 free_dmar_iommu(iommu);
3476 return ret;
3477}
3478
3479int dmar_iommu_hotplug(struct dmar_drhd_unit *dmaru, bool insert)
3480{
3481 int ret = 0;
3482 struct intel_iommu *iommu = dmaru->iommu;
3483
3484 if (!intel_iommu_enabled)
3485 return 0;
3486 if (iommu == NULL)
3487 return -EINVAL;
3488
3489 if (insert) {
3490 ret = intel_iommu_add(dmaru);
3491 } else {
3492 disable_dmar_iommu(iommu);
3493 free_dmar_iommu(iommu);
3494 }
3495
3496 return ret;
3497}
3498
3499static void intel_iommu_free_dmars(void)
3500{
3501 struct dmar_rmrr_unit *rmrru, *rmrr_n;
3502 struct dmar_atsr_unit *atsru, *atsr_n;
3503 struct dmar_satc_unit *satcu, *satc_n;
3504
3505 list_for_each_entry_safe(rmrru, rmrr_n, &dmar_rmrr_units, list) {
3506 list_del(&rmrru->list);
3507 dmar_free_dev_scope(&rmrru->devices, &rmrru->devices_cnt);
3508 kfree(rmrru);
3509 }
3510
3511 list_for_each_entry_safe(atsru, atsr_n, &dmar_atsr_units, list) {
3512 list_del(&atsru->list);
3513 intel_iommu_free_atsr(atsru);
3514 }
3515 list_for_each_entry_safe(satcu, satc_n, &dmar_satc_units, list) {
3516 list_del(&satcu->list);
3517 dmar_free_dev_scope(&satcu->devices, &satcu->devices_cnt);
3518 kfree(satcu);
3519 }
3520}
3521
3522static struct dmar_satc_unit *dmar_find_matched_satc_unit(struct pci_dev *dev)
3523{
3524 struct dmar_satc_unit *satcu;
3525 struct acpi_dmar_satc *satc;
3526 struct device *tmp;
3527 int i;
3528
3529 dev = pci_physfn(dev);
3530 rcu_read_lock();
3531
3532 list_for_each_entry_rcu(satcu, &dmar_satc_units, list) {
3533 satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
3534 if (satc->segment != pci_domain_nr(dev->bus))
3535 continue;
3536 for_each_dev_scope(satcu->devices, satcu->devices_cnt, i, tmp)
3537 if (to_pci_dev(tmp) == dev)
3538 goto out;
3539 }
3540 satcu = NULL;
3541out:
3542 rcu_read_unlock();
3543 return satcu;
3544}
3545
3546static int dmar_ats_supported(struct pci_dev *dev, struct intel_iommu *iommu)
3547{
3548 int i, ret = 1;
3549 struct pci_bus *bus;
3550 struct pci_dev *bridge = NULL;
3551 struct device *tmp;
3552 struct acpi_dmar_atsr *atsr;
3553 struct dmar_atsr_unit *atsru;
3554 struct dmar_satc_unit *satcu;
3555
3556 dev = pci_physfn(dev);
3557 satcu = dmar_find_matched_satc_unit(dev);
3558 if (satcu)
3559 /*
3560 * This device supports ATS as it is in SATC table.
3561 * When IOMMU is in legacy mode, enabling ATS is done
3562 * automatically by HW for the device that requires
3563 * ATS, hence OS should not enable this device ATS
3564 * to avoid duplicated TLB invalidation.
3565 */
3566 return !(satcu->atc_required && !sm_supported(iommu));
3567
3568 for (bus = dev->bus; bus; bus = bus->parent) {
3569 bridge = bus->self;
3570 /* If it's an integrated device, allow ATS */
3571 if (!bridge)
3572 return 1;
3573 /* Connected via non-PCIe: no ATS */
3574 if (!pci_is_pcie(bridge) ||
3575 pci_pcie_type(bridge) == PCI_EXP_TYPE_PCI_BRIDGE)
3576 return 0;
3577 /* If we found the root port, look it up in the ATSR */
3578 if (pci_pcie_type(bridge) == PCI_EXP_TYPE_ROOT_PORT)
3579 break;
3580 }
3581
3582 rcu_read_lock();
3583 list_for_each_entry_rcu(atsru, &dmar_atsr_units, list) {
3584 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3585 if (atsr->segment != pci_domain_nr(dev->bus))
3586 continue;
3587
3588 for_each_dev_scope(atsru->devices, atsru->devices_cnt, i, tmp)
3589 if (tmp == &bridge->dev)
3590 goto out;
3591
3592 if (atsru->include_all)
3593 goto out;
3594 }
3595 ret = 0;
3596out:
3597 rcu_read_unlock();
3598
3599 return ret;
3600}
3601
3602int dmar_iommu_notify_scope_dev(struct dmar_pci_notify_info *info)
3603{
3604 int ret;
3605 struct dmar_rmrr_unit *rmrru;
3606 struct dmar_atsr_unit *atsru;
3607 struct dmar_satc_unit *satcu;
3608 struct acpi_dmar_atsr *atsr;
3609 struct acpi_dmar_reserved_memory *rmrr;
3610 struct acpi_dmar_satc *satc;
3611
3612 if (!intel_iommu_enabled && system_state >= SYSTEM_RUNNING)
3613 return 0;
3614
3615 list_for_each_entry(rmrru, &dmar_rmrr_units, list) {
3616 rmrr = container_of(rmrru->hdr,
3617 struct acpi_dmar_reserved_memory, header);
3618 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3619 ret = dmar_insert_dev_scope(info, (void *)(rmrr + 1),
3620 ((void *)rmrr) + rmrr->header.length,
3621 rmrr->segment, rmrru->devices,
3622 rmrru->devices_cnt);
3623 if (ret < 0)
3624 return ret;
3625 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
3626 dmar_remove_dev_scope(info, rmrr->segment,
3627 rmrru->devices, rmrru->devices_cnt);
3628 }
3629 }
3630
3631 list_for_each_entry(atsru, &dmar_atsr_units, list) {
3632 if (atsru->include_all)
3633 continue;
3634
3635 atsr = container_of(atsru->hdr, struct acpi_dmar_atsr, header);
3636 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3637 ret = dmar_insert_dev_scope(info, (void *)(atsr + 1),
3638 (void *)atsr + atsr->header.length,
3639 atsr->segment, atsru->devices,
3640 atsru->devices_cnt);
3641 if (ret > 0)
3642 break;
3643 else if (ret < 0)
3644 return ret;
3645 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
3646 if (dmar_remove_dev_scope(info, atsr->segment,
3647 atsru->devices, atsru->devices_cnt))
3648 break;
3649 }
3650 }
3651 list_for_each_entry(satcu, &dmar_satc_units, list) {
3652 satc = container_of(satcu->hdr, struct acpi_dmar_satc, header);
3653 if (info->event == BUS_NOTIFY_ADD_DEVICE) {
3654 ret = dmar_insert_dev_scope(info, (void *)(satc + 1),
3655 (void *)satc + satc->header.length,
3656 satc->segment, satcu->devices,
3657 satcu->devices_cnt);
3658 if (ret > 0)
3659 break;
3660 else if (ret < 0)
3661 return ret;
3662 } else if (info->event == BUS_NOTIFY_REMOVED_DEVICE) {
3663 if (dmar_remove_dev_scope(info, satc->segment,
3664 satcu->devices, satcu->devices_cnt))
3665 break;
3666 }
3667 }
3668
3669 return 0;
3670}
3671
3672static int intel_iommu_memory_notifier(struct notifier_block *nb,
3673 unsigned long val, void *v)
3674{
3675 struct memory_notify *mhp = v;
3676 unsigned long start_vpfn = mm_to_dma_pfn(mhp->start_pfn);
3677 unsigned long last_vpfn = mm_to_dma_pfn(mhp->start_pfn +
3678 mhp->nr_pages - 1);
3679
3680 switch (val) {
3681 case MEM_GOING_ONLINE:
3682 if (iommu_domain_identity_map(si_domain,
3683 start_vpfn, last_vpfn)) {
3684 pr_warn("Failed to build identity map for [%lx-%lx]\n",
3685 start_vpfn, last_vpfn);
3686 return NOTIFY_BAD;
3687 }
3688 break;
3689
3690 case MEM_OFFLINE:
3691 case MEM_CANCEL_ONLINE:
3692 {
3693 struct dmar_drhd_unit *drhd;
3694 struct intel_iommu *iommu;
3695 LIST_HEAD(freelist);
3696
3697 domain_unmap(si_domain, start_vpfn, last_vpfn, &freelist);
3698
3699 rcu_read_lock();
3700 for_each_active_iommu(iommu, drhd)
3701 iommu_flush_iotlb_psi(iommu, si_domain,
3702 start_vpfn, mhp->nr_pages,
3703 list_empty(&freelist), 0);
3704 rcu_read_unlock();
3705 put_pages_list(&freelist);
3706 }
3707 break;
3708 }
3709
3710 return NOTIFY_OK;
3711}
3712
3713static struct notifier_block intel_iommu_memory_nb = {
3714 .notifier_call = intel_iommu_memory_notifier,
3715 .priority = 0
3716};
3717
3718static void intel_disable_iommus(void)
3719{
3720 struct intel_iommu *iommu = NULL;
3721 struct dmar_drhd_unit *drhd;
3722
3723 for_each_iommu(iommu, drhd)
3724 iommu_disable_translation(iommu);
3725}
3726
3727void intel_iommu_shutdown(void)
3728{
3729 struct dmar_drhd_unit *drhd;
3730 struct intel_iommu *iommu = NULL;
3731
3732 if (no_iommu || dmar_disabled)
3733 return;
3734
3735 down_write(&dmar_global_lock);
3736
3737 /* Disable PMRs explicitly here. */
3738 for_each_iommu(iommu, drhd)
3739 iommu_disable_protect_mem_regions(iommu);
3740
3741 /* Make sure the IOMMUs are switched off */
3742 intel_disable_iommus();
3743
3744 up_write(&dmar_global_lock);
3745}
3746
3747static inline struct intel_iommu *dev_to_intel_iommu(struct device *dev)
3748{
3749 struct iommu_device *iommu_dev = dev_to_iommu_device(dev);
3750
3751 return container_of(iommu_dev, struct intel_iommu, iommu);
3752}
3753
3754static ssize_t version_show(struct device *dev,
3755 struct device_attribute *attr, char *buf)
3756{
3757 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3758 u32 ver = readl(iommu->reg + DMAR_VER_REG);
3759 return sprintf(buf, "%d:%d\n",
3760 DMAR_VER_MAJOR(ver), DMAR_VER_MINOR(ver));
3761}
3762static DEVICE_ATTR_RO(version);
3763
3764static ssize_t address_show(struct device *dev,
3765 struct device_attribute *attr, char *buf)
3766{
3767 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3768 return sprintf(buf, "%llx\n", iommu->reg_phys);
3769}
3770static DEVICE_ATTR_RO(address);
3771
3772static ssize_t cap_show(struct device *dev,
3773 struct device_attribute *attr, char *buf)
3774{
3775 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3776 return sprintf(buf, "%llx\n", iommu->cap);
3777}
3778static DEVICE_ATTR_RO(cap);
3779
3780static ssize_t ecap_show(struct device *dev,
3781 struct device_attribute *attr, char *buf)
3782{
3783 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3784 return sprintf(buf, "%llx\n", iommu->ecap);
3785}
3786static DEVICE_ATTR_RO(ecap);
3787
3788static ssize_t domains_supported_show(struct device *dev,
3789 struct device_attribute *attr, char *buf)
3790{
3791 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3792 return sprintf(buf, "%ld\n", cap_ndoms(iommu->cap));
3793}
3794static DEVICE_ATTR_RO(domains_supported);
3795
3796static ssize_t domains_used_show(struct device *dev,
3797 struct device_attribute *attr, char *buf)
3798{
3799 struct intel_iommu *iommu = dev_to_intel_iommu(dev);
3800 return sprintf(buf, "%d\n", bitmap_weight(iommu->domain_ids,
3801 cap_ndoms(iommu->cap)));
3802}
3803static DEVICE_ATTR_RO(domains_used);
3804
3805static struct attribute *intel_iommu_attrs[] = {
3806 &dev_attr_version.attr,
3807 &dev_attr_address.attr,
3808 &dev_attr_cap.attr,
3809 &dev_attr_ecap.attr,
3810 &dev_attr_domains_supported.attr,
3811 &dev_attr_domains_used.attr,
3812 NULL,
3813};
3814
3815static struct attribute_group intel_iommu_group = {
3816 .name = "intel-iommu",
3817 .attrs = intel_iommu_attrs,
3818};
3819
3820const struct attribute_group *intel_iommu_groups[] = {
3821 &intel_iommu_group,
3822 NULL,
3823};
3824
3825static inline bool has_external_pci(void)
3826{
3827 struct pci_dev *pdev = NULL;
3828
3829 for_each_pci_dev(pdev)
3830 if (pdev->external_facing) {
3831 pci_dev_put(pdev);
3832 return true;
3833 }
3834
3835 return false;
3836}
3837
3838static int __init platform_optin_force_iommu(void)
3839{
3840 if (!dmar_platform_optin() || no_platform_optin || !has_external_pci())
3841 return 0;
3842
3843 if (no_iommu || dmar_disabled)
3844 pr_info("Intel-IOMMU force enabled due to platform opt in\n");
3845
3846 /*
3847 * If Intel-IOMMU is disabled by default, we will apply identity
3848 * map for all devices except those marked as being untrusted.
3849 */
3850 if (dmar_disabled)
3851 iommu_set_default_passthrough(false);
3852
3853 dmar_disabled = 0;
3854 no_iommu = 0;
3855
3856 return 1;
3857}
3858
3859static int __init probe_acpi_namespace_devices(void)
3860{
3861 struct dmar_drhd_unit *drhd;
3862 /* To avoid a -Wunused-but-set-variable warning. */
3863 struct intel_iommu *iommu __maybe_unused;
3864 struct device *dev;
3865 int i, ret = 0;
3866
3867 for_each_active_iommu(iommu, drhd) {
3868 for_each_active_dev_scope(drhd->devices,
3869 drhd->devices_cnt, i, dev) {
3870 struct acpi_device_physical_node *pn;
3871 struct iommu_group *group;
3872 struct acpi_device *adev;
3873
3874 if (dev->bus != &acpi_bus_type)
3875 continue;
3876
3877 adev = to_acpi_device(dev);
3878 mutex_lock(&adev->physical_node_lock);
3879 list_for_each_entry(pn,
3880 &adev->physical_node_list, node) {
3881 group = iommu_group_get(pn->dev);
3882 if (group) {
3883 iommu_group_put(group);
3884 continue;
3885 }
3886
3887 ret = iommu_probe_device(pn->dev);
3888 if (ret)
3889 break;
3890 }
3891 mutex_unlock(&adev->physical_node_lock);
3892
3893 if (ret)
3894 return ret;
3895 }
3896 }
3897
3898 return 0;
3899}
3900
3901static __init int tboot_force_iommu(void)
3902{
3903 if (!tboot_enabled())
3904 return 0;
3905
3906 if (no_iommu || dmar_disabled)
3907 pr_warn("Forcing Intel-IOMMU to enabled\n");
3908
3909 dmar_disabled = 0;
3910 no_iommu = 0;
3911
3912 return 1;
3913}
3914
3915int __init intel_iommu_init(void)
3916{
3917 int ret = -ENODEV;
3918 struct dmar_drhd_unit *drhd;
3919 struct intel_iommu *iommu;
3920
3921 /*
3922 * Intel IOMMU is required for a TXT/tboot launch or platform
3923 * opt in, so enforce that.
3924 */
3925 force_on = (!intel_iommu_tboot_noforce && tboot_force_iommu()) ||
3926 platform_optin_force_iommu();
3927
3928 down_write(&dmar_global_lock);
3929 if (dmar_table_init()) {
3930 if (force_on)
3931 panic("tboot: Failed to initialize DMAR table\n");
3932 goto out_free_dmar;
3933 }
3934
3935 if (dmar_dev_scope_init() < 0) {
3936 if (force_on)
3937 panic("tboot: Failed to initialize DMAR device scope\n");
3938 goto out_free_dmar;
3939 }
3940
3941 up_write(&dmar_global_lock);
3942
3943 /*
3944 * The bus notifier takes the dmar_global_lock, so lockdep will
3945 * complain later when we register it under the lock.
3946 */
3947 dmar_register_bus_notifier();
3948
3949 down_write(&dmar_global_lock);
3950
3951 if (!no_iommu)
3952 intel_iommu_debugfs_init();
3953
3954 if (no_iommu || dmar_disabled) {
3955 /*
3956 * We exit the function here to ensure IOMMU's remapping and
3957 * mempool aren't setup, which means that the IOMMU's PMRs
3958 * won't be disabled via the call to init_dmars(). So disable
3959 * it explicitly here. The PMRs were setup by tboot prior to
3960 * calling SENTER, but the kernel is expected to reset/tear
3961 * down the PMRs.
3962 */
3963 if (intel_iommu_tboot_noforce) {
3964 for_each_iommu(iommu, drhd)
3965 iommu_disable_protect_mem_regions(iommu);
3966 }
3967
3968 /*
3969 * Make sure the IOMMUs are switched off, even when we
3970 * boot into a kexec kernel and the previous kernel left
3971 * them enabled
3972 */
3973 intel_disable_iommus();
3974 goto out_free_dmar;
3975 }
3976
3977 if (list_empty(&dmar_rmrr_units))
3978 pr_info("No RMRR found\n");
3979
3980 if (list_empty(&dmar_atsr_units))
3981 pr_info("No ATSR found\n");
3982
3983 if (list_empty(&dmar_satc_units))
3984 pr_info("No SATC found\n");
3985
3986 init_no_remapping_devices();
3987
3988 ret = init_dmars();
3989 if (ret) {
3990 if (force_on)
3991 panic("tboot: Failed to initialize DMARs\n");
3992 pr_err("Initialization failed\n");
3993 goto out_free_dmar;
3994 }
3995 up_write(&dmar_global_lock);
3996
3997 init_iommu_pm_ops();
3998
3999 down_read(&dmar_global_lock);
4000 for_each_active_iommu(iommu, drhd) {
4001 /*
4002 * The flush queue implementation does not perform
4003 * page-selective invalidations that are required for efficient
4004 * TLB flushes in virtual environments. The benefit of batching
4005 * is likely to be much lower than the overhead of synchronizing
4006 * the virtual and physical IOMMU page-tables.
4007 */
4008 if (cap_caching_mode(iommu->cap)) {
4009 pr_info_once("IOMMU batching disallowed due to virtualization\n");
4010 iommu_set_dma_strict();
4011 }
4012 iommu_device_sysfs_add(&iommu->iommu, NULL,
4013 intel_iommu_groups,
4014 "%s", iommu->name);
4015 iommu_device_register(&iommu->iommu, &intel_iommu_ops, NULL);
4016 }
4017 up_read(&dmar_global_lock);
4018
4019 if (si_domain && !hw_pass_through)
4020 register_memory_notifier(&intel_iommu_memory_nb);
4021
4022 down_read(&dmar_global_lock);
4023 if (probe_acpi_namespace_devices())
4024 pr_warn("ACPI name space devices didn't probe correctly\n");
4025
4026 /* Finally, we enable the DMA remapping hardware. */
4027 for_each_iommu(iommu, drhd) {
4028 if (!drhd->ignored && !translation_pre_enabled(iommu))
4029 iommu_enable_translation(iommu);
4030
4031 iommu_disable_protect_mem_regions(iommu);
4032 }
4033 up_read(&dmar_global_lock);
4034
4035 pr_info("Intel(R) Virtualization Technology for Directed I/O\n");
4036
4037 intel_iommu_enabled = 1;
4038
4039 return 0;
4040
4041out_free_dmar:
4042 intel_iommu_free_dmars();
4043 up_write(&dmar_global_lock);
4044 return ret;
4045}
4046
4047static int domain_context_clear_one_cb(struct pci_dev *pdev, u16 alias, void *opaque)
4048{
4049 struct device_domain_info *info = opaque;
4050
4051 domain_context_clear_one(info, PCI_BUS_NUM(alias), alias & 0xff);
4052 return 0;
4053}
4054
4055/*
4056 * NB - intel-iommu lacks any sort of reference counting for the users of
4057 * dependent devices. If multiple endpoints have intersecting dependent
4058 * devices, unbinding the driver from any one of them will possibly leave
4059 * the others unable to operate.
4060 */
4061static void domain_context_clear(struct device_domain_info *info)
4062{
4063 if (!info->iommu || !info->dev || !dev_is_pci(info->dev))
4064 return;
4065
4066 pci_for_each_dma_alias(to_pci_dev(info->dev),
4067 &domain_context_clear_one_cb, info);
4068}
4069
4070static void dmar_remove_one_dev_info(struct device *dev)
4071{
4072 struct device_domain_info *info = dev_iommu_priv_get(dev);
4073 struct dmar_domain *domain = info->domain;
4074 struct intel_iommu *iommu = info->iommu;
4075 unsigned long flags;
4076
4077 if (!dev_is_real_dma_subdevice(info->dev)) {
4078 if (dev_is_pci(info->dev) && sm_supported(iommu))
4079 intel_pasid_tear_down_entry(iommu, info->dev,
4080 PASID_RID2PASID, false);
4081
4082 iommu_disable_pci_caps(info);
4083 domain_context_clear(info);
4084 }
4085
4086 spin_lock_irqsave(&domain->lock, flags);
4087 list_del(&info->link);
4088 spin_unlock_irqrestore(&domain->lock, flags);
4089
4090 domain_detach_iommu(domain, iommu);
4091 info->domain = NULL;
4092}
4093
4094/*
4095 * Clear the page table pointer in context or pasid table entries so that
4096 * all DMA requests without PASID from the device are blocked. If the page
4097 * table has been set, clean up the data structures.
4098 */
4099static void device_block_translation(struct device *dev)
4100{
4101 struct device_domain_info *info = dev_iommu_priv_get(dev);
4102 struct intel_iommu *iommu = info->iommu;
4103 unsigned long flags;
4104
4105 iommu_disable_pci_caps(info);
4106 if (!dev_is_real_dma_subdevice(dev)) {
4107 if (sm_supported(iommu))
4108 intel_pasid_tear_down_entry(iommu, dev,
4109 PASID_RID2PASID, false);
4110 else
4111 domain_context_clear(info);
4112 }
4113
4114 if (!info->domain)
4115 return;
4116
4117 spin_lock_irqsave(&info->domain->lock, flags);
4118 list_del(&info->link);
4119 spin_unlock_irqrestore(&info->domain->lock, flags);
4120
4121 domain_detach_iommu(info->domain, iommu);
4122 info->domain = NULL;
4123}
4124
4125static int md_domain_init(struct dmar_domain *domain, int guest_width)
4126{
4127 int adjust_width;
4128
4129 /* calculate AGAW */
4130 domain->gaw = guest_width;
4131 adjust_width = guestwidth_to_adjustwidth(guest_width);
4132 domain->agaw = width_to_agaw(adjust_width);
4133
4134 domain->iommu_coherency = false;
4135 domain->iommu_superpage = 0;
4136 domain->max_addr = 0;
4137
4138 /* always allocate the top pgd */
4139 domain->pgd = alloc_pgtable_page(domain->nid);
4140 if (!domain->pgd)
4141 return -ENOMEM;
4142 domain_flush_cache(domain, domain->pgd, PAGE_SIZE);
4143 return 0;
4144}
4145
4146static int blocking_domain_attach_dev(struct iommu_domain *domain,
4147 struct device *dev)
4148{
4149 device_block_translation(dev);
4150 return 0;
4151}
4152
4153static struct iommu_domain blocking_domain = {
4154 .ops = &(const struct iommu_domain_ops) {
4155 .attach_dev = blocking_domain_attach_dev,
4156 .free = intel_iommu_domain_free
4157 }
4158};
4159
4160static struct iommu_domain *intel_iommu_domain_alloc(unsigned type)
4161{
4162 struct dmar_domain *dmar_domain;
4163 struct iommu_domain *domain;
4164
4165 switch (type) {
4166 case IOMMU_DOMAIN_BLOCKED:
4167 return &blocking_domain;
4168 case IOMMU_DOMAIN_DMA:
4169 case IOMMU_DOMAIN_DMA_FQ:
4170 case IOMMU_DOMAIN_UNMANAGED:
4171 dmar_domain = alloc_domain(type);
4172 if (!dmar_domain) {
4173 pr_err("Can't allocate dmar_domain\n");
4174 return NULL;
4175 }
4176 if (md_domain_init(dmar_domain, DEFAULT_DOMAIN_ADDRESS_WIDTH)) {
4177 pr_err("Domain initialization failed\n");
4178 domain_exit(dmar_domain);
4179 return NULL;
4180 }
4181
4182 domain = &dmar_domain->domain;
4183 domain->geometry.aperture_start = 0;
4184 domain->geometry.aperture_end =
4185 __DOMAIN_MAX_ADDR(dmar_domain->gaw);
4186 domain->geometry.force_aperture = true;
4187
4188 return domain;
4189 case IOMMU_DOMAIN_IDENTITY:
4190 return &si_domain->domain;
4191 case IOMMU_DOMAIN_SVA:
4192 return intel_svm_domain_alloc();
4193 default:
4194 return NULL;
4195 }
4196
4197 return NULL;
4198}
4199
4200static void intel_iommu_domain_free(struct iommu_domain *domain)
4201{
4202 if (domain != &si_domain->domain && domain != &blocking_domain)
4203 domain_exit(to_dmar_domain(domain));
4204}
4205
4206static int prepare_domain_attach_device(struct iommu_domain *domain,
4207 struct device *dev)
4208{
4209 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4210 struct intel_iommu *iommu;
4211 int addr_width;
4212
4213 iommu = device_to_iommu(dev, NULL, NULL);
4214 if (!iommu)
4215 return -ENODEV;
4216
4217 if (dmar_domain->force_snooping && !ecap_sc_support(iommu->ecap))
4218 return -EINVAL;
4219
4220 /* check if this iommu agaw is sufficient for max mapped address */
4221 addr_width = agaw_to_width(iommu->agaw);
4222 if (addr_width > cap_mgaw(iommu->cap))
4223 addr_width = cap_mgaw(iommu->cap);
4224
4225 if (dmar_domain->max_addr > (1LL << addr_width))
4226 return -EINVAL;
4227 dmar_domain->gaw = addr_width;
4228
4229 /*
4230 * Knock out extra levels of page tables if necessary
4231 */
4232 while (iommu->agaw < dmar_domain->agaw) {
4233 struct dma_pte *pte;
4234
4235 pte = dmar_domain->pgd;
4236 if (dma_pte_present(pte)) {
4237 dmar_domain->pgd = phys_to_virt(dma_pte_addr(pte));
4238 free_pgtable_page(pte);
4239 }
4240 dmar_domain->agaw--;
4241 }
4242
4243 return 0;
4244}
4245
4246static int intel_iommu_attach_device(struct iommu_domain *domain,
4247 struct device *dev)
4248{
4249 struct device_domain_info *info = dev_iommu_priv_get(dev);
4250 int ret;
4251
4252 if (domain->type == IOMMU_DOMAIN_UNMANAGED &&
4253 device_is_rmrr_locked(dev)) {
4254 dev_warn(dev, "Device is ineligible for IOMMU domain attach due to platform RMRR requirement. Contact your platform vendor.\n");
4255 return -EPERM;
4256 }
4257
4258 if (info->domain)
4259 device_block_translation(dev);
4260
4261 ret = prepare_domain_attach_device(domain, dev);
4262 if (ret)
4263 return ret;
4264
4265 return dmar_domain_attach_device(to_dmar_domain(domain), dev);
4266}
4267
4268static int intel_iommu_map(struct iommu_domain *domain,
4269 unsigned long iova, phys_addr_t hpa,
4270 size_t size, int iommu_prot, gfp_t gfp)
4271{
4272 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4273 u64 max_addr;
4274 int prot = 0;
4275
4276 if (iommu_prot & IOMMU_READ)
4277 prot |= DMA_PTE_READ;
4278 if (iommu_prot & IOMMU_WRITE)
4279 prot |= DMA_PTE_WRITE;
4280 if (dmar_domain->set_pte_snp)
4281 prot |= DMA_PTE_SNP;
4282
4283 max_addr = iova + size;
4284 if (dmar_domain->max_addr < max_addr) {
4285 u64 end;
4286
4287 /* check if minimum agaw is sufficient for mapped address */
4288 end = __DOMAIN_MAX_ADDR(dmar_domain->gaw) + 1;
4289 if (end < max_addr) {
4290 pr_err("%s: iommu width (%d) is not "
4291 "sufficient for the mapped address (%llx)\n",
4292 __func__, dmar_domain->gaw, max_addr);
4293 return -EFAULT;
4294 }
4295 dmar_domain->max_addr = max_addr;
4296 }
4297 /* Round up size to next multiple of PAGE_SIZE, if it and
4298 the low bits of hpa would take us onto the next page */
4299 size = aligned_nrpages(hpa, size);
4300 return __domain_mapping(dmar_domain, iova >> VTD_PAGE_SHIFT,
4301 hpa >> VTD_PAGE_SHIFT, size, prot);
4302}
4303
4304static int intel_iommu_map_pages(struct iommu_domain *domain,
4305 unsigned long iova, phys_addr_t paddr,
4306 size_t pgsize, size_t pgcount,
4307 int prot, gfp_t gfp, size_t *mapped)
4308{
4309 unsigned long pgshift = __ffs(pgsize);
4310 size_t size = pgcount << pgshift;
4311 int ret;
4312
4313 if (pgsize != SZ_4K && pgsize != SZ_2M && pgsize != SZ_1G)
4314 return -EINVAL;
4315
4316 if (!IS_ALIGNED(iova | paddr, pgsize))
4317 return -EINVAL;
4318
4319 ret = intel_iommu_map(domain, iova, paddr, size, prot, gfp);
4320 if (!ret && mapped)
4321 *mapped = size;
4322
4323 return ret;
4324}
4325
4326static size_t intel_iommu_unmap(struct iommu_domain *domain,
4327 unsigned long iova, size_t size,
4328 struct iommu_iotlb_gather *gather)
4329{
4330 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4331 unsigned long start_pfn, last_pfn;
4332 int level = 0;
4333
4334 /* Cope with horrid API which requires us to unmap more than the
4335 size argument if it happens to be a large-page mapping. */
4336 BUG_ON(!pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level));
4337
4338 if (size < VTD_PAGE_SIZE << level_to_offset_bits(level))
4339 size = VTD_PAGE_SIZE << level_to_offset_bits(level);
4340
4341 start_pfn = iova >> VTD_PAGE_SHIFT;
4342 last_pfn = (iova + size - 1) >> VTD_PAGE_SHIFT;
4343
4344 domain_unmap(dmar_domain, start_pfn, last_pfn, &gather->freelist);
4345
4346 if (dmar_domain->max_addr == iova + size)
4347 dmar_domain->max_addr = iova;
4348
4349 iommu_iotlb_gather_add_page(domain, gather, iova, size);
4350
4351 return size;
4352}
4353
4354static size_t intel_iommu_unmap_pages(struct iommu_domain *domain,
4355 unsigned long iova,
4356 size_t pgsize, size_t pgcount,
4357 struct iommu_iotlb_gather *gather)
4358{
4359 unsigned long pgshift = __ffs(pgsize);
4360 size_t size = pgcount << pgshift;
4361
4362 return intel_iommu_unmap(domain, iova, size, gather);
4363}
4364
4365static void intel_iommu_tlb_sync(struct iommu_domain *domain,
4366 struct iommu_iotlb_gather *gather)
4367{
4368 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4369 unsigned long iova_pfn = IOVA_PFN(gather->start);
4370 size_t size = gather->end - gather->start;
4371 struct iommu_domain_info *info;
4372 unsigned long start_pfn;
4373 unsigned long nrpages;
4374 unsigned long i;
4375
4376 nrpages = aligned_nrpages(gather->start, size);
4377 start_pfn = mm_to_dma_pfn(iova_pfn);
4378
4379 xa_for_each(&dmar_domain->iommu_array, i, info)
4380 iommu_flush_iotlb_psi(info->iommu, dmar_domain,
4381 start_pfn, nrpages,
4382 list_empty(&gather->freelist), 0);
4383
4384 put_pages_list(&gather->freelist);
4385}
4386
4387static phys_addr_t intel_iommu_iova_to_phys(struct iommu_domain *domain,
4388 dma_addr_t iova)
4389{
4390 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4391 struct dma_pte *pte;
4392 int level = 0;
4393 u64 phys = 0;
4394
4395 pte = pfn_to_dma_pte(dmar_domain, iova >> VTD_PAGE_SHIFT, &level);
4396 if (pte && dma_pte_present(pte))
4397 phys = dma_pte_addr(pte) +
4398 (iova & (BIT_MASK(level_to_offset_bits(level) +
4399 VTD_PAGE_SHIFT) - 1));
4400
4401 return phys;
4402}
4403
4404static bool domain_support_force_snooping(struct dmar_domain *domain)
4405{
4406 struct device_domain_info *info;
4407 bool support = true;
4408
4409 assert_spin_locked(&domain->lock);
4410 list_for_each_entry(info, &domain->devices, link) {
4411 if (!ecap_sc_support(info->iommu->ecap)) {
4412 support = false;
4413 break;
4414 }
4415 }
4416
4417 return support;
4418}
4419
4420static void domain_set_force_snooping(struct dmar_domain *domain)
4421{
4422 struct device_domain_info *info;
4423
4424 assert_spin_locked(&domain->lock);
4425 /*
4426 * Second level page table supports per-PTE snoop control. The
4427 * iommu_map() interface will handle this by setting SNP bit.
4428 */
4429 if (!domain->use_first_level) {
4430 domain->set_pte_snp = true;
4431 return;
4432 }
4433
4434 list_for_each_entry(info, &domain->devices, link)
4435 intel_pasid_setup_page_snoop_control(info->iommu, info->dev,
4436 PASID_RID2PASID);
4437}
4438
4439static bool intel_iommu_enforce_cache_coherency(struct iommu_domain *domain)
4440{
4441 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4442 unsigned long flags;
4443
4444 if (dmar_domain->force_snooping)
4445 return true;
4446
4447 spin_lock_irqsave(&dmar_domain->lock, flags);
4448 if (!domain_support_force_snooping(dmar_domain)) {
4449 spin_unlock_irqrestore(&dmar_domain->lock, flags);
4450 return false;
4451 }
4452
4453 domain_set_force_snooping(dmar_domain);
4454 dmar_domain->force_snooping = true;
4455 spin_unlock_irqrestore(&dmar_domain->lock, flags);
4456
4457 return true;
4458}
4459
4460static bool intel_iommu_capable(struct device *dev, enum iommu_cap cap)
4461{
4462 struct device_domain_info *info = dev_iommu_priv_get(dev);
4463
4464 switch (cap) {
4465 case IOMMU_CAP_CACHE_COHERENCY:
4466 return true;
4467 case IOMMU_CAP_INTR_REMAP:
4468 return irq_remapping_enabled == 1;
4469 case IOMMU_CAP_PRE_BOOT_PROTECTION:
4470 return dmar_platform_optin();
4471 case IOMMU_CAP_ENFORCE_CACHE_COHERENCY:
4472 return ecap_sc_support(info->iommu->ecap);
4473 default:
4474 return false;
4475 }
4476}
4477
4478static struct iommu_device *intel_iommu_probe_device(struct device *dev)
4479{
4480 struct pci_dev *pdev = dev_is_pci(dev) ? to_pci_dev(dev) : NULL;
4481 struct device_domain_info *info;
4482 struct intel_iommu *iommu;
4483 u8 bus, devfn;
4484 int ret;
4485
4486 iommu = device_to_iommu(dev, &bus, &devfn);
4487 if (!iommu || !iommu->iommu.ops)
4488 return ERR_PTR(-ENODEV);
4489
4490 info = kzalloc(sizeof(*info), GFP_KERNEL);
4491 if (!info)
4492 return ERR_PTR(-ENOMEM);
4493
4494 if (dev_is_real_dma_subdevice(dev)) {
4495 info->bus = pdev->bus->number;
4496 info->devfn = pdev->devfn;
4497 info->segment = pci_domain_nr(pdev->bus);
4498 } else {
4499 info->bus = bus;
4500 info->devfn = devfn;
4501 info->segment = iommu->segment;
4502 }
4503
4504 info->dev = dev;
4505 info->iommu = iommu;
4506 if (dev_is_pci(dev)) {
4507 if (ecap_dev_iotlb_support(iommu->ecap) &&
4508 pci_ats_supported(pdev) &&
4509 dmar_ats_supported(pdev, iommu)) {
4510 info->ats_supported = 1;
4511 info->dtlb_extra_inval = dev_needs_extra_dtlb_flush(pdev);
4512 }
4513 if (sm_supported(iommu)) {
4514 if (pasid_supported(iommu)) {
4515 int features = pci_pasid_features(pdev);
4516
4517 if (features >= 0)
4518 info->pasid_supported = features | 1;
4519 }
4520
4521 if (info->ats_supported && ecap_prs(iommu->ecap) &&
4522 pci_pri_supported(pdev))
4523 info->pri_supported = 1;
4524 }
4525 }
4526
4527 dev_iommu_priv_set(dev, info);
4528
4529 if (sm_supported(iommu) && !dev_is_real_dma_subdevice(dev)) {
4530 ret = intel_pasid_alloc_table(dev);
4531 if (ret) {
4532 dev_err(dev, "PASID table allocation failed\n");
4533 dev_iommu_priv_set(dev, NULL);
4534 kfree(info);
4535 return ERR_PTR(ret);
4536 }
4537 }
4538
4539 return &iommu->iommu;
4540}
4541
4542static void intel_iommu_release_device(struct device *dev)
4543{
4544 struct device_domain_info *info = dev_iommu_priv_get(dev);
4545
4546 dmar_remove_one_dev_info(dev);
4547 intel_pasid_free_table(dev);
4548 dev_iommu_priv_set(dev, NULL);
4549 kfree(info);
4550 set_dma_ops(dev, NULL);
4551}
4552
4553static void intel_iommu_probe_finalize(struct device *dev)
4554{
4555 set_dma_ops(dev, NULL);
4556 iommu_setup_dma_ops(dev, 0, U64_MAX);
4557}
4558
4559static void intel_iommu_get_resv_regions(struct device *device,
4560 struct list_head *head)
4561{
4562 int prot = DMA_PTE_READ | DMA_PTE_WRITE;
4563 struct iommu_resv_region *reg;
4564 struct dmar_rmrr_unit *rmrr;
4565 struct device *i_dev;
4566 int i;
4567
4568 rcu_read_lock();
4569 for_each_rmrr_units(rmrr) {
4570 for_each_active_dev_scope(rmrr->devices, rmrr->devices_cnt,
4571 i, i_dev) {
4572 struct iommu_resv_region *resv;
4573 enum iommu_resv_type type;
4574 size_t length;
4575
4576 if (i_dev != device &&
4577 !is_downstream_to_pci_bridge(device, i_dev))
4578 continue;
4579
4580 length = rmrr->end_address - rmrr->base_address + 1;
4581
4582 type = device_rmrr_is_relaxable(device) ?
4583 IOMMU_RESV_DIRECT_RELAXABLE : IOMMU_RESV_DIRECT;
4584
4585 resv = iommu_alloc_resv_region(rmrr->base_address,
4586 length, prot, type,
4587 GFP_ATOMIC);
4588 if (!resv)
4589 break;
4590
4591 list_add_tail(&resv->list, head);
4592 }
4593 }
4594 rcu_read_unlock();
4595
4596#ifdef CONFIG_INTEL_IOMMU_FLOPPY_WA
4597 if (dev_is_pci(device)) {
4598 struct pci_dev *pdev = to_pci_dev(device);
4599
4600 if ((pdev->class >> 8) == PCI_CLASS_BRIDGE_ISA) {
4601 reg = iommu_alloc_resv_region(0, 1UL << 24, prot,
4602 IOMMU_RESV_DIRECT_RELAXABLE,
4603 GFP_KERNEL);
4604 if (reg)
4605 list_add_tail(®->list, head);
4606 }
4607 }
4608#endif /* CONFIG_INTEL_IOMMU_FLOPPY_WA */
4609
4610 reg = iommu_alloc_resv_region(IOAPIC_RANGE_START,
4611 IOAPIC_RANGE_END - IOAPIC_RANGE_START + 1,
4612 0, IOMMU_RESV_MSI, GFP_KERNEL);
4613 if (!reg)
4614 return;
4615 list_add_tail(®->list, head);
4616}
4617
4618static struct iommu_group *intel_iommu_device_group(struct device *dev)
4619{
4620 if (dev_is_pci(dev))
4621 return pci_device_group(dev);
4622 return generic_device_group(dev);
4623}
4624
4625static int intel_iommu_enable_sva(struct device *dev)
4626{
4627 struct device_domain_info *info = dev_iommu_priv_get(dev);
4628 struct intel_iommu *iommu;
4629 int ret;
4630
4631 if (!info || dmar_disabled)
4632 return -EINVAL;
4633
4634 iommu = info->iommu;
4635 if (!iommu)
4636 return -EINVAL;
4637
4638 if (!(iommu->flags & VTD_FLAG_SVM_CAPABLE))
4639 return -ENODEV;
4640
4641 if (!info->pasid_enabled || !info->pri_enabled || !info->ats_enabled)
4642 return -EINVAL;
4643
4644 ret = iopf_queue_add_device(iommu->iopf_queue, dev);
4645 if (!ret)
4646 ret = iommu_register_device_fault_handler(dev, iommu_queue_iopf, dev);
4647
4648 return ret;
4649}
4650
4651static int intel_iommu_disable_sva(struct device *dev)
4652{
4653 struct device_domain_info *info = dev_iommu_priv_get(dev);
4654 struct intel_iommu *iommu = info->iommu;
4655 int ret;
4656
4657 ret = iommu_unregister_device_fault_handler(dev);
4658 if (!ret)
4659 ret = iopf_queue_remove_device(iommu->iopf_queue, dev);
4660
4661 return ret;
4662}
4663
4664static int intel_iommu_enable_iopf(struct device *dev)
4665{
4666 struct device_domain_info *info = dev_iommu_priv_get(dev);
4667
4668 if (info && info->pri_supported)
4669 return 0;
4670
4671 return -ENODEV;
4672}
4673
4674static int
4675intel_iommu_dev_enable_feat(struct device *dev, enum iommu_dev_features feat)
4676{
4677 switch (feat) {
4678 case IOMMU_DEV_FEAT_IOPF:
4679 return intel_iommu_enable_iopf(dev);
4680
4681 case IOMMU_DEV_FEAT_SVA:
4682 return intel_iommu_enable_sva(dev);
4683
4684 default:
4685 return -ENODEV;
4686 }
4687}
4688
4689static int
4690intel_iommu_dev_disable_feat(struct device *dev, enum iommu_dev_features feat)
4691{
4692 switch (feat) {
4693 case IOMMU_DEV_FEAT_IOPF:
4694 return 0;
4695
4696 case IOMMU_DEV_FEAT_SVA:
4697 return intel_iommu_disable_sva(dev);
4698
4699 default:
4700 return -ENODEV;
4701 }
4702}
4703
4704static bool intel_iommu_is_attach_deferred(struct device *dev)
4705{
4706 struct device_domain_info *info = dev_iommu_priv_get(dev);
4707
4708 return translation_pre_enabled(info->iommu) && !info->domain;
4709}
4710
4711/*
4712 * Check that the device does not live on an external facing PCI port that is
4713 * marked as untrusted. Such devices should not be able to apply quirks and
4714 * thus not be able to bypass the IOMMU restrictions.
4715 */
4716static bool risky_device(struct pci_dev *pdev)
4717{
4718 if (pdev->untrusted) {
4719 pci_info(pdev,
4720 "Skipping IOMMU quirk for dev [%04X:%04X] on untrusted PCI link\n",
4721 pdev->vendor, pdev->device);
4722 pci_info(pdev, "Please check with your BIOS/Platform vendor about this\n");
4723 return true;
4724 }
4725 return false;
4726}
4727
4728static void intel_iommu_iotlb_sync_map(struct iommu_domain *domain,
4729 unsigned long iova, size_t size)
4730{
4731 struct dmar_domain *dmar_domain = to_dmar_domain(domain);
4732 unsigned long pages = aligned_nrpages(iova, size);
4733 unsigned long pfn = iova >> VTD_PAGE_SHIFT;
4734 struct iommu_domain_info *info;
4735 unsigned long i;
4736
4737 xa_for_each(&dmar_domain->iommu_array, i, info)
4738 __mapping_notify_one(info->iommu, dmar_domain, pfn, pages);
4739}
4740
4741static void intel_iommu_remove_dev_pasid(struct device *dev, ioasid_t pasid)
4742{
4743 struct intel_iommu *iommu = device_to_iommu(dev, NULL, NULL);
4744 struct iommu_domain *domain;
4745
4746 /* Domain type specific cleanup: */
4747 domain = iommu_get_domain_for_dev_pasid(dev, pasid, 0);
4748 if (domain) {
4749 switch (domain->type) {
4750 case IOMMU_DOMAIN_SVA:
4751 intel_svm_remove_dev_pasid(dev, pasid);
4752 break;
4753 default:
4754 /* should never reach here */
4755 WARN_ON(1);
4756 break;
4757 }
4758 }
4759
4760 intel_pasid_tear_down_entry(iommu, dev, pasid, false);
4761}
4762
4763const struct iommu_ops intel_iommu_ops = {
4764 .capable = intel_iommu_capable,
4765 .domain_alloc = intel_iommu_domain_alloc,
4766 .probe_device = intel_iommu_probe_device,
4767 .probe_finalize = intel_iommu_probe_finalize,
4768 .release_device = intel_iommu_release_device,
4769 .get_resv_regions = intel_iommu_get_resv_regions,
4770 .device_group = intel_iommu_device_group,
4771 .dev_enable_feat = intel_iommu_dev_enable_feat,
4772 .dev_disable_feat = intel_iommu_dev_disable_feat,
4773 .is_attach_deferred = intel_iommu_is_attach_deferred,
4774 .def_domain_type = device_def_domain_type,
4775 .remove_dev_pasid = intel_iommu_remove_dev_pasid,
4776 .pgsize_bitmap = SZ_4K,
4777#ifdef CONFIG_INTEL_IOMMU_SVM
4778 .page_response = intel_svm_page_response,
4779#endif
4780 .default_domain_ops = &(const struct iommu_domain_ops) {
4781 .attach_dev = intel_iommu_attach_device,
4782 .map_pages = intel_iommu_map_pages,
4783 .unmap_pages = intel_iommu_unmap_pages,
4784 .iotlb_sync_map = intel_iommu_iotlb_sync_map,
4785 .flush_iotlb_all = intel_flush_iotlb_all,
4786 .iotlb_sync = intel_iommu_tlb_sync,
4787 .iova_to_phys = intel_iommu_iova_to_phys,
4788 .free = intel_iommu_domain_free,
4789 .enforce_cache_coherency = intel_iommu_enforce_cache_coherency,
4790 }
4791};
4792
4793static void quirk_iommu_igfx(struct pci_dev *dev)
4794{
4795 if (risky_device(dev))
4796 return;
4797
4798 pci_info(dev, "Disabling IOMMU for graphics on this chipset\n");
4799 dmar_map_gfx = 0;
4800}
4801
4802/* G4x/GM45 integrated gfx dmar support is totally busted. */
4803DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_igfx);
4804DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_igfx);
4805DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_igfx);
4806DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_igfx);
4807DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_igfx);
4808DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_igfx);
4809DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_igfx);
4810
4811/* Broadwell igfx malfunctions with dmar */
4812DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1606, quirk_iommu_igfx);
4813DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160B, quirk_iommu_igfx);
4814DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160E, quirk_iommu_igfx);
4815DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1602, quirk_iommu_igfx);
4816DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160A, quirk_iommu_igfx);
4817DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x160D, quirk_iommu_igfx);
4818DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1616, quirk_iommu_igfx);
4819DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161B, quirk_iommu_igfx);
4820DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161E, quirk_iommu_igfx);
4821DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1612, quirk_iommu_igfx);
4822DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161A, quirk_iommu_igfx);
4823DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x161D, quirk_iommu_igfx);
4824DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1626, quirk_iommu_igfx);
4825DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162B, quirk_iommu_igfx);
4826DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162E, quirk_iommu_igfx);
4827DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1622, quirk_iommu_igfx);
4828DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162A, quirk_iommu_igfx);
4829DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x162D, quirk_iommu_igfx);
4830DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1636, quirk_iommu_igfx);
4831DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163B, quirk_iommu_igfx);
4832DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163E, quirk_iommu_igfx);
4833DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x1632, quirk_iommu_igfx);
4834DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163A, quirk_iommu_igfx);
4835DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x163D, quirk_iommu_igfx);
4836
4837static void quirk_iommu_rwbf(struct pci_dev *dev)
4838{
4839 if (risky_device(dev))
4840 return;
4841
4842 /*
4843 * Mobile 4 Series Chipset neglects to set RWBF capability,
4844 * but needs it. Same seems to hold for the desktop versions.
4845 */
4846 pci_info(dev, "Forcing write-buffer flush capability\n");
4847 rwbf_quirk = 1;
4848}
4849
4850DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2a40, quirk_iommu_rwbf);
4851DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e00, quirk_iommu_rwbf);
4852DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e10, quirk_iommu_rwbf);
4853DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e20, quirk_iommu_rwbf);
4854DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e30, quirk_iommu_rwbf);
4855DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e40, quirk_iommu_rwbf);
4856DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x2e90, quirk_iommu_rwbf);
4857
4858#define GGC 0x52
4859#define GGC_MEMORY_SIZE_MASK (0xf << 8)
4860#define GGC_MEMORY_SIZE_NONE (0x0 << 8)
4861#define GGC_MEMORY_SIZE_1M (0x1 << 8)
4862#define GGC_MEMORY_SIZE_2M (0x3 << 8)
4863#define GGC_MEMORY_VT_ENABLED (0x8 << 8)
4864#define GGC_MEMORY_SIZE_2M_VT (0x9 << 8)
4865#define GGC_MEMORY_SIZE_3M_VT (0xa << 8)
4866#define GGC_MEMORY_SIZE_4M_VT (0xb << 8)
4867
4868static void quirk_calpella_no_shadow_gtt(struct pci_dev *dev)
4869{
4870 unsigned short ggc;
4871
4872 if (risky_device(dev))
4873 return;
4874
4875 if (pci_read_config_word(dev, GGC, &ggc))
4876 return;
4877
4878 if (!(ggc & GGC_MEMORY_VT_ENABLED)) {
4879 pci_info(dev, "BIOS has allocated no shadow GTT; disabling IOMMU for graphics\n");
4880 dmar_map_gfx = 0;
4881 } else if (dmar_map_gfx) {
4882 /* we have to ensure the gfx device is idle before we flush */
4883 pci_info(dev, "Disabling batched IOTLB flush on Ironlake\n");
4884 iommu_set_dma_strict();
4885 }
4886}
4887DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0040, quirk_calpella_no_shadow_gtt);
4888DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0044, quirk_calpella_no_shadow_gtt);
4889DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x0062, quirk_calpella_no_shadow_gtt);
4890DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, 0x006a, quirk_calpella_no_shadow_gtt);
4891
4892static void quirk_igfx_skip_te_disable(struct pci_dev *dev)
4893{
4894 unsigned short ver;
4895
4896 if (!IS_GFX_DEVICE(dev))
4897 return;
4898
4899 ver = (dev->device >> 8) & 0xff;
4900 if (ver != 0x45 && ver != 0x46 && ver != 0x4c &&
4901 ver != 0x4e && ver != 0x8a && ver != 0x98 &&
4902 ver != 0x9a && ver != 0xa7)
4903 return;
4904
4905 if (risky_device(dev))
4906 return;
4907
4908 pci_info(dev, "Skip IOMMU disabling for graphics\n");
4909 iommu_skip_te_disable = 1;
4910}
4911DECLARE_PCI_FIXUP_HEADER(PCI_VENDOR_ID_INTEL, PCI_ANY_ID, quirk_igfx_skip_te_disable);
4912
4913/* On Tylersburg chipsets, some BIOSes have been known to enable the
4914 ISOCH DMAR unit for the Azalia sound device, but not give it any
4915 TLB entries, which causes it to deadlock. Check for that. We do
4916 this in a function called from init_dmars(), instead of in a PCI
4917 quirk, because we don't want to print the obnoxious "BIOS broken"
4918 message if VT-d is actually disabled.
4919*/
4920static void __init check_tylersburg_isoch(void)
4921{
4922 struct pci_dev *pdev;
4923 uint32_t vtisochctrl;
4924
4925 /* If there's no Azalia in the system anyway, forget it. */
4926 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x3a3e, NULL);
4927 if (!pdev)
4928 return;
4929
4930 if (risky_device(pdev)) {
4931 pci_dev_put(pdev);
4932 return;
4933 }
4934
4935 pci_dev_put(pdev);
4936
4937 /* System Management Registers. Might be hidden, in which case
4938 we can't do the sanity check. But that's OK, because the
4939 known-broken BIOSes _don't_ actually hide it, so far. */
4940 pdev = pci_get_device(PCI_VENDOR_ID_INTEL, 0x342e, NULL);
4941 if (!pdev)
4942 return;
4943
4944 if (risky_device(pdev)) {
4945 pci_dev_put(pdev);
4946 return;
4947 }
4948
4949 if (pci_read_config_dword(pdev, 0x188, &vtisochctrl)) {
4950 pci_dev_put(pdev);
4951 return;
4952 }
4953
4954 pci_dev_put(pdev);
4955
4956 /* If Azalia DMA is routed to the non-isoch DMAR unit, fine. */
4957 if (vtisochctrl & 1)
4958 return;
4959
4960 /* Drop all bits other than the number of TLB entries */
4961 vtisochctrl &= 0x1c;
4962
4963 /* If we have the recommended number of TLB entries (16), fine. */
4964 if (vtisochctrl == 0x10)
4965 return;
4966
4967 /* Zero TLB entries? You get to ride the short bus to school. */
4968 if (!vtisochctrl) {
4969 WARN(1, "Your BIOS is broken; DMA routed to ISOCH DMAR unit but no TLB space.\n"
4970 "BIOS vendor: %s; Ver: %s; Product Version: %s\n",
4971 dmi_get_system_info(DMI_BIOS_VENDOR),
4972 dmi_get_system_info(DMI_BIOS_VERSION),
4973 dmi_get_system_info(DMI_PRODUCT_VERSION));
4974 iommu_identity_mapping |= IDENTMAP_AZALIA;
4975 return;
4976 }
4977
4978 pr_warn("Recommended TLB entries for ISOCH unit is 16; your BIOS set %d\n",
4979 vtisochctrl);
4980}
4981
4982/*
4983 * Here we deal with a device TLB defect where device may inadvertently issue ATS
4984 * invalidation completion before posted writes initiated with translated address
4985 * that utilized translations matching the invalidation address range, violating
4986 * the invalidation completion ordering.
4987 * Therefore, any use cases that cannot guarantee DMA is stopped before unmap is
4988 * vulnerable to this defect. In other words, any dTLB invalidation initiated not
4989 * under the control of the trusted/privileged host device driver must use this
4990 * quirk.
4991 * Device TLBs are invalidated under the following six conditions:
4992 * 1. Device driver does DMA API unmap IOVA
4993 * 2. Device driver unbind a PASID from a process, sva_unbind_device()
4994 * 3. PASID is torn down, after PASID cache is flushed. e.g. process
4995 * exit_mmap() due to crash
4996 * 4. Under SVA usage, called by mmu_notifier.invalidate_range() where
4997 * VM has to free pages that were unmapped
4998 * 5. Userspace driver unmaps a DMA buffer
4999 * 6. Cache invalidation in vSVA usage (upcoming)
5000 *
5001 * For #1 and #2, device drivers are responsible for stopping DMA traffic
5002 * before unmap/unbind. For #3, iommu driver gets mmu_notifier to
5003 * invalidate TLB the same way as normal user unmap which will use this quirk.
5004 * The dTLB invalidation after PASID cache flush does not need this quirk.
5005 *
5006 * As a reminder, #6 will *NEED* this quirk as we enable nested translation.
5007 */
5008void quirk_extra_dev_tlb_flush(struct device_domain_info *info,
5009 unsigned long address, unsigned long mask,
5010 u32 pasid, u16 qdep)
5011{
5012 u16 sid;
5013
5014 if (likely(!info->dtlb_extra_inval))
5015 return;
5016
5017 sid = PCI_DEVID(info->bus, info->devfn);
5018 if (pasid == PASID_RID2PASID) {
5019 qi_flush_dev_iotlb(info->iommu, sid, info->pfsid,
5020 qdep, address, mask);
5021 } else {
5022 qi_flush_dev_iotlb_pasid(info->iommu, sid, info->pfsid,
5023 pasid, qdep, address, mask);
5024 }
5025}