Loading...
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * A fairly generic DMA-API to IOMMU-API glue layer.
4 *
5 * Copyright (C) 2014-2015 ARM Ltd.
6 *
7 * based in part on arch/arm/mm/dma-mapping.c:
8 * Copyright (C) 2000-2004 Russell King
9 */
10
11#include <linux/acpi_iort.h>
12#include <linux/atomic.h>
13#include <linux/crash_dump.h>
14#include <linux/device.h>
15#include <linux/dma-direct.h>
16#include <linux/dma-map-ops.h>
17#include <linux/gfp.h>
18#include <linux/huge_mm.h>
19#include <linux/iommu.h>
20#include <linux/iova.h>
21#include <linux/irq.h>
22#include <linux/list_sort.h>
23#include <linux/memremap.h>
24#include <linux/mm.h>
25#include <linux/mutex.h>
26#include <linux/pci.h>
27#include <linux/scatterlist.h>
28#include <linux/spinlock.h>
29#include <linux/swiotlb.h>
30#include <linux/vmalloc.h>
31
32#include "dma-iommu.h"
33
34struct iommu_dma_msi_page {
35 struct list_head list;
36 dma_addr_t iova;
37 phys_addr_t phys;
38};
39
40enum iommu_dma_cookie_type {
41 IOMMU_DMA_IOVA_COOKIE,
42 IOMMU_DMA_MSI_COOKIE,
43};
44
45struct iommu_dma_cookie {
46 enum iommu_dma_cookie_type type;
47 union {
48 /* Full allocator for IOMMU_DMA_IOVA_COOKIE */
49 struct {
50 struct iova_domain iovad;
51
52 struct iova_fq __percpu *fq; /* Flush queue */
53 /* Number of TLB flushes that have been started */
54 atomic64_t fq_flush_start_cnt;
55 /* Number of TLB flushes that have been finished */
56 atomic64_t fq_flush_finish_cnt;
57 /* Timer to regularily empty the flush queues */
58 struct timer_list fq_timer;
59 /* 1 when timer is active, 0 when not */
60 atomic_t fq_timer_on;
61 };
62 /* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */
63 dma_addr_t msi_iova;
64 };
65 struct list_head msi_page_list;
66
67 /* Domain for flush queue callback; NULL if flush queue not in use */
68 struct iommu_domain *fq_domain;
69 struct mutex mutex;
70};
71
72static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled);
73bool iommu_dma_forcedac __read_mostly;
74
75static int __init iommu_dma_forcedac_setup(char *str)
76{
77 int ret = kstrtobool(str, &iommu_dma_forcedac);
78
79 if (!ret && iommu_dma_forcedac)
80 pr_info("Forcing DAC for PCI devices\n");
81 return ret;
82}
83early_param("iommu.forcedac", iommu_dma_forcedac_setup);
84
85/* Number of entries per flush queue */
86#define IOVA_FQ_SIZE 256
87
88/* Timeout (in ms) after which entries are flushed from the queue */
89#define IOVA_FQ_TIMEOUT 10
90
91/* Flush queue entry for deferred flushing */
92struct iova_fq_entry {
93 unsigned long iova_pfn;
94 unsigned long pages;
95 struct list_head freelist;
96 u64 counter; /* Flush counter when this entry was added */
97};
98
99/* Per-CPU flush queue structure */
100struct iova_fq {
101 struct iova_fq_entry entries[IOVA_FQ_SIZE];
102 unsigned int head, tail;
103 spinlock_t lock;
104};
105
106#define fq_ring_for_each(i, fq) \
107 for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
108
109static inline bool fq_full(struct iova_fq *fq)
110{
111 assert_spin_locked(&fq->lock);
112 return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
113}
114
115static inline unsigned int fq_ring_add(struct iova_fq *fq)
116{
117 unsigned int idx = fq->tail;
118
119 assert_spin_locked(&fq->lock);
120
121 fq->tail = (idx + 1) % IOVA_FQ_SIZE;
122
123 return idx;
124}
125
126static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq)
127{
128 u64 counter = atomic64_read(&cookie->fq_flush_finish_cnt);
129 unsigned int idx;
130
131 assert_spin_locked(&fq->lock);
132
133 fq_ring_for_each(idx, fq) {
134
135 if (fq->entries[idx].counter >= counter)
136 break;
137
138 put_pages_list(&fq->entries[idx].freelist);
139 free_iova_fast(&cookie->iovad,
140 fq->entries[idx].iova_pfn,
141 fq->entries[idx].pages);
142
143 fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
144 }
145}
146
147static void fq_flush_iotlb(struct iommu_dma_cookie *cookie)
148{
149 atomic64_inc(&cookie->fq_flush_start_cnt);
150 cookie->fq_domain->ops->flush_iotlb_all(cookie->fq_domain);
151 atomic64_inc(&cookie->fq_flush_finish_cnt);
152}
153
154static void fq_flush_timeout(struct timer_list *t)
155{
156 struct iommu_dma_cookie *cookie = from_timer(cookie, t, fq_timer);
157 int cpu;
158
159 atomic_set(&cookie->fq_timer_on, 0);
160 fq_flush_iotlb(cookie);
161
162 for_each_possible_cpu(cpu) {
163 unsigned long flags;
164 struct iova_fq *fq;
165
166 fq = per_cpu_ptr(cookie->fq, cpu);
167 spin_lock_irqsave(&fq->lock, flags);
168 fq_ring_free(cookie, fq);
169 spin_unlock_irqrestore(&fq->lock, flags);
170 }
171}
172
173static void queue_iova(struct iommu_dma_cookie *cookie,
174 unsigned long pfn, unsigned long pages,
175 struct list_head *freelist)
176{
177 struct iova_fq *fq;
178 unsigned long flags;
179 unsigned int idx;
180
181 /*
182 * Order against the IOMMU driver's pagetable update from unmapping
183 * @pte, to guarantee that fq_flush_iotlb() observes that if called
184 * from a different CPU before we release the lock below. Full barrier
185 * so it also pairs with iommu_dma_init_fq() to avoid seeing partially
186 * written fq state here.
187 */
188 smp_mb();
189
190 fq = raw_cpu_ptr(cookie->fq);
191 spin_lock_irqsave(&fq->lock, flags);
192
193 /*
194 * First remove all entries from the flush queue that have already been
195 * flushed out on another CPU. This makes the fq_full() check below less
196 * likely to be true.
197 */
198 fq_ring_free(cookie, fq);
199
200 if (fq_full(fq)) {
201 fq_flush_iotlb(cookie);
202 fq_ring_free(cookie, fq);
203 }
204
205 idx = fq_ring_add(fq);
206
207 fq->entries[idx].iova_pfn = pfn;
208 fq->entries[idx].pages = pages;
209 fq->entries[idx].counter = atomic64_read(&cookie->fq_flush_start_cnt);
210 list_splice(freelist, &fq->entries[idx].freelist);
211
212 spin_unlock_irqrestore(&fq->lock, flags);
213
214 /* Avoid false sharing as much as possible. */
215 if (!atomic_read(&cookie->fq_timer_on) &&
216 !atomic_xchg(&cookie->fq_timer_on, 1))
217 mod_timer(&cookie->fq_timer,
218 jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
219}
220
221static void iommu_dma_free_fq(struct iommu_dma_cookie *cookie)
222{
223 int cpu, idx;
224
225 if (!cookie->fq)
226 return;
227
228 del_timer_sync(&cookie->fq_timer);
229 /* The IOVAs will be torn down separately, so just free our queued pages */
230 for_each_possible_cpu(cpu) {
231 struct iova_fq *fq = per_cpu_ptr(cookie->fq, cpu);
232
233 fq_ring_for_each(idx, fq)
234 put_pages_list(&fq->entries[idx].freelist);
235 }
236
237 free_percpu(cookie->fq);
238}
239
240/* sysfs updates are serialised by the mutex of the group owning @domain */
241int iommu_dma_init_fq(struct iommu_domain *domain)
242{
243 struct iommu_dma_cookie *cookie = domain->iova_cookie;
244 struct iova_fq __percpu *queue;
245 int i, cpu;
246
247 if (cookie->fq_domain)
248 return 0;
249
250 atomic64_set(&cookie->fq_flush_start_cnt, 0);
251 atomic64_set(&cookie->fq_flush_finish_cnt, 0);
252
253 queue = alloc_percpu(struct iova_fq);
254 if (!queue) {
255 pr_warn("iova flush queue initialization failed\n");
256 return -ENOMEM;
257 }
258
259 for_each_possible_cpu(cpu) {
260 struct iova_fq *fq = per_cpu_ptr(queue, cpu);
261
262 fq->head = 0;
263 fq->tail = 0;
264
265 spin_lock_init(&fq->lock);
266
267 for (i = 0; i < IOVA_FQ_SIZE; i++)
268 INIT_LIST_HEAD(&fq->entries[i].freelist);
269 }
270
271 cookie->fq = queue;
272
273 timer_setup(&cookie->fq_timer, fq_flush_timeout, 0);
274 atomic_set(&cookie->fq_timer_on, 0);
275 /*
276 * Prevent incomplete fq state being observable. Pairs with path from
277 * __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova()
278 */
279 smp_wmb();
280 WRITE_ONCE(cookie->fq_domain, domain);
281 return 0;
282}
283
284static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
285{
286 if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
287 return cookie->iovad.granule;
288 return PAGE_SIZE;
289}
290
291static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type)
292{
293 struct iommu_dma_cookie *cookie;
294
295 cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
296 if (cookie) {
297 INIT_LIST_HEAD(&cookie->msi_page_list);
298 cookie->type = type;
299 }
300 return cookie;
301}
302
303/**
304 * iommu_get_dma_cookie - Acquire DMA-API resources for a domain
305 * @domain: IOMMU domain to prepare for DMA-API usage
306 */
307int iommu_get_dma_cookie(struct iommu_domain *domain)
308{
309 if (domain->iova_cookie)
310 return -EEXIST;
311
312 domain->iova_cookie = cookie_alloc(IOMMU_DMA_IOVA_COOKIE);
313 if (!domain->iova_cookie)
314 return -ENOMEM;
315
316 mutex_init(&domain->iova_cookie->mutex);
317 return 0;
318}
319
320/**
321 * iommu_get_msi_cookie - Acquire just MSI remapping resources
322 * @domain: IOMMU domain to prepare
323 * @base: Start address of IOVA region for MSI mappings
324 *
325 * Users who manage their own IOVA allocation and do not want DMA API support,
326 * but would still like to take advantage of automatic MSI remapping, can use
327 * this to initialise their own domain appropriately. Users should reserve a
328 * contiguous IOVA region, starting at @base, large enough to accommodate the
329 * number of PAGE_SIZE mappings necessary to cover every MSI doorbell address
330 * used by the devices attached to @domain.
331 */
332int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)
333{
334 struct iommu_dma_cookie *cookie;
335
336 if (domain->type != IOMMU_DOMAIN_UNMANAGED)
337 return -EINVAL;
338
339 if (domain->iova_cookie)
340 return -EEXIST;
341
342 cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE);
343 if (!cookie)
344 return -ENOMEM;
345
346 cookie->msi_iova = base;
347 domain->iova_cookie = cookie;
348 return 0;
349}
350EXPORT_SYMBOL(iommu_get_msi_cookie);
351
352/**
353 * iommu_put_dma_cookie - Release a domain's DMA mapping resources
354 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() or
355 * iommu_get_msi_cookie()
356 */
357void iommu_put_dma_cookie(struct iommu_domain *domain)
358{
359 struct iommu_dma_cookie *cookie = domain->iova_cookie;
360 struct iommu_dma_msi_page *msi, *tmp;
361
362 if (!cookie)
363 return;
364
365 if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {
366 iommu_dma_free_fq(cookie);
367 put_iova_domain(&cookie->iovad);
368 }
369
370 list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
371 list_del(&msi->list);
372 kfree(msi);
373 }
374 kfree(cookie);
375 domain->iova_cookie = NULL;
376}
377
378/**
379 * iommu_dma_get_resv_regions - Reserved region driver helper
380 * @dev: Device from iommu_get_resv_regions()
381 * @list: Reserved region list from iommu_get_resv_regions()
382 *
383 * IOMMU drivers can use this to implement their .get_resv_regions callback
384 * for general non-IOMMU-specific reservations. Currently, this covers GICv3
385 * ITS region reservation on ACPI based ARM platforms that may require HW MSI
386 * reservation.
387 */
388void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list)
389{
390
391 if (!is_of_node(dev_iommu_fwspec_get(dev)->iommu_fwnode))
392 iort_iommu_get_resv_regions(dev, list);
393
394}
395EXPORT_SYMBOL(iommu_dma_get_resv_regions);
396
397static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie,
398 phys_addr_t start, phys_addr_t end)
399{
400 struct iova_domain *iovad = &cookie->iovad;
401 struct iommu_dma_msi_page *msi_page;
402 int i, num_pages;
403
404 start -= iova_offset(iovad, start);
405 num_pages = iova_align(iovad, end - start) >> iova_shift(iovad);
406
407 for (i = 0; i < num_pages; i++) {
408 msi_page = kmalloc(sizeof(*msi_page), GFP_KERNEL);
409 if (!msi_page)
410 return -ENOMEM;
411
412 msi_page->phys = start;
413 msi_page->iova = start;
414 INIT_LIST_HEAD(&msi_page->list);
415 list_add(&msi_page->list, &cookie->msi_page_list);
416 start += iovad->granule;
417 }
418
419 return 0;
420}
421
422static int iommu_dma_ranges_sort(void *priv, const struct list_head *a,
423 const struct list_head *b)
424{
425 struct resource_entry *res_a = list_entry(a, typeof(*res_a), node);
426 struct resource_entry *res_b = list_entry(b, typeof(*res_b), node);
427
428 return res_a->res->start > res_b->res->start;
429}
430
431static int iova_reserve_pci_windows(struct pci_dev *dev,
432 struct iova_domain *iovad)
433{
434 struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus);
435 struct resource_entry *window;
436 unsigned long lo, hi;
437 phys_addr_t start = 0, end;
438
439 resource_list_for_each_entry(window, &bridge->windows) {
440 if (resource_type(window->res) != IORESOURCE_MEM)
441 continue;
442
443 lo = iova_pfn(iovad, window->res->start - window->offset);
444 hi = iova_pfn(iovad, window->res->end - window->offset);
445 reserve_iova(iovad, lo, hi);
446 }
447
448 /* Get reserved DMA windows from host bridge */
449 list_sort(NULL, &bridge->dma_ranges, iommu_dma_ranges_sort);
450 resource_list_for_each_entry(window, &bridge->dma_ranges) {
451 end = window->res->start - window->offset;
452resv_iova:
453 if (end > start) {
454 lo = iova_pfn(iovad, start);
455 hi = iova_pfn(iovad, end);
456 reserve_iova(iovad, lo, hi);
457 } else if (end < start) {
458 /* DMA ranges should be non-overlapping */
459 dev_err(&dev->dev,
460 "Failed to reserve IOVA [%pa-%pa]\n",
461 &start, &end);
462 return -EINVAL;
463 }
464
465 start = window->res->end - window->offset + 1;
466 /* If window is last entry */
467 if (window->node.next == &bridge->dma_ranges &&
468 end != ~(phys_addr_t)0) {
469 end = ~(phys_addr_t)0;
470 goto resv_iova;
471 }
472 }
473
474 return 0;
475}
476
477static int iova_reserve_iommu_regions(struct device *dev,
478 struct iommu_domain *domain)
479{
480 struct iommu_dma_cookie *cookie = domain->iova_cookie;
481 struct iova_domain *iovad = &cookie->iovad;
482 struct iommu_resv_region *region;
483 LIST_HEAD(resv_regions);
484 int ret = 0;
485
486 if (dev_is_pci(dev)) {
487 ret = iova_reserve_pci_windows(to_pci_dev(dev), iovad);
488 if (ret)
489 return ret;
490 }
491
492 iommu_get_resv_regions(dev, &resv_regions);
493 list_for_each_entry(region, &resv_regions, list) {
494 unsigned long lo, hi;
495
496 /* We ARE the software that manages these! */
497 if (region->type == IOMMU_RESV_SW_MSI)
498 continue;
499
500 lo = iova_pfn(iovad, region->start);
501 hi = iova_pfn(iovad, region->start + region->length - 1);
502 reserve_iova(iovad, lo, hi);
503
504 if (region->type == IOMMU_RESV_MSI)
505 ret = cookie_init_hw_msi_region(cookie, region->start,
506 region->start + region->length);
507 if (ret)
508 break;
509 }
510 iommu_put_resv_regions(dev, &resv_regions);
511
512 return ret;
513}
514
515static bool dev_is_untrusted(struct device *dev)
516{
517 return dev_is_pci(dev) && to_pci_dev(dev)->untrusted;
518}
519
520static bool dev_use_swiotlb(struct device *dev)
521{
522 return IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev);
523}
524
525/**
526 * iommu_dma_init_domain - Initialise a DMA mapping domain
527 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
528 * @base: IOVA at which the mappable address space starts
529 * @limit: Last address of the IOVA space
530 * @dev: Device the domain is being initialised for
531 *
532 * @base and @limit + 1 should be exact multiples of IOMMU page granularity to
533 * avoid rounding surprises. If necessary, we reserve the page at address 0
534 * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but
535 * any change which could make prior IOVAs invalid will fail.
536 */
537static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
538 dma_addr_t limit, struct device *dev)
539{
540 struct iommu_dma_cookie *cookie = domain->iova_cookie;
541 unsigned long order, base_pfn;
542 struct iova_domain *iovad;
543 int ret;
544
545 if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
546 return -EINVAL;
547
548 iovad = &cookie->iovad;
549
550 /* Use the smallest supported page size for IOVA granularity */
551 order = __ffs(domain->pgsize_bitmap);
552 base_pfn = max_t(unsigned long, 1, base >> order);
553
554 /* Check the domain allows at least some access to the device... */
555 if (domain->geometry.force_aperture) {
556 if (base > domain->geometry.aperture_end ||
557 limit < domain->geometry.aperture_start) {
558 pr_warn("specified DMA range outside IOMMU capability\n");
559 return -EFAULT;
560 }
561 /* ...then finally give it a kicking to make sure it fits */
562 base_pfn = max_t(unsigned long, base_pfn,
563 domain->geometry.aperture_start >> order);
564 }
565
566 /* start_pfn is always nonzero for an already-initialised domain */
567 mutex_lock(&cookie->mutex);
568 if (iovad->start_pfn) {
569 if (1UL << order != iovad->granule ||
570 base_pfn != iovad->start_pfn) {
571 pr_warn("Incompatible range for DMA domain\n");
572 ret = -EFAULT;
573 goto done_unlock;
574 }
575
576 ret = 0;
577 goto done_unlock;
578 }
579
580 init_iova_domain(iovad, 1UL << order, base_pfn);
581 ret = iova_domain_init_rcaches(iovad);
582 if (ret)
583 goto done_unlock;
584
585 /* If the FQ fails we can simply fall back to strict mode */
586 if (domain->type == IOMMU_DOMAIN_DMA_FQ && iommu_dma_init_fq(domain))
587 domain->type = IOMMU_DOMAIN_DMA;
588
589 ret = iova_reserve_iommu_regions(dev, domain);
590
591done_unlock:
592 mutex_unlock(&cookie->mutex);
593 return ret;
594}
595
596/**
597 * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API
598 * page flags.
599 * @dir: Direction of DMA transfer
600 * @coherent: Is the DMA master cache-coherent?
601 * @attrs: DMA attributes for the mapping
602 *
603 * Return: corresponding IOMMU API page protection flags
604 */
605static int dma_info_to_prot(enum dma_data_direction dir, bool coherent,
606 unsigned long attrs)
607{
608 int prot = coherent ? IOMMU_CACHE : 0;
609
610 if (attrs & DMA_ATTR_PRIVILEGED)
611 prot |= IOMMU_PRIV;
612
613 switch (dir) {
614 case DMA_BIDIRECTIONAL:
615 return prot | IOMMU_READ | IOMMU_WRITE;
616 case DMA_TO_DEVICE:
617 return prot | IOMMU_READ;
618 case DMA_FROM_DEVICE:
619 return prot | IOMMU_WRITE;
620 default:
621 return 0;
622 }
623}
624
625static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
626 size_t size, u64 dma_limit, struct device *dev)
627{
628 struct iommu_dma_cookie *cookie = domain->iova_cookie;
629 struct iova_domain *iovad = &cookie->iovad;
630 unsigned long shift, iova_len, iova = 0;
631
632 if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
633 cookie->msi_iova += size;
634 return cookie->msi_iova - size;
635 }
636
637 shift = iova_shift(iovad);
638 iova_len = size >> shift;
639
640 dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit);
641
642 if (domain->geometry.force_aperture)
643 dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end);
644
645 /* Try to get PCI devices a SAC address */
646 if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && dev_is_pci(dev))
647 iova = alloc_iova_fast(iovad, iova_len,
648 DMA_BIT_MASK(32) >> shift, false);
649
650 if (!iova)
651 iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift,
652 true);
653
654 return (dma_addr_t)iova << shift;
655}
656
657static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
658 dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)
659{
660 struct iova_domain *iovad = &cookie->iovad;
661
662 /* The MSI case is only ever cleaning up its most recent allocation */
663 if (cookie->type == IOMMU_DMA_MSI_COOKIE)
664 cookie->msi_iova -= size;
665 else if (gather && gather->queued)
666 queue_iova(cookie, iova_pfn(iovad, iova),
667 size >> iova_shift(iovad),
668 &gather->freelist);
669 else
670 free_iova_fast(iovad, iova_pfn(iovad, iova),
671 size >> iova_shift(iovad));
672}
673
674static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
675 size_t size)
676{
677 struct iommu_domain *domain = iommu_get_dma_domain(dev);
678 struct iommu_dma_cookie *cookie = domain->iova_cookie;
679 struct iova_domain *iovad = &cookie->iovad;
680 size_t iova_off = iova_offset(iovad, dma_addr);
681 struct iommu_iotlb_gather iotlb_gather;
682 size_t unmapped;
683
684 dma_addr -= iova_off;
685 size = iova_align(iovad, size + iova_off);
686 iommu_iotlb_gather_init(&iotlb_gather);
687 iotlb_gather.queued = READ_ONCE(cookie->fq_domain);
688
689 unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather);
690 WARN_ON(unmapped != size);
691
692 if (!iotlb_gather.queued)
693 iommu_iotlb_sync(domain, &iotlb_gather);
694 iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
695}
696
697static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
698 size_t size, int prot, u64 dma_mask)
699{
700 struct iommu_domain *domain = iommu_get_dma_domain(dev);
701 struct iommu_dma_cookie *cookie = domain->iova_cookie;
702 struct iova_domain *iovad = &cookie->iovad;
703 size_t iova_off = iova_offset(iovad, phys);
704 dma_addr_t iova;
705
706 if (static_branch_unlikely(&iommu_deferred_attach_enabled) &&
707 iommu_deferred_attach(dev, domain))
708 return DMA_MAPPING_ERROR;
709
710 size = iova_align(iovad, size + iova_off);
711
712 iova = iommu_dma_alloc_iova(domain, size, dma_mask, dev);
713 if (!iova)
714 return DMA_MAPPING_ERROR;
715
716 if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) {
717 iommu_dma_free_iova(cookie, iova, size, NULL);
718 return DMA_MAPPING_ERROR;
719 }
720 return iova + iova_off;
721}
722
723static void __iommu_dma_free_pages(struct page **pages, int count)
724{
725 while (count--)
726 __free_page(pages[count]);
727 kvfree(pages);
728}
729
730static struct page **__iommu_dma_alloc_pages(struct device *dev,
731 unsigned int count, unsigned long order_mask, gfp_t gfp)
732{
733 struct page **pages;
734 unsigned int i = 0, nid = dev_to_node(dev);
735
736 order_mask &= (2U << MAX_ORDER) - 1;
737 if (!order_mask)
738 return NULL;
739
740 pages = kvcalloc(count, sizeof(*pages), GFP_KERNEL);
741 if (!pages)
742 return NULL;
743
744 /* IOMMU can map any pages, so himem can also be used here */
745 gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
746
747 while (count) {
748 struct page *page = NULL;
749 unsigned int order_size;
750
751 /*
752 * Higher-order allocations are a convenience rather
753 * than a necessity, hence using __GFP_NORETRY until
754 * falling back to minimum-order allocations.
755 */
756 for (order_mask &= (2U << __fls(count)) - 1;
757 order_mask; order_mask &= ~order_size) {
758 unsigned int order = __fls(order_mask);
759 gfp_t alloc_flags = gfp;
760
761 order_size = 1U << order;
762 if (order_mask > order_size)
763 alloc_flags |= __GFP_NORETRY;
764 page = alloc_pages_node(nid, alloc_flags, order);
765 if (!page)
766 continue;
767 if (order)
768 split_page(page, order);
769 break;
770 }
771 if (!page) {
772 __iommu_dma_free_pages(pages, i);
773 return NULL;
774 }
775 count -= order_size;
776 while (order_size--)
777 pages[i++] = page++;
778 }
779 return pages;
780}
781
782/*
783 * If size is less than PAGE_SIZE, then a full CPU page will be allocated,
784 * but an IOMMU which supports smaller pages might not map the whole thing.
785 */
786static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev,
787 size_t size, struct sg_table *sgt, gfp_t gfp, pgprot_t prot,
788 unsigned long attrs)
789{
790 struct iommu_domain *domain = iommu_get_dma_domain(dev);
791 struct iommu_dma_cookie *cookie = domain->iova_cookie;
792 struct iova_domain *iovad = &cookie->iovad;
793 bool coherent = dev_is_dma_coherent(dev);
794 int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
795 unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap;
796 struct page **pages;
797 dma_addr_t iova;
798 ssize_t ret;
799
800 if (static_branch_unlikely(&iommu_deferred_attach_enabled) &&
801 iommu_deferred_attach(dev, domain))
802 return NULL;
803
804 min_size = alloc_sizes & -alloc_sizes;
805 if (min_size < PAGE_SIZE) {
806 min_size = PAGE_SIZE;
807 alloc_sizes |= PAGE_SIZE;
808 } else {
809 size = ALIGN(size, min_size);
810 }
811 if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES)
812 alloc_sizes = min_size;
813
814 count = PAGE_ALIGN(size) >> PAGE_SHIFT;
815 pages = __iommu_dma_alloc_pages(dev, count, alloc_sizes >> PAGE_SHIFT,
816 gfp);
817 if (!pages)
818 return NULL;
819
820 size = iova_align(iovad, size);
821 iova = iommu_dma_alloc_iova(domain, size, dev->coherent_dma_mask, dev);
822 if (!iova)
823 goto out_free_pages;
824
825 if (sg_alloc_table_from_pages(sgt, pages, count, 0, size, GFP_KERNEL))
826 goto out_free_iova;
827
828 if (!(ioprot & IOMMU_CACHE)) {
829 struct scatterlist *sg;
830 int i;
831
832 for_each_sg(sgt->sgl, sg, sgt->orig_nents, i)
833 arch_dma_prep_coherent(sg_page(sg), sg->length);
834 }
835
836 ret = iommu_map_sg_atomic(domain, iova, sgt->sgl, sgt->orig_nents, ioprot);
837 if (ret < 0 || ret < size)
838 goto out_free_sg;
839
840 sgt->sgl->dma_address = iova;
841 sgt->sgl->dma_length = size;
842 return pages;
843
844out_free_sg:
845 sg_free_table(sgt);
846out_free_iova:
847 iommu_dma_free_iova(cookie, iova, size, NULL);
848out_free_pages:
849 __iommu_dma_free_pages(pages, count);
850 return NULL;
851}
852
853static void *iommu_dma_alloc_remap(struct device *dev, size_t size,
854 dma_addr_t *dma_handle, gfp_t gfp, pgprot_t prot,
855 unsigned long attrs)
856{
857 struct page **pages;
858 struct sg_table sgt;
859 void *vaddr;
860
861 pages = __iommu_dma_alloc_noncontiguous(dev, size, &sgt, gfp, prot,
862 attrs);
863 if (!pages)
864 return NULL;
865 *dma_handle = sgt.sgl->dma_address;
866 sg_free_table(&sgt);
867 vaddr = dma_common_pages_remap(pages, size, prot,
868 __builtin_return_address(0));
869 if (!vaddr)
870 goto out_unmap;
871 return vaddr;
872
873out_unmap:
874 __iommu_dma_unmap(dev, *dma_handle, size);
875 __iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT);
876 return NULL;
877}
878
879static struct sg_table *iommu_dma_alloc_noncontiguous(struct device *dev,
880 size_t size, enum dma_data_direction dir, gfp_t gfp,
881 unsigned long attrs)
882{
883 struct dma_sgt_handle *sh;
884
885 sh = kmalloc(sizeof(*sh), gfp);
886 if (!sh)
887 return NULL;
888
889 sh->pages = __iommu_dma_alloc_noncontiguous(dev, size, &sh->sgt, gfp,
890 PAGE_KERNEL, attrs);
891 if (!sh->pages) {
892 kfree(sh);
893 return NULL;
894 }
895 return &sh->sgt;
896}
897
898static void iommu_dma_free_noncontiguous(struct device *dev, size_t size,
899 struct sg_table *sgt, enum dma_data_direction dir)
900{
901 struct dma_sgt_handle *sh = sgt_handle(sgt);
902
903 __iommu_dma_unmap(dev, sgt->sgl->dma_address, size);
904 __iommu_dma_free_pages(sh->pages, PAGE_ALIGN(size) >> PAGE_SHIFT);
905 sg_free_table(&sh->sgt);
906 kfree(sh);
907}
908
909static void iommu_dma_sync_single_for_cpu(struct device *dev,
910 dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
911{
912 phys_addr_t phys;
913
914 if (dev_is_dma_coherent(dev) && !dev_use_swiotlb(dev))
915 return;
916
917 phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
918 if (!dev_is_dma_coherent(dev))
919 arch_sync_dma_for_cpu(phys, size, dir);
920
921 if (is_swiotlb_buffer(dev, phys))
922 swiotlb_sync_single_for_cpu(dev, phys, size, dir);
923}
924
925static void iommu_dma_sync_single_for_device(struct device *dev,
926 dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
927{
928 phys_addr_t phys;
929
930 if (dev_is_dma_coherent(dev) && !dev_use_swiotlb(dev))
931 return;
932
933 phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
934 if (is_swiotlb_buffer(dev, phys))
935 swiotlb_sync_single_for_device(dev, phys, size, dir);
936
937 if (!dev_is_dma_coherent(dev))
938 arch_sync_dma_for_device(phys, size, dir);
939}
940
941static void iommu_dma_sync_sg_for_cpu(struct device *dev,
942 struct scatterlist *sgl, int nelems,
943 enum dma_data_direction dir)
944{
945 struct scatterlist *sg;
946 int i;
947
948 if (dev_use_swiotlb(dev))
949 for_each_sg(sgl, sg, nelems, i)
950 iommu_dma_sync_single_for_cpu(dev, sg_dma_address(sg),
951 sg->length, dir);
952 else if (!dev_is_dma_coherent(dev))
953 for_each_sg(sgl, sg, nelems, i)
954 arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir);
955}
956
957static void iommu_dma_sync_sg_for_device(struct device *dev,
958 struct scatterlist *sgl, int nelems,
959 enum dma_data_direction dir)
960{
961 struct scatterlist *sg;
962 int i;
963
964 if (dev_use_swiotlb(dev))
965 for_each_sg(sgl, sg, nelems, i)
966 iommu_dma_sync_single_for_device(dev,
967 sg_dma_address(sg),
968 sg->length, dir);
969 else if (!dev_is_dma_coherent(dev))
970 for_each_sg(sgl, sg, nelems, i)
971 arch_sync_dma_for_device(sg_phys(sg), sg->length, dir);
972}
973
974static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
975 unsigned long offset, size_t size, enum dma_data_direction dir,
976 unsigned long attrs)
977{
978 phys_addr_t phys = page_to_phys(page) + offset;
979 bool coherent = dev_is_dma_coherent(dev);
980 int prot = dma_info_to_prot(dir, coherent, attrs);
981 struct iommu_domain *domain = iommu_get_dma_domain(dev);
982 struct iommu_dma_cookie *cookie = domain->iova_cookie;
983 struct iova_domain *iovad = &cookie->iovad;
984 dma_addr_t iova, dma_mask = dma_get_mask(dev);
985
986 /*
987 * If both the physical buffer start address and size are
988 * page aligned, we don't need to use a bounce page.
989 */
990 if (dev_use_swiotlb(dev) && iova_offset(iovad, phys | size)) {
991 void *padding_start;
992 size_t padding_size, aligned_size;
993
994 if (!is_swiotlb_active(dev)) {
995 dev_warn_once(dev, "DMA bounce buffers are inactive, unable to map unaligned transaction.\n");
996 return DMA_MAPPING_ERROR;
997 }
998
999 aligned_size = iova_align(iovad, size);
1000 phys = swiotlb_tbl_map_single(dev, phys, size, aligned_size,
1001 iova_mask(iovad), dir, attrs);
1002
1003 if (phys == DMA_MAPPING_ERROR)
1004 return DMA_MAPPING_ERROR;
1005
1006 /* Cleanup the padding area. */
1007 padding_start = phys_to_virt(phys);
1008 padding_size = aligned_size;
1009
1010 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
1011 (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) {
1012 padding_start += size;
1013 padding_size -= size;
1014 }
1015
1016 memset(padding_start, 0, padding_size);
1017 }
1018
1019 if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1020 arch_sync_dma_for_device(phys, size, dir);
1021
1022 iova = __iommu_dma_map(dev, phys, size, prot, dma_mask);
1023 if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys))
1024 swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
1025 return iova;
1026}
1027
1028static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
1029 size_t size, enum dma_data_direction dir, unsigned long attrs)
1030{
1031 struct iommu_domain *domain = iommu_get_dma_domain(dev);
1032 phys_addr_t phys;
1033
1034 phys = iommu_iova_to_phys(domain, dma_handle);
1035 if (WARN_ON(!phys))
1036 return;
1037
1038 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && !dev_is_dma_coherent(dev))
1039 arch_sync_dma_for_cpu(phys, size, dir);
1040
1041 __iommu_dma_unmap(dev, dma_handle, size);
1042
1043 if (unlikely(is_swiotlb_buffer(dev, phys)))
1044 swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
1045}
1046
1047/*
1048 * Prepare a successfully-mapped scatterlist to give back to the caller.
1049 *
1050 * At this point the segments are already laid out by iommu_dma_map_sg() to
1051 * avoid individually crossing any boundaries, so we merely need to check a
1052 * segment's start address to avoid concatenating across one.
1053 */
1054static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
1055 dma_addr_t dma_addr)
1056{
1057 struct scatterlist *s, *cur = sg;
1058 unsigned long seg_mask = dma_get_seg_boundary(dev);
1059 unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev);
1060 int i, count = 0;
1061
1062 for_each_sg(sg, s, nents, i) {
1063 /* Restore this segment's original unaligned fields first */
1064 dma_addr_t s_dma_addr = sg_dma_address(s);
1065 unsigned int s_iova_off = sg_dma_address(s);
1066 unsigned int s_length = sg_dma_len(s);
1067 unsigned int s_iova_len = s->length;
1068
1069 sg_dma_address(s) = DMA_MAPPING_ERROR;
1070 sg_dma_len(s) = 0;
1071
1072 if (sg_is_dma_bus_address(s)) {
1073 if (i > 0)
1074 cur = sg_next(cur);
1075
1076 sg_dma_unmark_bus_address(s);
1077 sg_dma_address(cur) = s_dma_addr;
1078 sg_dma_len(cur) = s_length;
1079 sg_dma_mark_bus_address(cur);
1080 count++;
1081 cur_len = 0;
1082 continue;
1083 }
1084
1085 s->offset += s_iova_off;
1086 s->length = s_length;
1087
1088 /*
1089 * Now fill in the real DMA data. If...
1090 * - there is a valid output segment to append to
1091 * - and this segment starts on an IOVA page boundary
1092 * - but doesn't fall at a segment boundary
1093 * - and wouldn't make the resulting output segment too long
1094 */
1095 if (cur_len && !s_iova_off && (dma_addr & seg_mask) &&
1096 (max_len - cur_len >= s_length)) {
1097 /* ...then concatenate it with the previous one */
1098 cur_len += s_length;
1099 } else {
1100 /* Otherwise start the next output segment */
1101 if (i > 0)
1102 cur = sg_next(cur);
1103 cur_len = s_length;
1104 count++;
1105
1106 sg_dma_address(cur) = dma_addr + s_iova_off;
1107 }
1108
1109 sg_dma_len(cur) = cur_len;
1110 dma_addr += s_iova_len;
1111
1112 if (s_length + s_iova_off < s_iova_len)
1113 cur_len = 0;
1114 }
1115 return count;
1116}
1117
1118/*
1119 * If mapping failed, then just restore the original list,
1120 * but making sure the DMA fields are invalidated.
1121 */
1122static void __invalidate_sg(struct scatterlist *sg, int nents)
1123{
1124 struct scatterlist *s;
1125 int i;
1126
1127 for_each_sg(sg, s, nents, i) {
1128 if (sg_is_dma_bus_address(s)) {
1129 sg_dma_unmark_bus_address(s);
1130 } else {
1131 if (sg_dma_address(s) != DMA_MAPPING_ERROR)
1132 s->offset += sg_dma_address(s);
1133 if (sg_dma_len(s))
1134 s->length = sg_dma_len(s);
1135 }
1136 sg_dma_address(s) = DMA_MAPPING_ERROR;
1137 sg_dma_len(s) = 0;
1138 }
1139}
1140
1141static void iommu_dma_unmap_sg_swiotlb(struct device *dev, struct scatterlist *sg,
1142 int nents, enum dma_data_direction dir, unsigned long attrs)
1143{
1144 struct scatterlist *s;
1145 int i;
1146
1147 for_each_sg(sg, s, nents, i)
1148 iommu_dma_unmap_page(dev, sg_dma_address(s),
1149 sg_dma_len(s), dir, attrs);
1150}
1151
1152static int iommu_dma_map_sg_swiotlb(struct device *dev, struct scatterlist *sg,
1153 int nents, enum dma_data_direction dir, unsigned long attrs)
1154{
1155 struct scatterlist *s;
1156 int i;
1157
1158 for_each_sg(sg, s, nents, i) {
1159 sg_dma_address(s) = iommu_dma_map_page(dev, sg_page(s),
1160 s->offset, s->length, dir, attrs);
1161 if (sg_dma_address(s) == DMA_MAPPING_ERROR)
1162 goto out_unmap;
1163 sg_dma_len(s) = s->length;
1164 }
1165
1166 return nents;
1167
1168out_unmap:
1169 iommu_dma_unmap_sg_swiotlb(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
1170 return -EIO;
1171}
1172
1173/*
1174 * The DMA API client is passing in a scatterlist which could describe
1175 * any old buffer layout, but the IOMMU API requires everything to be
1176 * aligned to IOMMU pages. Hence the need for this complicated bit of
1177 * impedance-matching, to be able to hand off a suitably-aligned list,
1178 * but still preserve the original offsets and sizes for the caller.
1179 */
1180static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
1181 int nents, enum dma_data_direction dir, unsigned long attrs)
1182{
1183 struct iommu_domain *domain = iommu_get_dma_domain(dev);
1184 struct iommu_dma_cookie *cookie = domain->iova_cookie;
1185 struct iova_domain *iovad = &cookie->iovad;
1186 struct scatterlist *s, *prev = NULL;
1187 int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs);
1188 struct pci_p2pdma_map_state p2pdma_state = {};
1189 enum pci_p2pdma_map_type map;
1190 dma_addr_t iova;
1191 size_t iova_len = 0;
1192 unsigned long mask = dma_get_seg_boundary(dev);
1193 ssize_t ret;
1194 int i;
1195
1196 if (static_branch_unlikely(&iommu_deferred_attach_enabled)) {
1197 ret = iommu_deferred_attach(dev, domain);
1198 if (ret)
1199 goto out;
1200 }
1201
1202 if (dev_use_swiotlb(dev))
1203 return iommu_dma_map_sg_swiotlb(dev, sg, nents, dir, attrs);
1204
1205 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1206 iommu_dma_sync_sg_for_device(dev, sg, nents, dir);
1207
1208 /*
1209 * Work out how much IOVA space we need, and align the segments to
1210 * IOVA granules for the IOMMU driver to handle. With some clever
1211 * trickery we can modify the list in-place, but reversibly, by
1212 * stashing the unaligned parts in the as-yet-unused DMA fields.
1213 */
1214 for_each_sg(sg, s, nents, i) {
1215 size_t s_iova_off = iova_offset(iovad, s->offset);
1216 size_t s_length = s->length;
1217 size_t pad_len = (mask - iova_len + 1) & mask;
1218
1219 if (is_pci_p2pdma_page(sg_page(s))) {
1220 map = pci_p2pdma_map_segment(&p2pdma_state, dev, s);
1221 switch (map) {
1222 case PCI_P2PDMA_MAP_BUS_ADDR:
1223 /*
1224 * iommu_map_sg() will skip this segment as
1225 * it is marked as a bus address,
1226 * __finalise_sg() will copy the dma address
1227 * into the output segment.
1228 */
1229 continue;
1230 case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
1231 /*
1232 * Mapping through host bridge should be
1233 * mapped with regular IOVAs, thus we
1234 * do nothing here and continue below.
1235 */
1236 break;
1237 default:
1238 ret = -EREMOTEIO;
1239 goto out_restore_sg;
1240 }
1241 }
1242
1243 sg_dma_address(s) = s_iova_off;
1244 sg_dma_len(s) = s_length;
1245 s->offset -= s_iova_off;
1246 s_length = iova_align(iovad, s_length + s_iova_off);
1247 s->length = s_length;
1248
1249 /*
1250 * Due to the alignment of our single IOVA allocation, we can
1251 * depend on these assumptions about the segment boundary mask:
1252 * - If mask size >= IOVA size, then the IOVA range cannot
1253 * possibly fall across a boundary, so we don't care.
1254 * - If mask size < IOVA size, then the IOVA range must start
1255 * exactly on a boundary, therefore we can lay things out
1256 * based purely on segment lengths without needing to know
1257 * the actual addresses beforehand.
1258 * - The mask must be a power of 2, so pad_len == 0 if
1259 * iova_len == 0, thus we cannot dereference prev the first
1260 * time through here (i.e. before it has a meaningful value).
1261 */
1262 if (pad_len && pad_len < s_length - 1) {
1263 prev->length += pad_len;
1264 iova_len += pad_len;
1265 }
1266
1267 iova_len += s_length;
1268 prev = s;
1269 }
1270
1271 if (!iova_len)
1272 return __finalise_sg(dev, sg, nents, 0);
1273
1274 iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);
1275 if (!iova) {
1276 ret = -ENOMEM;
1277 goto out_restore_sg;
1278 }
1279
1280 /*
1281 * We'll leave any physical concatenation to the IOMMU driver's
1282 * implementation - it knows better than we do.
1283 */
1284 ret = iommu_map_sg_atomic(domain, iova, sg, nents, prot);
1285 if (ret < 0 || ret < iova_len)
1286 goto out_free_iova;
1287
1288 return __finalise_sg(dev, sg, nents, iova);
1289
1290out_free_iova:
1291 iommu_dma_free_iova(cookie, iova, iova_len, NULL);
1292out_restore_sg:
1293 __invalidate_sg(sg, nents);
1294out:
1295 if (ret != -ENOMEM && ret != -EREMOTEIO)
1296 return -EINVAL;
1297 return ret;
1298}
1299
1300static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
1301 int nents, enum dma_data_direction dir, unsigned long attrs)
1302{
1303 dma_addr_t end = 0, start;
1304 struct scatterlist *tmp;
1305 int i;
1306
1307 if (dev_use_swiotlb(dev)) {
1308 iommu_dma_unmap_sg_swiotlb(dev, sg, nents, dir, attrs);
1309 return;
1310 }
1311
1312 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1313 iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir);
1314
1315 /*
1316 * The scatterlist segments are mapped into a single
1317 * contiguous IOVA allocation, the start and end points
1318 * just have to be determined.
1319 */
1320 for_each_sg(sg, tmp, nents, i) {
1321 if (sg_is_dma_bus_address(tmp)) {
1322 sg_dma_unmark_bus_address(tmp);
1323 continue;
1324 }
1325
1326 if (sg_dma_len(tmp) == 0)
1327 break;
1328
1329 start = sg_dma_address(tmp);
1330 break;
1331 }
1332
1333 nents -= i;
1334 for_each_sg(tmp, tmp, nents, i) {
1335 if (sg_is_dma_bus_address(tmp)) {
1336 sg_dma_unmark_bus_address(tmp);
1337 continue;
1338 }
1339
1340 if (sg_dma_len(tmp) == 0)
1341 break;
1342
1343 end = sg_dma_address(tmp) + sg_dma_len(tmp);
1344 }
1345
1346 if (end)
1347 __iommu_dma_unmap(dev, start, end - start);
1348}
1349
1350static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
1351 size_t size, enum dma_data_direction dir, unsigned long attrs)
1352{
1353 return __iommu_dma_map(dev, phys, size,
1354 dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO,
1355 dma_get_mask(dev));
1356}
1357
1358static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
1359 size_t size, enum dma_data_direction dir, unsigned long attrs)
1360{
1361 __iommu_dma_unmap(dev, handle, size);
1362}
1363
1364static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)
1365{
1366 size_t alloc_size = PAGE_ALIGN(size);
1367 int count = alloc_size >> PAGE_SHIFT;
1368 struct page *page = NULL, **pages = NULL;
1369
1370 /* Non-coherent atomic allocation? Easy */
1371 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
1372 dma_free_from_pool(dev, cpu_addr, alloc_size))
1373 return;
1374
1375 if (is_vmalloc_addr(cpu_addr)) {
1376 /*
1377 * If it the address is remapped, then it's either non-coherent
1378 * or highmem CMA, or an iommu_dma_alloc_remap() construction.
1379 */
1380 pages = dma_common_find_pages(cpu_addr);
1381 if (!pages)
1382 page = vmalloc_to_page(cpu_addr);
1383 dma_common_free_remap(cpu_addr, alloc_size);
1384 } else {
1385 /* Lowmem means a coherent atomic or CMA allocation */
1386 page = virt_to_page(cpu_addr);
1387 }
1388
1389 if (pages)
1390 __iommu_dma_free_pages(pages, count);
1391 if (page)
1392 dma_free_contiguous(dev, page, alloc_size);
1393}
1394
1395static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr,
1396 dma_addr_t handle, unsigned long attrs)
1397{
1398 __iommu_dma_unmap(dev, handle, size);
1399 __iommu_dma_free(dev, size, cpu_addr);
1400}
1401
1402static void *iommu_dma_alloc_pages(struct device *dev, size_t size,
1403 struct page **pagep, gfp_t gfp, unsigned long attrs)
1404{
1405 bool coherent = dev_is_dma_coherent(dev);
1406 size_t alloc_size = PAGE_ALIGN(size);
1407 int node = dev_to_node(dev);
1408 struct page *page = NULL;
1409 void *cpu_addr;
1410
1411 page = dma_alloc_contiguous(dev, alloc_size, gfp);
1412 if (!page)
1413 page = alloc_pages_node(node, gfp, get_order(alloc_size));
1414 if (!page)
1415 return NULL;
1416
1417 if (!coherent || PageHighMem(page)) {
1418 pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs);
1419
1420 cpu_addr = dma_common_contiguous_remap(page, alloc_size,
1421 prot, __builtin_return_address(0));
1422 if (!cpu_addr)
1423 goto out_free_pages;
1424
1425 if (!coherent)
1426 arch_dma_prep_coherent(page, size);
1427 } else {
1428 cpu_addr = page_address(page);
1429 }
1430
1431 *pagep = page;
1432 memset(cpu_addr, 0, alloc_size);
1433 return cpu_addr;
1434out_free_pages:
1435 dma_free_contiguous(dev, page, alloc_size);
1436 return NULL;
1437}
1438
1439static void *iommu_dma_alloc(struct device *dev, size_t size,
1440 dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
1441{
1442 bool coherent = dev_is_dma_coherent(dev);
1443 int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
1444 struct page *page = NULL;
1445 void *cpu_addr;
1446
1447 gfp |= __GFP_ZERO;
1448
1449 if (gfpflags_allow_blocking(gfp) &&
1450 !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) {
1451 return iommu_dma_alloc_remap(dev, size, handle, gfp,
1452 dma_pgprot(dev, PAGE_KERNEL, attrs), attrs);
1453 }
1454
1455 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
1456 !gfpflags_allow_blocking(gfp) && !coherent)
1457 page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
1458 gfp, NULL);
1459 else
1460 cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
1461 if (!cpu_addr)
1462 return NULL;
1463
1464 *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot,
1465 dev->coherent_dma_mask);
1466 if (*handle == DMA_MAPPING_ERROR) {
1467 __iommu_dma_free(dev, size, cpu_addr);
1468 return NULL;
1469 }
1470
1471 return cpu_addr;
1472}
1473
1474static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
1475 void *cpu_addr, dma_addr_t dma_addr, size_t size,
1476 unsigned long attrs)
1477{
1478 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1479 unsigned long pfn, off = vma->vm_pgoff;
1480 int ret;
1481
1482 vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
1483
1484 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
1485 return ret;
1486
1487 if (off >= nr_pages || vma_pages(vma) > nr_pages - off)
1488 return -ENXIO;
1489
1490 if (is_vmalloc_addr(cpu_addr)) {
1491 struct page **pages = dma_common_find_pages(cpu_addr);
1492
1493 if (pages)
1494 return vm_map_pages(vma, pages, nr_pages);
1495 pfn = vmalloc_to_pfn(cpu_addr);
1496 } else {
1497 pfn = page_to_pfn(virt_to_page(cpu_addr));
1498 }
1499
1500 return remap_pfn_range(vma, vma->vm_start, pfn + off,
1501 vma->vm_end - vma->vm_start,
1502 vma->vm_page_prot);
1503}
1504
1505static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
1506 void *cpu_addr, dma_addr_t dma_addr, size_t size,
1507 unsigned long attrs)
1508{
1509 struct page *page;
1510 int ret;
1511
1512 if (is_vmalloc_addr(cpu_addr)) {
1513 struct page **pages = dma_common_find_pages(cpu_addr);
1514
1515 if (pages) {
1516 return sg_alloc_table_from_pages(sgt, pages,
1517 PAGE_ALIGN(size) >> PAGE_SHIFT,
1518 0, size, GFP_KERNEL);
1519 }
1520
1521 page = vmalloc_to_page(cpu_addr);
1522 } else {
1523 page = virt_to_page(cpu_addr);
1524 }
1525
1526 ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
1527 if (!ret)
1528 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
1529 return ret;
1530}
1531
1532static unsigned long iommu_dma_get_merge_boundary(struct device *dev)
1533{
1534 struct iommu_domain *domain = iommu_get_dma_domain(dev);
1535
1536 return (1UL << __ffs(domain->pgsize_bitmap)) - 1;
1537}
1538
1539static size_t iommu_dma_opt_mapping_size(void)
1540{
1541 return iova_rcache_range();
1542}
1543
1544static const struct dma_map_ops iommu_dma_ops = {
1545 .flags = DMA_F_PCI_P2PDMA_SUPPORTED,
1546 .alloc = iommu_dma_alloc,
1547 .free = iommu_dma_free,
1548 .alloc_pages = dma_common_alloc_pages,
1549 .free_pages = dma_common_free_pages,
1550 .alloc_noncontiguous = iommu_dma_alloc_noncontiguous,
1551 .free_noncontiguous = iommu_dma_free_noncontiguous,
1552 .mmap = iommu_dma_mmap,
1553 .get_sgtable = iommu_dma_get_sgtable,
1554 .map_page = iommu_dma_map_page,
1555 .unmap_page = iommu_dma_unmap_page,
1556 .map_sg = iommu_dma_map_sg,
1557 .unmap_sg = iommu_dma_unmap_sg,
1558 .sync_single_for_cpu = iommu_dma_sync_single_for_cpu,
1559 .sync_single_for_device = iommu_dma_sync_single_for_device,
1560 .sync_sg_for_cpu = iommu_dma_sync_sg_for_cpu,
1561 .sync_sg_for_device = iommu_dma_sync_sg_for_device,
1562 .map_resource = iommu_dma_map_resource,
1563 .unmap_resource = iommu_dma_unmap_resource,
1564 .get_merge_boundary = iommu_dma_get_merge_boundary,
1565 .opt_mapping_size = iommu_dma_opt_mapping_size,
1566};
1567
1568/*
1569 * The IOMMU core code allocates the default DMA domain, which the underlying
1570 * IOMMU driver needs to support via the dma-iommu layer.
1571 */
1572void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit)
1573{
1574 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1575
1576 if (!domain)
1577 goto out_err;
1578
1579 /*
1580 * The IOMMU core code allocates the default DMA domain, which the
1581 * underlying IOMMU driver needs to support via the dma-iommu layer.
1582 */
1583 if (iommu_is_dma_domain(domain)) {
1584 if (iommu_dma_init_domain(domain, dma_base, dma_limit, dev))
1585 goto out_err;
1586 dev->dma_ops = &iommu_dma_ops;
1587 }
1588
1589 return;
1590out_err:
1591 pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
1592 dev_name(dev));
1593}
1594EXPORT_SYMBOL_GPL(iommu_setup_dma_ops);
1595
1596static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
1597 phys_addr_t msi_addr, struct iommu_domain *domain)
1598{
1599 struct iommu_dma_cookie *cookie = domain->iova_cookie;
1600 struct iommu_dma_msi_page *msi_page;
1601 dma_addr_t iova;
1602 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1603 size_t size = cookie_msi_granule(cookie);
1604
1605 msi_addr &= ~(phys_addr_t)(size - 1);
1606 list_for_each_entry(msi_page, &cookie->msi_page_list, list)
1607 if (msi_page->phys == msi_addr)
1608 return msi_page;
1609
1610 msi_page = kzalloc(sizeof(*msi_page), GFP_KERNEL);
1611 if (!msi_page)
1612 return NULL;
1613
1614 iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev);
1615 if (!iova)
1616 goto out_free_page;
1617
1618 if (iommu_map(domain, iova, msi_addr, size, prot))
1619 goto out_free_iova;
1620
1621 INIT_LIST_HEAD(&msi_page->list);
1622 msi_page->phys = msi_addr;
1623 msi_page->iova = iova;
1624 list_add(&msi_page->list, &cookie->msi_page_list);
1625 return msi_page;
1626
1627out_free_iova:
1628 iommu_dma_free_iova(cookie, iova, size, NULL);
1629out_free_page:
1630 kfree(msi_page);
1631 return NULL;
1632}
1633
1634/**
1635 * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain
1636 * @desc: MSI descriptor, will store the MSI page
1637 * @msi_addr: MSI target address to be mapped
1638 *
1639 * Return: 0 on success or negative error code if the mapping failed.
1640 */
1641int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
1642{
1643 struct device *dev = msi_desc_to_dev(desc);
1644 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1645 struct iommu_dma_msi_page *msi_page;
1646 static DEFINE_MUTEX(msi_prepare_lock); /* see below */
1647
1648 if (!domain || !domain->iova_cookie) {
1649 desc->iommu_cookie = NULL;
1650 return 0;
1651 }
1652
1653 /*
1654 * In fact the whole prepare operation should already be serialised by
1655 * irq_domain_mutex further up the callchain, but that's pretty subtle
1656 * on its own, so consider this locking as failsafe documentation...
1657 */
1658 mutex_lock(&msi_prepare_lock);
1659 msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain);
1660 mutex_unlock(&msi_prepare_lock);
1661
1662 msi_desc_set_iommu_cookie(desc, msi_page);
1663
1664 if (!msi_page)
1665 return -ENOMEM;
1666 return 0;
1667}
1668
1669/**
1670 * iommu_dma_compose_msi_msg() - Apply translation to an MSI message
1671 * @desc: MSI descriptor prepared by iommu_dma_prepare_msi()
1672 * @msg: MSI message containing target physical address
1673 */
1674void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
1675{
1676 struct device *dev = msi_desc_to_dev(desc);
1677 const struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1678 const struct iommu_dma_msi_page *msi_page;
1679
1680 msi_page = msi_desc_get_iommu_cookie(desc);
1681
1682 if (!domain || !domain->iova_cookie || WARN_ON(!msi_page))
1683 return;
1684
1685 msg->address_hi = upper_32_bits(msi_page->iova);
1686 msg->address_lo &= cookie_msi_granule(domain->iova_cookie) - 1;
1687 msg->address_lo += lower_32_bits(msi_page->iova);
1688}
1689
1690static int iommu_dma_init(void)
1691{
1692 if (is_kdump_kernel())
1693 static_branch_enable(&iommu_deferred_attach_enabled);
1694
1695 return iova_cache_get();
1696}
1697arch_initcall(iommu_dma_init);
1// SPDX-License-Identifier: GPL-2.0-only
2/*
3 * A fairly generic DMA-API to IOMMU-API glue layer.
4 *
5 * Copyright (C) 2014-2015 ARM Ltd.
6 *
7 * based in part on arch/arm/mm/dma-mapping.c:
8 * Copyright (C) 2000-2004 Russell King
9 */
10
11#include <linux/acpi_iort.h>
12#include <linux/device.h>
13#include <linux/dma-contiguous.h>
14#include <linux/dma-iommu.h>
15#include <linux/dma-noncoherent.h>
16#include <linux/gfp.h>
17#include <linux/huge_mm.h>
18#include <linux/iommu.h>
19#include <linux/iova.h>
20#include <linux/irq.h>
21#include <linux/mm.h>
22#include <linux/pci.h>
23#include <linux/scatterlist.h>
24#include <linux/vmalloc.h>
25
26struct iommu_dma_msi_page {
27 struct list_head list;
28 dma_addr_t iova;
29 phys_addr_t phys;
30};
31
32enum iommu_dma_cookie_type {
33 IOMMU_DMA_IOVA_COOKIE,
34 IOMMU_DMA_MSI_COOKIE,
35};
36
37struct iommu_dma_cookie {
38 enum iommu_dma_cookie_type type;
39 union {
40 /* Full allocator for IOMMU_DMA_IOVA_COOKIE */
41 struct iova_domain iovad;
42 /* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */
43 dma_addr_t msi_iova;
44 };
45 struct list_head msi_page_list;
46 spinlock_t msi_lock;
47
48 /* Domain for flush queue callback; NULL if flush queue not in use */
49 struct iommu_domain *fq_domain;
50};
51
52static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
53{
54 if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
55 return cookie->iovad.granule;
56 return PAGE_SIZE;
57}
58
59static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type)
60{
61 struct iommu_dma_cookie *cookie;
62
63 cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
64 if (cookie) {
65 spin_lock_init(&cookie->msi_lock);
66 INIT_LIST_HEAD(&cookie->msi_page_list);
67 cookie->type = type;
68 }
69 return cookie;
70}
71
72/**
73 * iommu_get_dma_cookie - Acquire DMA-API resources for a domain
74 * @domain: IOMMU domain to prepare for DMA-API usage
75 *
76 * IOMMU drivers should normally call this from their domain_alloc
77 * callback when domain->type == IOMMU_DOMAIN_DMA.
78 */
79int iommu_get_dma_cookie(struct iommu_domain *domain)
80{
81 if (domain->iova_cookie)
82 return -EEXIST;
83
84 domain->iova_cookie = cookie_alloc(IOMMU_DMA_IOVA_COOKIE);
85 if (!domain->iova_cookie)
86 return -ENOMEM;
87
88 return 0;
89}
90EXPORT_SYMBOL(iommu_get_dma_cookie);
91
92/**
93 * iommu_get_msi_cookie - Acquire just MSI remapping resources
94 * @domain: IOMMU domain to prepare
95 * @base: Start address of IOVA region for MSI mappings
96 *
97 * Users who manage their own IOVA allocation and do not want DMA API support,
98 * but would still like to take advantage of automatic MSI remapping, can use
99 * this to initialise their own domain appropriately. Users should reserve a
100 * contiguous IOVA region, starting at @base, large enough to accommodate the
101 * number of PAGE_SIZE mappings necessary to cover every MSI doorbell address
102 * used by the devices attached to @domain.
103 */
104int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)
105{
106 struct iommu_dma_cookie *cookie;
107
108 if (domain->type != IOMMU_DOMAIN_UNMANAGED)
109 return -EINVAL;
110
111 if (domain->iova_cookie)
112 return -EEXIST;
113
114 cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE);
115 if (!cookie)
116 return -ENOMEM;
117
118 cookie->msi_iova = base;
119 domain->iova_cookie = cookie;
120 return 0;
121}
122EXPORT_SYMBOL(iommu_get_msi_cookie);
123
124/**
125 * iommu_put_dma_cookie - Release a domain's DMA mapping resources
126 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() or
127 * iommu_get_msi_cookie()
128 *
129 * IOMMU drivers should normally call this from their domain_free callback.
130 */
131void iommu_put_dma_cookie(struct iommu_domain *domain)
132{
133 struct iommu_dma_cookie *cookie = domain->iova_cookie;
134 struct iommu_dma_msi_page *msi, *tmp;
135
136 if (!cookie)
137 return;
138
139 if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule)
140 put_iova_domain(&cookie->iovad);
141
142 list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
143 list_del(&msi->list);
144 kfree(msi);
145 }
146 kfree(cookie);
147 domain->iova_cookie = NULL;
148}
149EXPORT_SYMBOL(iommu_put_dma_cookie);
150
151/**
152 * iommu_dma_get_resv_regions - Reserved region driver helper
153 * @dev: Device from iommu_get_resv_regions()
154 * @list: Reserved region list from iommu_get_resv_regions()
155 *
156 * IOMMU drivers can use this to implement their .get_resv_regions callback
157 * for general non-IOMMU-specific reservations. Currently, this covers GICv3
158 * ITS region reservation on ACPI based ARM platforms that may require HW MSI
159 * reservation.
160 */
161void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list)
162{
163
164 if (!is_of_node(dev_iommu_fwspec_get(dev)->iommu_fwnode))
165 iort_iommu_msi_get_resv_regions(dev, list);
166
167}
168EXPORT_SYMBOL(iommu_dma_get_resv_regions);
169
170static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie,
171 phys_addr_t start, phys_addr_t end)
172{
173 struct iova_domain *iovad = &cookie->iovad;
174 struct iommu_dma_msi_page *msi_page;
175 int i, num_pages;
176
177 start -= iova_offset(iovad, start);
178 num_pages = iova_align(iovad, end - start) >> iova_shift(iovad);
179
180 msi_page = kcalloc(num_pages, sizeof(*msi_page), GFP_KERNEL);
181 if (!msi_page)
182 return -ENOMEM;
183
184 for (i = 0; i < num_pages; i++) {
185 msi_page[i].phys = start;
186 msi_page[i].iova = start;
187 INIT_LIST_HEAD(&msi_page[i].list);
188 list_add(&msi_page[i].list, &cookie->msi_page_list);
189 start += iovad->granule;
190 }
191
192 return 0;
193}
194
195static int iova_reserve_pci_windows(struct pci_dev *dev,
196 struct iova_domain *iovad)
197{
198 struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus);
199 struct resource_entry *window;
200 unsigned long lo, hi;
201 phys_addr_t start = 0, end;
202
203 resource_list_for_each_entry(window, &bridge->windows) {
204 if (resource_type(window->res) != IORESOURCE_MEM)
205 continue;
206
207 lo = iova_pfn(iovad, window->res->start - window->offset);
208 hi = iova_pfn(iovad, window->res->end - window->offset);
209 reserve_iova(iovad, lo, hi);
210 }
211
212 /* Get reserved DMA windows from host bridge */
213 resource_list_for_each_entry(window, &bridge->dma_ranges) {
214 end = window->res->start - window->offset;
215resv_iova:
216 if (end > start) {
217 lo = iova_pfn(iovad, start);
218 hi = iova_pfn(iovad, end);
219 reserve_iova(iovad, lo, hi);
220 } else {
221 /* dma_ranges list should be sorted */
222 dev_err(&dev->dev, "Failed to reserve IOVA\n");
223 return -EINVAL;
224 }
225
226 start = window->res->end - window->offset + 1;
227 /* If window is last entry */
228 if (window->node.next == &bridge->dma_ranges &&
229 end != ~(phys_addr_t)0) {
230 end = ~(phys_addr_t)0;
231 goto resv_iova;
232 }
233 }
234
235 return 0;
236}
237
238static int iova_reserve_iommu_regions(struct device *dev,
239 struct iommu_domain *domain)
240{
241 struct iommu_dma_cookie *cookie = domain->iova_cookie;
242 struct iova_domain *iovad = &cookie->iovad;
243 struct iommu_resv_region *region;
244 LIST_HEAD(resv_regions);
245 int ret = 0;
246
247 if (dev_is_pci(dev)) {
248 ret = iova_reserve_pci_windows(to_pci_dev(dev), iovad);
249 if (ret)
250 return ret;
251 }
252
253 iommu_get_resv_regions(dev, &resv_regions);
254 list_for_each_entry(region, &resv_regions, list) {
255 unsigned long lo, hi;
256
257 /* We ARE the software that manages these! */
258 if (region->type == IOMMU_RESV_SW_MSI)
259 continue;
260
261 lo = iova_pfn(iovad, region->start);
262 hi = iova_pfn(iovad, region->start + region->length - 1);
263 reserve_iova(iovad, lo, hi);
264
265 if (region->type == IOMMU_RESV_MSI)
266 ret = cookie_init_hw_msi_region(cookie, region->start,
267 region->start + region->length);
268 if (ret)
269 break;
270 }
271 iommu_put_resv_regions(dev, &resv_regions);
272
273 return ret;
274}
275
276static void iommu_dma_flush_iotlb_all(struct iova_domain *iovad)
277{
278 struct iommu_dma_cookie *cookie;
279 struct iommu_domain *domain;
280
281 cookie = container_of(iovad, struct iommu_dma_cookie, iovad);
282 domain = cookie->fq_domain;
283 /*
284 * The IOMMU driver supporting DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE
285 * implies that ops->flush_iotlb_all must be non-NULL.
286 */
287 domain->ops->flush_iotlb_all(domain);
288}
289
290/**
291 * iommu_dma_init_domain - Initialise a DMA mapping domain
292 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
293 * @base: IOVA at which the mappable address space starts
294 * @size: Size of IOVA space
295 * @dev: Device the domain is being initialised for
296 *
297 * @base and @size should be exact multiples of IOMMU page granularity to
298 * avoid rounding surprises. If necessary, we reserve the page at address 0
299 * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but
300 * any change which could make prior IOVAs invalid will fail.
301 */
302static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
303 u64 size, struct device *dev)
304{
305 struct iommu_dma_cookie *cookie = domain->iova_cookie;
306 unsigned long order, base_pfn;
307 struct iova_domain *iovad;
308 int attr;
309
310 if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
311 return -EINVAL;
312
313 iovad = &cookie->iovad;
314
315 /* Use the smallest supported page size for IOVA granularity */
316 order = __ffs(domain->pgsize_bitmap);
317 base_pfn = max_t(unsigned long, 1, base >> order);
318
319 /* Check the domain allows at least some access to the device... */
320 if (domain->geometry.force_aperture) {
321 if (base > domain->geometry.aperture_end ||
322 base + size <= domain->geometry.aperture_start) {
323 pr_warn("specified DMA range outside IOMMU capability\n");
324 return -EFAULT;
325 }
326 /* ...then finally give it a kicking to make sure it fits */
327 base_pfn = max_t(unsigned long, base_pfn,
328 domain->geometry.aperture_start >> order);
329 }
330
331 /* start_pfn is always nonzero for an already-initialised domain */
332 if (iovad->start_pfn) {
333 if (1UL << order != iovad->granule ||
334 base_pfn != iovad->start_pfn) {
335 pr_warn("Incompatible range for DMA domain\n");
336 return -EFAULT;
337 }
338
339 return 0;
340 }
341
342 init_iova_domain(iovad, 1UL << order, base_pfn);
343
344 if (!cookie->fq_domain && !iommu_domain_get_attr(domain,
345 DOMAIN_ATTR_DMA_USE_FLUSH_QUEUE, &attr) && attr) {
346 cookie->fq_domain = domain;
347 init_iova_flush_queue(iovad, iommu_dma_flush_iotlb_all, NULL);
348 }
349
350 if (!dev)
351 return 0;
352
353 return iova_reserve_iommu_regions(dev, domain);
354}
355
356/**
357 * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API
358 * page flags.
359 * @dir: Direction of DMA transfer
360 * @coherent: Is the DMA master cache-coherent?
361 * @attrs: DMA attributes for the mapping
362 *
363 * Return: corresponding IOMMU API page protection flags
364 */
365static int dma_info_to_prot(enum dma_data_direction dir, bool coherent,
366 unsigned long attrs)
367{
368 int prot = coherent ? IOMMU_CACHE : 0;
369
370 if (attrs & DMA_ATTR_PRIVILEGED)
371 prot |= IOMMU_PRIV;
372
373 switch (dir) {
374 case DMA_BIDIRECTIONAL:
375 return prot | IOMMU_READ | IOMMU_WRITE;
376 case DMA_TO_DEVICE:
377 return prot | IOMMU_READ;
378 case DMA_FROM_DEVICE:
379 return prot | IOMMU_WRITE;
380 default:
381 return 0;
382 }
383}
384
385static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
386 size_t size, dma_addr_t dma_limit, struct device *dev)
387{
388 struct iommu_dma_cookie *cookie = domain->iova_cookie;
389 struct iova_domain *iovad = &cookie->iovad;
390 unsigned long shift, iova_len, iova = 0;
391
392 if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
393 cookie->msi_iova += size;
394 return cookie->msi_iova - size;
395 }
396
397 shift = iova_shift(iovad);
398 iova_len = size >> shift;
399 /*
400 * Freeing non-power-of-two-sized allocations back into the IOVA caches
401 * will come back to bite us badly, so we have to waste a bit of space
402 * rounding up anything cacheable to make sure that can't happen. The
403 * order of the unadjusted size will still match upon freeing.
404 */
405 if (iova_len < (1 << (IOVA_RANGE_CACHE_MAX_SIZE - 1)))
406 iova_len = roundup_pow_of_two(iova_len);
407
408 if (dev->bus_dma_mask)
409 dma_limit &= dev->bus_dma_mask;
410
411 if (domain->geometry.force_aperture)
412 dma_limit = min(dma_limit, domain->geometry.aperture_end);
413
414 /* Try to get PCI devices a SAC address */
415 if (dma_limit > DMA_BIT_MASK(32) && dev_is_pci(dev))
416 iova = alloc_iova_fast(iovad, iova_len,
417 DMA_BIT_MASK(32) >> shift, false);
418
419 if (!iova)
420 iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift,
421 true);
422
423 return (dma_addr_t)iova << shift;
424}
425
426static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
427 dma_addr_t iova, size_t size)
428{
429 struct iova_domain *iovad = &cookie->iovad;
430
431 /* The MSI case is only ever cleaning up its most recent allocation */
432 if (cookie->type == IOMMU_DMA_MSI_COOKIE)
433 cookie->msi_iova -= size;
434 else if (cookie->fq_domain) /* non-strict mode */
435 queue_iova(iovad, iova_pfn(iovad, iova),
436 size >> iova_shift(iovad), 0);
437 else
438 free_iova_fast(iovad, iova_pfn(iovad, iova),
439 size >> iova_shift(iovad));
440}
441
442static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
443 size_t size)
444{
445 struct iommu_domain *domain = iommu_get_dma_domain(dev);
446 struct iommu_dma_cookie *cookie = domain->iova_cookie;
447 struct iova_domain *iovad = &cookie->iovad;
448 size_t iova_off = iova_offset(iovad, dma_addr);
449 struct iommu_iotlb_gather iotlb_gather;
450 size_t unmapped;
451
452 dma_addr -= iova_off;
453 size = iova_align(iovad, size + iova_off);
454 iommu_iotlb_gather_init(&iotlb_gather);
455
456 unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather);
457 WARN_ON(unmapped != size);
458
459 if (!cookie->fq_domain)
460 iommu_tlb_sync(domain, &iotlb_gather);
461 iommu_dma_free_iova(cookie, dma_addr, size);
462}
463
464static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
465 size_t size, int prot)
466{
467 struct iommu_domain *domain = iommu_get_dma_domain(dev);
468 struct iommu_dma_cookie *cookie = domain->iova_cookie;
469 struct iova_domain *iovad = &cookie->iovad;
470 size_t iova_off = iova_offset(iovad, phys);
471 dma_addr_t iova;
472
473 size = iova_align(iovad, size + iova_off);
474
475 iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev);
476 if (!iova)
477 return DMA_MAPPING_ERROR;
478
479 if (iommu_map(domain, iova, phys - iova_off, size, prot)) {
480 iommu_dma_free_iova(cookie, iova, size);
481 return DMA_MAPPING_ERROR;
482 }
483 return iova + iova_off;
484}
485
486static void __iommu_dma_free_pages(struct page **pages, int count)
487{
488 while (count--)
489 __free_page(pages[count]);
490 kvfree(pages);
491}
492
493static struct page **__iommu_dma_alloc_pages(struct device *dev,
494 unsigned int count, unsigned long order_mask, gfp_t gfp)
495{
496 struct page **pages;
497 unsigned int i = 0, nid = dev_to_node(dev);
498
499 order_mask &= (2U << MAX_ORDER) - 1;
500 if (!order_mask)
501 return NULL;
502
503 pages = kvzalloc(count * sizeof(*pages), GFP_KERNEL);
504 if (!pages)
505 return NULL;
506
507 /* IOMMU can map any pages, so himem can also be used here */
508 gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
509
510 while (count) {
511 struct page *page = NULL;
512 unsigned int order_size;
513
514 /*
515 * Higher-order allocations are a convenience rather
516 * than a necessity, hence using __GFP_NORETRY until
517 * falling back to minimum-order allocations.
518 */
519 for (order_mask &= (2U << __fls(count)) - 1;
520 order_mask; order_mask &= ~order_size) {
521 unsigned int order = __fls(order_mask);
522 gfp_t alloc_flags = gfp;
523
524 order_size = 1U << order;
525 if (order_mask > order_size)
526 alloc_flags |= __GFP_NORETRY;
527 page = alloc_pages_node(nid, alloc_flags, order);
528 if (!page)
529 continue;
530 if (!order)
531 break;
532 if (!PageCompound(page)) {
533 split_page(page, order);
534 break;
535 } else if (!split_huge_page(page)) {
536 break;
537 }
538 __free_pages(page, order);
539 }
540 if (!page) {
541 __iommu_dma_free_pages(pages, i);
542 return NULL;
543 }
544 count -= order_size;
545 while (order_size--)
546 pages[i++] = page++;
547 }
548 return pages;
549}
550
551/**
552 * iommu_dma_alloc_remap - Allocate and map a buffer contiguous in IOVA space
553 * @dev: Device to allocate memory for. Must be a real device
554 * attached to an iommu_dma_domain
555 * @size: Size of buffer in bytes
556 * @dma_handle: Out argument for allocated DMA handle
557 * @gfp: Allocation flags
558 * @attrs: DMA attributes for this allocation
559 *
560 * If @size is less than PAGE_SIZE, then a full CPU page will be allocated,
561 * but an IOMMU which supports smaller pages might not map the whole thing.
562 *
563 * Return: Mapped virtual address, or NULL on failure.
564 */
565static void *iommu_dma_alloc_remap(struct device *dev, size_t size,
566 dma_addr_t *dma_handle, gfp_t gfp, unsigned long attrs)
567{
568 struct iommu_domain *domain = iommu_get_dma_domain(dev);
569 struct iommu_dma_cookie *cookie = domain->iova_cookie;
570 struct iova_domain *iovad = &cookie->iovad;
571 bool coherent = dev_is_dma_coherent(dev);
572 int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
573 pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs);
574 unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap;
575 struct page **pages;
576 struct sg_table sgt;
577 dma_addr_t iova;
578 void *vaddr;
579
580 *dma_handle = DMA_MAPPING_ERROR;
581
582 min_size = alloc_sizes & -alloc_sizes;
583 if (min_size < PAGE_SIZE) {
584 min_size = PAGE_SIZE;
585 alloc_sizes |= PAGE_SIZE;
586 } else {
587 size = ALIGN(size, min_size);
588 }
589 if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES)
590 alloc_sizes = min_size;
591
592 count = PAGE_ALIGN(size) >> PAGE_SHIFT;
593 pages = __iommu_dma_alloc_pages(dev, count, alloc_sizes >> PAGE_SHIFT,
594 gfp);
595 if (!pages)
596 return NULL;
597
598 size = iova_align(iovad, size);
599 iova = iommu_dma_alloc_iova(domain, size, dev->coherent_dma_mask, dev);
600 if (!iova)
601 goto out_free_pages;
602
603 if (sg_alloc_table_from_pages(&sgt, pages, count, 0, size, GFP_KERNEL))
604 goto out_free_iova;
605
606 if (!(ioprot & IOMMU_CACHE)) {
607 struct scatterlist *sg;
608 int i;
609
610 for_each_sg(sgt.sgl, sg, sgt.orig_nents, i)
611 arch_dma_prep_coherent(sg_page(sg), sg->length);
612 }
613
614 if (iommu_map_sg(domain, iova, sgt.sgl, sgt.orig_nents, ioprot)
615 < size)
616 goto out_free_sg;
617
618 vaddr = dma_common_pages_remap(pages, size, prot,
619 __builtin_return_address(0));
620 if (!vaddr)
621 goto out_unmap;
622
623 *dma_handle = iova;
624 sg_free_table(&sgt);
625 return vaddr;
626
627out_unmap:
628 __iommu_dma_unmap(dev, iova, size);
629out_free_sg:
630 sg_free_table(&sgt);
631out_free_iova:
632 iommu_dma_free_iova(cookie, iova, size);
633out_free_pages:
634 __iommu_dma_free_pages(pages, count);
635 return NULL;
636}
637
638/**
639 * __iommu_dma_mmap - Map a buffer into provided user VMA
640 * @pages: Array representing buffer from __iommu_dma_alloc()
641 * @size: Size of buffer in bytes
642 * @vma: VMA describing requested userspace mapping
643 *
644 * Maps the pages of the buffer in @pages into @vma. The caller is responsible
645 * for verifying the correct size and protection of @vma beforehand.
646 */
647static int __iommu_dma_mmap(struct page **pages, size_t size,
648 struct vm_area_struct *vma)
649{
650 return vm_map_pages(vma, pages, PAGE_ALIGN(size) >> PAGE_SHIFT);
651}
652
653static void iommu_dma_sync_single_for_cpu(struct device *dev,
654 dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
655{
656 phys_addr_t phys;
657
658 if (dev_is_dma_coherent(dev))
659 return;
660
661 phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
662 arch_sync_dma_for_cpu(dev, phys, size, dir);
663}
664
665static void iommu_dma_sync_single_for_device(struct device *dev,
666 dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
667{
668 phys_addr_t phys;
669
670 if (dev_is_dma_coherent(dev))
671 return;
672
673 phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
674 arch_sync_dma_for_device(dev, phys, size, dir);
675}
676
677static void iommu_dma_sync_sg_for_cpu(struct device *dev,
678 struct scatterlist *sgl, int nelems,
679 enum dma_data_direction dir)
680{
681 struct scatterlist *sg;
682 int i;
683
684 if (dev_is_dma_coherent(dev))
685 return;
686
687 for_each_sg(sgl, sg, nelems, i)
688 arch_sync_dma_for_cpu(dev, sg_phys(sg), sg->length, dir);
689}
690
691static void iommu_dma_sync_sg_for_device(struct device *dev,
692 struct scatterlist *sgl, int nelems,
693 enum dma_data_direction dir)
694{
695 struct scatterlist *sg;
696 int i;
697
698 if (dev_is_dma_coherent(dev))
699 return;
700
701 for_each_sg(sgl, sg, nelems, i)
702 arch_sync_dma_for_device(dev, sg_phys(sg), sg->length, dir);
703}
704
705static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
706 unsigned long offset, size_t size, enum dma_data_direction dir,
707 unsigned long attrs)
708{
709 phys_addr_t phys = page_to_phys(page) + offset;
710 bool coherent = dev_is_dma_coherent(dev);
711 int prot = dma_info_to_prot(dir, coherent, attrs);
712 dma_addr_t dma_handle;
713
714 dma_handle =__iommu_dma_map(dev, phys, size, prot);
715 if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
716 dma_handle != DMA_MAPPING_ERROR)
717 arch_sync_dma_for_device(dev, phys, size, dir);
718 return dma_handle;
719}
720
721static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
722 size_t size, enum dma_data_direction dir, unsigned long attrs)
723{
724 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
725 iommu_dma_sync_single_for_cpu(dev, dma_handle, size, dir);
726 __iommu_dma_unmap(dev, dma_handle, size);
727}
728
729/*
730 * Prepare a successfully-mapped scatterlist to give back to the caller.
731 *
732 * At this point the segments are already laid out by iommu_dma_map_sg() to
733 * avoid individually crossing any boundaries, so we merely need to check a
734 * segment's start address to avoid concatenating across one.
735 */
736static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
737 dma_addr_t dma_addr)
738{
739 struct scatterlist *s, *cur = sg;
740 unsigned long seg_mask = dma_get_seg_boundary(dev);
741 unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev);
742 int i, count = 0;
743
744 for_each_sg(sg, s, nents, i) {
745 /* Restore this segment's original unaligned fields first */
746 unsigned int s_iova_off = sg_dma_address(s);
747 unsigned int s_length = sg_dma_len(s);
748 unsigned int s_iova_len = s->length;
749
750 s->offset += s_iova_off;
751 s->length = s_length;
752 sg_dma_address(s) = DMA_MAPPING_ERROR;
753 sg_dma_len(s) = 0;
754
755 /*
756 * Now fill in the real DMA data. If...
757 * - there is a valid output segment to append to
758 * - and this segment starts on an IOVA page boundary
759 * - but doesn't fall at a segment boundary
760 * - and wouldn't make the resulting output segment too long
761 */
762 if (cur_len && !s_iova_off && (dma_addr & seg_mask) &&
763 (max_len - cur_len >= s_length)) {
764 /* ...then concatenate it with the previous one */
765 cur_len += s_length;
766 } else {
767 /* Otherwise start the next output segment */
768 if (i > 0)
769 cur = sg_next(cur);
770 cur_len = s_length;
771 count++;
772
773 sg_dma_address(cur) = dma_addr + s_iova_off;
774 }
775
776 sg_dma_len(cur) = cur_len;
777 dma_addr += s_iova_len;
778
779 if (s_length + s_iova_off < s_iova_len)
780 cur_len = 0;
781 }
782 return count;
783}
784
785/*
786 * If mapping failed, then just restore the original list,
787 * but making sure the DMA fields are invalidated.
788 */
789static void __invalidate_sg(struct scatterlist *sg, int nents)
790{
791 struct scatterlist *s;
792 int i;
793
794 for_each_sg(sg, s, nents, i) {
795 if (sg_dma_address(s) != DMA_MAPPING_ERROR)
796 s->offset += sg_dma_address(s);
797 if (sg_dma_len(s))
798 s->length = sg_dma_len(s);
799 sg_dma_address(s) = DMA_MAPPING_ERROR;
800 sg_dma_len(s) = 0;
801 }
802}
803
804/*
805 * The DMA API client is passing in a scatterlist which could describe
806 * any old buffer layout, but the IOMMU API requires everything to be
807 * aligned to IOMMU pages. Hence the need for this complicated bit of
808 * impedance-matching, to be able to hand off a suitably-aligned list,
809 * but still preserve the original offsets and sizes for the caller.
810 */
811static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
812 int nents, enum dma_data_direction dir, unsigned long attrs)
813{
814 struct iommu_domain *domain = iommu_get_dma_domain(dev);
815 struct iommu_dma_cookie *cookie = domain->iova_cookie;
816 struct iova_domain *iovad = &cookie->iovad;
817 struct scatterlist *s, *prev = NULL;
818 int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs);
819 dma_addr_t iova;
820 size_t iova_len = 0;
821 unsigned long mask = dma_get_seg_boundary(dev);
822 int i;
823
824 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
825 iommu_dma_sync_sg_for_device(dev, sg, nents, dir);
826
827 /*
828 * Work out how much IOVA space we need, and align the segments to
829 * IOVA granules for the IOMMU driver to handle. With some clever
830 * trickery we can modify the list in-place, but reversibly, by
831 * stashing the unaligned parts in the as-yet-unused DMA fields.
832 */
833 for_each_sg(sg, s, nents, i) {
834 size_t s_iova_off = iova_offset(iovad, s->offset);
835 size_t s_length = s->length;
836 size_t pad_len = (mask - iova_len + 1) & mask;
837
838 sg_dma_address(s) = s_iova_off;
839 sg_dma_len(s) = s_length;
840 s->offset -= s_iova_off;
841 s_length = iova_align(iovad, s_length + s_iova_off);
842 s->length = s_length;
843
844 /*
845 * Due to the alignment of our single IOVA allocation, we can
846 * depend on these assumptions about the segment boundary mask:
847 * - If mask size >= IOVA size, then the IOVA range cannot
848 * possibly fall across a boundary, so we don't care.
849 * - If mask size < IOVA size, then the IOVA range must start
850 * exactly on a boundary, therefore we can lay things out
851 * based purely on segment lengths without needing to know
852 * the actual addresses beforehand.
853 * - The mask must be a power of 2, so pad_len == 0 if
854 * iova_len == 0, thus we cannot dereference prev the first
855 * time through here (i.e. before it has a meaningful value).
856 */
857 if (pad_len && pad_len < s_length - 1) {
858 prev->length += pad_len;
859 iova_len += pad_len;
860 }
861
862 iova_len += s_length;
863 prev = s;
864 }
865
866 iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);
867 if (!iova)
868 goto out_restore_sg;
869
870 /*
871 * We'll leave any physical concatenation to the IOMMU driver's
872 * implementation - it knows better than we do.
873 */
874 if (iommu_map_sg(domain, iova, sg, nents, prot) < iova_len)
875 goto out_free_iova;
876
877 return __finalise_sg(dev, sg, nents, iova);
878
879out_free_iova:
880 iommu_dma_free_iova(cookie, iova, iova_len);
881out_restore_sg:
882 __invalidate_sg(sg, nents);
883 return 0;
884}
885
886static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
887 int nents, enum dma_data_direction dir, unsigned long attrs)
888{
889 dma_addr_t start, end;
890 struct scatterlist *tmp;
891 int i;
892
893 if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
894 iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir);
895
896 /*
897 * The scatterlist segments are mapped into a single
898 * contiguous IOVA allocation, so this is incredibly easy.
899 */
900 start = sg_dma_address(sg);
901 for_each_sg(sg_next(sg), tmp, nents - 1, i) {
902 if (sg_dma_len(tmp) == 0)
903 break;
904 sg = tmp;
905 }
906 end = sg_dma_address(sg) + sg_dma_len(sg);
907 __iommu_dma_unmap(dev, start, end - start);
908}
909
910static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
911 size_t size, enum dma_data_direction dir, unsigned long attrs)
912{
913 return __iommu_dma_map(dev, phys, size,
914 dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO);
915}
916
917static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
918 size_t size, enum dma_data_direction dir, unsigned long attrs)
919{
920 __iommu_dma_unmap(dev, handle, size);
921}
922
923static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)
924{
925 size_t alloc_size = PAGE_ALIGN(size);
926 int count = alloc_size >> PAGE_SHIFT;
927 struct page *page = NULL, **pages = NULL;
928
929 /* Non-coherent atomic allocation? Easy */
930 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
931 dma_free_from_pool(cpu_addr, alloc_size))
932 return;
933
934 if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {
935 /*
936 * If it the address is remapped, then it's either non-coherent
937 * or highmem CMA, or an iommu_dma_alloc_remap() construction.
938 */
939 pages = dma_common_find_pages(cpu_addr);
940 if (!pages)
941 page = vmalloc_to_page(cpu_addr);
942 dma_common_free_remap(cpu_addr, alloc_size);
943 } else {
944 /* Lowmem means a coherent atomic or CMA allocation */
945 page = virt_to_page(cpu_addr);
946 }
947
948 if (pages)
949 __iommu_dma_free_pages(pages, count);
950 if (page)
951 dma_free_contiguous(dev, page, alloc_size);
952}
953
954static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr,
955 dma_addr_t handle, unsigned long attrs)
956{
957 __iommu_dma_unmap(dev, handle, size);
958 __iommu_dma_free(dev, size, cpu_addr);
959}
960
961static void *iommu_dma_alloc_pages(struct device *dev, size_t size,
962 struct page **pagep, gfp_t gfp, unsigned long attrs)
963{
964 bool coherent = dev_is_dma_coherent(dev);
965 size_t alloc_size = PAGE_ALIGN(size);
966 int node = dev_to_node(dev);
967 struct page *page = NULL;
968 void *cpu_addr;
969
970 page = dma_alloc_contiguous(dev, alloc_size, gfp);
971 if (!page)
972 page = alloc_pages_node(node, gfp, get_order(alloc_size));
973 if (!page)
974 return NULL;
975
976 if (IS_ENABLED(CONFIG_DMA_REMAP) && (!coherent || PageHighMem(page))) {
977 pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs);
978
979 cpu_addr = dma_common_contiguous_remap(page, alloc_size,
980 prot, __builtin_return_address(0));
981 if (!cpu_addr)
982 goto out_free_pages;
983
984 if (!coherent)
985 arch_dma_prep_coherent(page, size);
986 } else {
987 cpu_addr = page_address(page);
988 }
989
990 *pagep = page;
991 memset(cpu_addr, 0, alloc_size);
992 return cpu_addr;
993out_free_pages:
994 dma_free_contiguous(dev, page, alloc_size);
995 return NULL;
996}
997
998static void *iommu_dma_alloc(struct device *dev, size_t size,
999 dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
1000{
1001 bool coherent = dev_is_dma_coherent(dev);
1002 int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
1003 struct page *page = NULL;
1004 void *cpu_addr;
1005
1006 gfp |= __GFP_ZERO;
1007
1008 if (IS_ENABLED(CONFIG_DMA_REMAP) && gfpflags_allow_blocking(gfp) &&
1009 !(attrs & DMA_ATTR_FORCE_CONTIGUOUS))
1010 return iommu_dma_alloc_remap(dev, size, handle, gfp, attrs);
1011
1012 if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
1013 !gfpflags_allow_blocking(gfp) && !coherent)
1014 cpu_addr = dma_alloc_from_pool(PAGE_ALIGN(size), &page, gfp);
1015 else
1016 cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
1017 if (!cpu_addr)
1018 return NULL;
1019
1020 *handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot);
1021 if (*handle == DMA_MAPPING_ERROR) {
1022 __iommu_dma_free(dev, size, cpu_addr);
1023 return NULL;
1024 }
1025
1026 return cpu_addr;
1027}
1028
1029static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
1030 void *cpu_addr, dma_addr_t dma_addr, size_t size,
1031 unsigned long attrs)
1032{
1033 unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1034 unsigned long pfn, off = vma->vm_pgoff;
1035 int ret;
1036
1037 vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
1038
1039 if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
1040 return ret;
1041
1042 if (off >= nr_pages || vma_pages(vma) > nr_pages - off)
1043 return -ENXIO;
1044
1045 if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {
1046 struct page **pages = dma_common_find_pages(cpu_addr);
1047
1048 if (pages)
1049 return __iommu_dma_mmap(pages, size, vma);
1050 pfn = vmalloc_to_pfn(cpu_addr);
1051 } else {
1052 pfn = page_to_pfn(virt_to_page(cpu_addr));
1053 }
1054
1055 return remap_pfn_range(vma, vma->vm_start, pfn + off,
1056 vma->vm_end - vma->vm_start,
1057 vma->vm_page_prot);
1058}
1059
1060static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
1061 void *cpu_addr, dma_addr_t dma_addr, size_t size,
1062 unsigned long attrs)
1063{
1064 struct page *page;
1065 int ret;
1066
1067 if (IS_ENABLED(CONFIG_DMA_REMAP) && is_vmalloc_addr(cpu_addr)) {
1068 struct page **pages = dma_common_find_pages(cpu_addr);
1069
1070 if (pages) {
1071 return sg_alloc_table_from_pages(sgt, pages,
1072 PAGE_ALIGN(size) >> PAGE_SHIFT,
1073 0, size, GFP_KERNEL);
1074 }
1075
1076 page = vmalloc_to_page(cpu_addr);
1077 } else {
1078 page = virt_to_page(cpu_addr);
1079 }
1080
1081 ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
1082 if (!ret)
1083 sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
1084 return ret;
1085}
1086
1087static unsigned long iommu_dma_get_merge_boundary(struct device *dev)
1088{
1089 struct iommu_domain *domain = iommu_get_dma_domain(dev);
1090
1091 return (1UL << __ffs(domain->pgsize_bitmap)) - 1;
1092}
1093
1094static const struct dma_map_ops iommu_dma_ops = {
1095 .alloc = iommu_dma_alloc,
1096 .free = iommu_dma_free,
1097 .mmap = iommu_dma_mmap,
1098 .get_sgtable = iommu_dma_get_sgtable,
1099 .map_page = iommu_dma_map_page,
1100 .unmap_page = iommu_dma_unmap_page,
1101 .map_sg = iommu_dma_map_sg,
1102 .unmap_sg = iommu_dma_unmap_sg,
1103 .sync_single_for_cpu = iommu_dma_sync_single_for_cpu,
1104 .sync_single_for_device = iommu_dma_sync_single_for_device,
1105 .sync_sg_for_cpu = iommu_dma_sync_sg_for_cpu,
1106 .sync_sg_for_device = iommu_dma_sync_sg_for_device,
1107 .map_resource = iommu_dma_map_resource,
1108 .unmap_resource = iommu_dma_unmap_resource,
1109 .get_merge_boundary = iommu_dma_get_merge_boundary,
1110};
1111
1112/*
1113 * The IOMMU core code allocates the default DMA domain, which the underlying
1114 * IOMMU driver needs to support via the dma-iommu layer.
1115 */
1116void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 size)
1117{
1118 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1119
1120 if (!domain)
1121 goto out_err;
1122
1123 /*
1124 * The IOMMU core code allocates the default DMA domain, which the
1125 * underlying IOMMU driver needs to support via the dma-iommu layer.
1126 */
1127 if (domain->type == IOMMU_DOMAIN_DMA) {
1128 if (iommu_dma_init_domain(domain, dma_base, size, dev))
1129 goto out_err;
1130 dev->dma_ops = &iommu_dma_ops;
1131 }
1132
1133 return;
1134out_err:
1135 pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
1136 dev_name(dev));
1137}
1138
1139static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
1140 phys_addr_t msi_addr, struct iommu_domain *domain)
1141{
1142 struct iommu_dma_cookie *cookie = domain->iova_cookie;
1143 struct iommu_dma_msi_page *msi_page;
1144 dma_addr_t iova;
1145 int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1146 size_t size = cookie_msi_granule(cookie);
1147
1148 msi_addr &= ~(phys_addr_t)(size - 1);
1149 list_for_each_entry(msi_page, &cookie->msi_page_list, list)
1150 if (msi_page->phys == msi_addr)
1151 return msi_page;
1152
1153 msi_page = kzalloc(sizeof(*msi_page), GFP_ATOMIC);
1154 if (!msi_page)
1155 return NULL;
1156
1157 iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev);
1158 if (!iova)
1159 goto out_free_page;
1160
1161 if (iommu_map(domain, iova, msi_addr, size, prot))
1162 goto out_free_iova;
1163
1164 INIT_LIST_HEAD(&msi_page->list);
1165 msi_page->phys = msi_addr;
1166 msi_page->iova = iova;
1167 list_add(&msi_page->list, &cookie->msi_page_list);
1168 return msi_page;
1169
1170out_free_iova:
1171 iommu_dma_free_iova(cookie, iova, size);
1172out_free_page:
1173 kfree(msi_page);
1174 return NULL;
1175}
1176
1177int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
1178{
1179 struct device *dev = msi_desc_to_dev(desc);
1180 struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1181 struct iommu_dma_cookie *cookie;
1182 struct iommu_dma_msi_page *msi_page;
1183 unsigned long flags;
1184
1185 if (!domain || !domain->iova_cookie) {
1186 desc->iommu_cookie = NULL;
1187 return 0;
1188 }
1189
1190 cookie = domain->iova_cookie;
1191
1192 /*
1193 * We disable IRQs to rule out a possible inversion against
1194 * irq_desc_lock if, say, someone tries to retarget the affinity
1195 * of an MSI from within an IPI handler.
1196 */
1197 spin_lock_irqsave(&cookie->msi_lock, flags);
1198 msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain);
1199 spin_unlock_irqrestore(&cookie->msi_lock, flags);
1200
1201 msi_desc_set_iommu_cookie(desc, msi_page);
1202
1203 if (!msi_page)
1204 return -ENOMEM;
1205 return 0;
1206}
1207
1208void iommu_dma_compose_msi_msg(struct msi_desc *desc,
1209 struct msi_msg *msg)
1210{
1211 struct device *dev = msi_desc_to_dev(desc);
1212 const struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1213 const struct iommu_dma_msi_page *msi_page;
1214
1215 msi_page = msi_desc_get_iommu_cookie(desc);
1216
1217 if (!domain || !domain->iova_cookie || WARN_ON(!msi_page))
1218 return;
1219
1220 msg->address_hi = upper_32_bits(msi_page->iova);
1221 msg->address_lo &= cookie_msi_granule(domain->iova_cookie) - 1;
1222 msg->address_lo += lower_32_bits(msi_page->iova);
1223}
1224
1225static int iommu_dma_init(void)
1226{
1227 return iova_cache_get();
1228}
1229arch_initcall(iommu_dma_init);