1// SPDX-License-Identifier: GPL-2.0-only
   2/*
   3 * A fairly generic DMA-API to IOMMU-API glue layer.
   4 *
   5 * Copyright (C) 2014-2015 ARM Ltd.
   6 *
   7 * based in part on arch/arm/mm/dma-mapping.c:
   8 * Copyright (C) 2000-2004 Russell King
   9 */
  10
  11#include <linux/acpi_iort.h>
  12#include <linux/atomic.h>
  13#include <linux/crash_dump.h>
  14#include <linux/device.h>
  15#include <linux/dma-direct.h>
  16#include <linux/dma-map-ops.h>
 
  17#include <linux/gfp.h>
  18#include <linux/huge_mm.h>
  19#include <linux/iommu.h>
  20#include <linux/iova.h>
  21#include <linux/irq.h>
  22#include <linux/list_sort.h>
  23#include <linux/memremap.h>
  24#include <linux/mm.h>
  25#include <linux/mutex.h>
  26#include <linux/pci.h>
  27#include <linux/scatterlist.h>
  28#include <linux/spinlock.h>
  29#include <linux/swiotlb.h>
 
  30#include <linux/vmalloc.h>
  31
  32#include "dma-iommu.h"
  33
  34struct iommu_dma_msi_page {
  35	struct list_head	list;
  36	dma_addr_t		iova;
  37	phys_addr_t		phys;
  38};
  39
  40enum iommu_dma_cookie_type {
  41	IOMMU_DMA_IOVA_COOKIE,
  42	IOMMU_DMA_MSI_COOKIE,
  43};
  44
  45struct iommu_dma_cookie {
  46	enum iommu_dma_cookie_type	type;
  47	union {
  48		/* Full allocator for IOMMU_DMA_IOVA_COOKIE */
  49		struct {
  50			struct iova_domain	iovad;
  51
  52			struct iova_fq __percpu *fq;	/* Flush queue */
  53			/* Number of TLB flushes that have been started */
  54			atomic64_t		fq_flush_start_cnt;
  55			/* Number of TLB flushes that have been finished */
  56			atomic64_t		fq_flush_finish_cnt;
  57			/* Timer to regularily empty the flush queues */
  58			struct timer_list	fq_timer;
  59			/* 1 when timer is active, 0 when not */
  60			atomic_t		fq_timer_on;
  61		};
  62		/* Trivial linear page allocator for IOMMU_DMA_MSI_COOKIE */
  63		dma_addr_t		msi_iova;
  64	};
  65	struct list_head		msi_page_list;
  66
  67	/* Domain for flush queue callback; NULL if flush queue not in use */
  68	struct iommu_domain		*fq_domain;
  69	struct mutex			mutex;
  70};
  71
  72static DEFINE_STATIC_KEY_FALSE(iommu_deferred_attach_enabled);
  73bool iommu_dma_forcedac __read_mostly;
  74
  75static int __init iommu_dma_forcedac_setup(char *str)
  76{
  77	int ret = kstrtobool(str, &iommu_dma_forcedac);
  78
  79	if (!ret && iommu_dma_forcedac)
  80		pr_info("Forcing DAC for PCI devices\n");
  81	return ret;
  82}
  83early_param("iommu.forcedac", iommu_dma_forcedac_setup);
  84
  85/* Number of entries per flush queue */
  86#define IOVA_FQ_SIZE	256
  87
  88/* Timeout (in ms) after which entries are flushed from the queue */
  89#define IOVA_FQ_TIMEOUT	10
  90
  91/* Flush queue entry for deferred flushing */
  92struct iova_fq_entry {
  93	unsigned long iova_pfn;
  94	unsigned long pages;
  95	struct list_head freelist;
  96	u64 counter; /* Flush counter when this entry was added */
  97};
  98
  99/* Per-CPU flush queue structure */
 100struct iova_fq {
 101	struct iova_fq_entry entries[IOVA_FQ_SIZE];
 102	unsigned int head, tail;
 103	spinlock_t lock;
 104};
 105
 106#define fq_ring_for_each(i, fq) \
 107	for ((i) = (fq)->head; (i) != (fq)->tail; (i) = ((i) + 1) % IOVA_FQ_SIZE)
 108
 109static inline bool fq_full(struct iova_fq *fq)
 110{
 111	assert_spin_locked(&fq->lock);
 112	return (((fq->tail + 1) % IOVA_FQ_SIZE) == fq->head);
 113}
 114
 115static inline unsigned int fq_ring_add(struct iova_fq *fq)
 116{
 117	unsigned int idx = fq->tail;
 118
 119	assert_spin_locked(&fq->lock);
 120
 121	fq->tail = (idx + 1) % IOVA_FQ_SIZE;
 122
 123	return idx;
 124}
 125
 126static void fq_ring_free(struct iommu_dma_cookie *cookie, struct iova_fq *fq)
 127{
 128	u64 counter = atomic64_read(&cookie->fq_flush_finish_cnt);
 129	unsigned int idx;
 130
 131	assert_spin_locked(&fq->lock);
 132
 133	fq_ring_for_each(idx, fq) {
 134
 135		if (fq->entries[idx].counter >= counter)
 136			break;
 137
 138		put_pages_list(&fq->entries[idx].freelist);
 139		free_iova_fast(&cookie->iovad,
 140			       fq->entries[idx].iova_pfn,
 141			       fq->entries[idx].pages);
 142
 143		fq->head = (fq->head + 1) % IOVA_FQ_SIZE;
 144	}
 145}
 146
 147static void fq_flush_iotlb(struct iommu_dma_cookie *cookie)
 148{
 149	atomic64_inc(&cookie->fq_flush_start_cnt);
 150	cookie->fq_domain->ops->flush_iotlb_all(cookie->fq_domain);
 151	atomic64_inc(&cookie->fq_flush_finish_cnt);
 152}
 153
 154static void fq_flush_timeout(struct timer_list *t)
 155{
 156	struct iommu_dma_cookie *cookie = from_timer(cookie, t, fq_timer);
 157	int cpu;
 158
 159	atomic_set(&cookie->fq_timer_on, 0);
 160	fq_flush_iotlb(cookie);
 161
 162	for_each_possible_cpu(cpu) {
 163		unsigned long flags;
 164		struct iova_fq *fq;
 165
 166		fq = per_cpu_ptr(cookie->fq, cpu);
 167		spin_lock_irqsave(&fq->lock, flags);
 168		fq_ring_free(cookie, fq);
 169		spin_unlock_irqrestore(&fq->lock, flags);
 170	}
 171}
 172
 173static void queue_iova(struct iommu_dma_cookie *cookie,
 174		unsigned long pfn, unsigned long pages,
 175		struct list_head *freelist)
 176{
 177	struct iova_fq *fq;
 178	unsigned long flags;
 179	unsigned int idx;
 180
 181	/*
 182	 * Order against the IOMMU driver's pagetable update from unmapping
 183	 * @pte, to guarantee that fq_flush_iotlb() observes that if called
 184	 * from a different CPU before we release the lock below. Full barrier
 185	 * so it also pairs with iommu_dma_init_fq() to avoid seeing partially
 186	 * written fq state here.
 187	 */
 188	smp_mb();
 189
 190	fq = raw_cpu_ptr(cookie->fq);
 191	spin_lock_irqsave(&fq->lock, flags);
 192
 193	/*
 194	 * First remove all entries from the flush queue that have already been
 195	 * flushed out on another CPU. This makes the fq_full() check below less
 196	 * likely to be true.
 197	 */
 198	fq_ring_free(cookie, fq);
 199
 200	if (fq_full(fq)) {
 201		fq_flush_iotlb(cookie);
 202		fq_ring_free(cookie, fq);
 203	}
 204
 205	idx = fq_ring_add(fq);
 206
 207	fq->entries[idx].iova_pfn = pfn;
 208	fq->entries[idx].pages    = pages;
 209	fq->entries[idx].counter  = atomic64_read(&cookie->fq_flush_start_cnt);
 210	list_splice(freelist, &fq->entries[idx].freelist);
 211
 212	spin_unlock_irqrestore(&fq->lock, flags);
 213
 214	/* Avoid false sharing as much as possible. */
 215	if (!atomic_read(&cookie->fq_timer_on) &&
 216	    !atomic_xchg(&cookie->fq_timer_on, 1))
 217		mod_timer(&cookie->fq_timer,
 218			  jiffies + msecs_to_jiffies(IOVA_FQ_TIMEOUT));
 219}
 220
 221static void iommu_dma_free_fq(struct iommu_dma_cookie *cookie)
 222{
 223	int cpu, idx;
 224
 225	if (!cookie->fq)
 226		return;
 227
 228	del_timer_sync(&cookie->fq_timer);
 229	/* The IOVAs will be torn down separately, so just free our queued pages */
 230	for_each_possible_cpu(cpu) {
 231		struct iova_fq *fq = per_cpu_ptr(cookie->fq, cpu);
 232
 233		fq_ring_for_each(idx, fq)
 234			put_pages_list(&fq->entries[idx].freelist);
 235	}
 236
 237	free_percpu(cookie->fq);
 238}
 239
 240/* sysfs updates are serialised by the mutex of the group owning @domain */
 241int iommu_dma_init_fq(struct iommu_domain *domain)
 242{
 243	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 244	struct iova_fq __percpu *queue;
 245	int i, cpu;
 246
 247	if (cookie->fq_domain)
 248		return 0;
 249
 250	atomic64_set(&cookie->fq_flush_start_cnt,  0);
 251	atomic64_set(&cookie->fq_flush_finish_cnt, 0);
 252
 253	queue = alloc_percpu(struct iova_fq);
 254	if (!queue) {
 255		pr_warn("iova flush queue initialization failed\n");
 256		return -ENOMEM;
 257	}
 258
 259	for_each_possible_cpu(cpu) {
 260		struct iova_fq *fq = per_cpu_ptr(queue, cpu);
 261
 262		fq->head = 0;
 263		fq->tail = 0;
 264
 265		spin_lock_init(&fq->lock);
 266
 267		for (i = 0; i < IOVA_FQ_SIZE; i++)
 268			INIT_LIST_HEAD(&fq->entries[i].freelist);
 269	}
 270
 271	cookie->fq = queue;
 272
 273	timer_setup(&cookie->fq_timer, fq_flush_timeout, 0);
 274	atomic_set(&cookie->fq_timer_on, 0);
 275	/*
 276	 * Prevent incomplete fq state being observable. Pairs with path from
 277	 * __iommu_dma_unmap() through iommu_dma_free_iova() to queue_iova()
 278	 */
 279	smp_wmb();
 280	WRITE_ONCE(cookie->fq_domain, domain);
 281	return 0;
 282}
 283
 284static inline size_t cookie_msi_granule(struct iommu_dma_cookie *cookie)
 285{
 286	if (cookie->type == IOMMU_DMA_IOVA_COOKIE)
 287		return cookie->iovad.granule;
 288	return PAGE_SIZE;
 289}
 290
 291static struct iommu_dma_cookie *cookie_alloc(enum iommu_dma_cookie_type type)
 292{
 293	struct iommu_dma_cookie *cookie;
 294
 295	cookie = kzalloc(sizeof(*cookie), GFP_KERNEL);
 296	if (cookie) {
 297		INIT_LIST_HEAD(&cookie->msi_page_list);
 298		cookie->type = type;
 299	}
 300	return cookie;
 301}
 302
 303/**
 304 * iommu_get_dma_cookie - Acquire DMA-API resources for a domain
 305 * @domain: IOMMU domain to prepare for DMA-API usage
 
 
 
 306 */
 307int iommu_get_dma_cookie(struct iommu_domain *domain)
 308{
 309	if (domain->iova_cookie)
 310		return -EEXIST;
 311
 312	domain->iova_cookie = cookie_alloc(IOMMU_DMA_IOVA_COOKIE);
 313	if (!domain->iova_cookie)
 314		return -ENOMEM;
 315
 316	mutex_init(&domain->iova_cookie->mutex);
 317	return 0;
 318}
 
 319
 320/**
 321 * iommu_get_msi_cookie - Acquire just MSI remapping resources
 322 * @domain: IOMMU domain to prepare
 323 * @base: Start address of IOVA region for MSI mappings
 324 *
 325 * Users who manage their own IOVA allocation and do not want DMA API support,
 326 * but would still like to take advantage of automatic MSI remapping, can use
 327 * this to initialise their own domain appropriately. Users should reserve a
 328 * contiguous IOVA region, starting at @base, large enough to accommodate the
 329 * number of PAGE_SIZE mappings necessary to cover every MSI doorbell address
 330 * used by the devices attached to @domain.
 331 */
 332int iommu_get_msi_cookie(struct iommu_domain *domain, dma_addr_t base)
 333{
 334	struct iommu_dma_cookie *cookie;
 335
 336	if (domain->type != IOMMU_DOMAIN_UNMANAGED)
 337		return -EINVAL;
 338
 339	if (domain->iova_cookie)
 340		return -EEXIST;
 341
 342	cookie = cookie_alloc(IOMMU_DMA_MSI_COOKIE);
 343	if (!cookie)
 344		return -ENOMEM;
 345
 346	cookie->msi_iova = base;
 347	domain->iova_cookie = cookie;
 348	return 0;
 349}
 350EXPORT_SYMBOL(iommu_get_msi_cookie);
 351
 352/**
 353 * iommu_put_dma_cookie - Release a domain's DMA mapping resources
 354 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie() or
 355 *          iommu_get_msi_cookie()
 
 
 356 */
 357void iommu_put_dma_cookie(struct iommu_domain *domain)
 358{
 359	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 360	struct iommu_dma_msi_page *msi, *tmp;
 361
 362	if (!cookie)
 363		return;
 364
 365	if (cookie->type == IOMMU_DMA_IOVA_COOKIE && cookie->iovad.granule) {
 366		iommu_dma_free_fq(cookie);
 367		put_iova_domain(&cookie->iovad);
 368	}
 369
 370	list_for_each_entry_safe(msi, tmp, &cookie->msi_page_list, list) {
 371		list_del(&msi->list);
 372		kfree(msi);
 373	}
 374	kfree(cookie);
 375	domain->iova_cookie = NULL;
 376}
 
 377
 378/**
 379 * iommu_dma_get_resv_regions - Reserved region driver helper
 380 * @dev: Device from iommu_get_resv_regions()
 381 * @list: Reserved region list from iommu_get_resv_regions()
 382 *
 383 * IOMMU drivers can use this to implement their .get_resv_regions callback
 384 * for general non-IOMMU-specific reservations. Currently, this covers GICv3
 385 * ITS region reservation on ACPI based ARM platforms that may require HW MSI
 386 * reservation.
 387 */
 388void iommu_dma_get_resv_regions(struct device *dev, struct list_head *list)
 389{
 390
 391	if (!is_of_node(dev_iommu_fwspec_get(dev)->iommu_fwnode))
 392		iort_iommu_get_resv_regions(dev, list);
 393
 394}
 395EXPORT_SYMBOL(iommu_dma_get_resv_regions);
 396
 397static int cookie_init_hw_msi_region(struct iommu_dma_cookie *cookie,
 398		phys_addr_t start, phys_addr_t end)
 399{
 400	struct iova_domain *iovad = &cookie->iovad;
 401	struct iommu_dma_msi_page *msi_page;
 402	int i, num_pages;
 403
 404	start -= iova_offset(iovad, start);
 405	num_pages = iova_align(iovad, end - start) >> iova_shift(iovad);
 406
 407	for (i = 0; i < num_pages; i++) {
 408		msi_page = kmalloc(sizeof(*msi_page), GFP_KERNEL);
 409		if (!msi_page)
 410			return -ENOMEM;
 411
 412		msi_page->phys = start;
 413		msi_page->iova = start;
 414		INIT_LIST_HEAD(&msi_page->list);
 415		list_add(&msi_page->list, &cookie->msi_page_list);
 416		start += iovad->granule;
 417	}
 418
 419	return 0;
 420}
 421
 422static int iommu_dma_ranges_sort(void *priv, const struct list_head *a,
 423		const struct list_head *b)
 424{
 425	struct resource_entry *res_a = list_entry(a, typeof(*res_a), node);
 426	struct resource_entry *res_b = list_entry(b, typeof(*res_b), node);
 427
 428	return res_a->res->start > res_b->res->start;
 429}
 430
 431static int iova_reserve_pci_windows(struct pci_dev *dev,
 432		struct iova_domain *iovad)
 433{
 434	struct pci_host_bridge *bridge = pci_find_host_bridge(dev->bus);
 435	struct resource_entry *window;
 436	unsigned long lo, hi;
 437	phys_addr_t start = 0, end;
 438
 439	resource_list_for_each_entry(window, &bridge->windows) {
 440		if (resource_type(window->res) != IORESOURCE_MEM)
 441			continue;
 442
 443		lo = iova_pfn(iovad, window->res->start - window->offset);
 444		hi = iova_pfn(iovad, window->res->end - window->offset);
 445		reserve_iova(iovad, lo, hi);
 446	}
 447
 448	/* Get reserved DMA windows from host bridge */
 449	list_sort(NULL, &bridge->dma_ranges, iommu_dma_ranges_sort);
 450	resource_list_for_each_entry(window, &bridge->dma_ranges) {
 451		end = window->res->start - window->offset;
 452resv_iova:
 453		if (end > start) {
 454			lo = iova_pfn(iovad, start);
 455			hi = iova_pfn(iovad, end);
 456			reserve_iova(iovad, lo, hi);
 457		} else if (end < start) {
 458			/* DMA ranges should be non-overlapping */
 459			dev_err(&dev->dev,
 460				"Failed to reserve IOVA [%pa-%pa]\n",
 461				&start, &end);
 462			return -EINVAL;
 463		}
 464
 465		start = window->res->end - window->offset + 1;
 466		/* If window is last entry */
 467		if (window->node.next == &bridge->dma_ranges &&
 468		    end != ~(phys_addr_t)0) {
 469			end = ~(phys_addr_t)0;
 470			goto resv_iova;
 471		}
 472	}
 473
 474	return 0;
 475}
 476
 477static int iova_reserve_iommu_regions(struct device *dev,
 478		struct iommu_domain *domain)
 479{
 480	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 481	struct iova_domain *iovad = &cookie->iovad;
 482	struct iommu_resv_region *region;
 483	LIST_HEAD(resv_regions);
 484	int ret = 0;
 485
 486	if (dev_is_pci(dev)) {
 487		ret = iova_reserve_pci_windows(to_pci_dev(dev), iovad);
 488		if (ret)
 489			return ret;
 490	}
 491
 492	iommu_get_resv_regions(dev, &resv_regions);
 493	list_for_each_entry(region, &resv_regions, list) {
 494		unsigned long lo, hi;
 495
 496		/* We ARE the software that manages these! */
 497		if (region->type == IOMMU_RESV_SW_MSI)
 498			continue;
 499
 500		lo = iova_pfn(iovad, region->start);
 501		hi = iova_pfn(iovad, region->start + region->length - 1);
 502		reserve_iova(iovad, lo, hi);
 503
 504		if (region->type == IOMMU_RESV_MSI)
 505			ret = cookie_init_hw_msi_region(cookie, region->start,
 506					region->start + region->length);
 507		if (ret)
 508			break;
 509	}
 510	iommu_put_resv_regions(dev, &resv_regions);
 511
 512	return ret;
 513}
 514
 515static bool dev_is_untrusted(struct device *dev)
 516{
 517	return dev_is_pci(dev) && to_pci_dev(dev)->untrusted;
 
 
 
 
 
 
 518}
 519
 520static bool dev_use_swiotlb(struct device *dev)
 521{
 522	return IS_ENABLED(CONFIG_SWIOTLB) && dev_is_untrusted(dev);
 523}
 524
 525/**
 526 * iommu_dma_init_domain - Initialise a DMA mapping domain
 527 * @domain: IOMMU domain previously prepared by iommu_get_dma_cookie()
 528 * @base: IOVA at which the mappable address space starts
 529 * @limit: Last address of the IOVA space
 530 * @dev: Device the domain is being initialised for
 531 *
 532 * @base and @limit + 1 should be exact multiples of IOMMU page granularity to
 533 * avoid rounding surprises. If necessary, we reserve the page at address 0
 534 * to ensure it is an invalid IOVA. It is safe to reinitialise a domain, but
 535 * any change which could make prior IOVAs invalid will fail.
 536 */
 537static int iommu_dma_init_domain(struct iommu_domain *domain, dma_addr_t base,
 538				 dma_addr_t limit, struct device *dev)
 539{
 540	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 541	unsigned long order, base_pfn;
 542	struct iova_domain *iovad;
 543	int ret;
 544
 545	if (!cookie || cookie->type != IOMMU_DMA_IOVA_COOKIE)
 546		return -EINVAL;
 547
 548	iovad = &cookie->iovad;
 549
 550	/* Use the smallest supported page size for IOVA granularity */
 551	order = __ffs(domain->pgsize_bitmap);
 552	base_pfn = max_t(unsigned long, 1, base >> order);
 553
 554	/* Check the domain allows at least some access to the device... */
 555	if (domain->geometry.force_aperture) {
 556		if (base > domain->geometry.aperture_end ||
 557		    limit < domain->geometry.aperture_start) {
 558			pr_warn("specified DMA range outside IOMMU capability\n");
 559			return -EFAULT;
 560		}
 561		/* ...then finally give it a kicking to make sure it fits */
 562		base_pfn = max_t(unsigned long, base_pfn,
 563				domain->geometry.aperture_start >> order);
 564	}
 565
 566	/* start_pfn is always nonzero for an already-initialised domain */
 567	mutex_lock(&cookie->mutex);
 568	if (iovad->start_pfn) {
 569		if (1UL << order != iovad->granule ||
 570		    base_pfn != iovad->start_pfn) {
 571			pr_warn("Incompatible range for DMA domain\n");
 572			ret = -EFAULT;
 573			goto done_unlock;
 574		}
 575
 576		ret = 0;
 577		goto done_unlock;
 578	}
 579
 580	init_iova_domain(iovad, 1UL << order, base_pfn);
 581	ret = iova_domain_init_rcaches(iovad);
 582	if (ret)
 583		goto done_unlock;
 584
 585	/* If the FQ fails we can simply fall back to strict mode */
 586	if (domain->type == IOMMU_DOMAIN_DMA_FQ && iommu_dma_init_fq(domain))
 587		domain->type = IOMMU_DOMAIN_DMA;
 588
 589	ret = iova_reserve_iommu_regions(dev, domain);
 
 
 
 
 
 
 
 590
 591done_unlock:
 592	mutex_unlock(&cookie->mutex);
 593	return ret;
 
 594}
 595
 596/**
 597 * dma_info_to_prot - Translate DMA API directions and attributes to IOMMU API
 598 *                    page flags.
 599 * @dir: Direction of DMA transfer
 600 * @coherent: Is the DMA master cache-coherent?
 601 * @attrs: DMA attributes for the mapping
 602 *
 603 * Return: corresponding IOMMU API page protection flags
 604 */
 605static int dma_info_to_prot(enum dma_data_direction dir, bool coherent,
 606		     unsigned long attrs)
 607{
 608	int prot = coherent ? IOMMU_CACHE : 0;
 609
 610	if (attrs & DMA_ATTR_PRIVILEGED)
 611		prot |= IOMMU_PRIV;
 612
 613	switch (dir) {
 614	case DMA_BIDIRECTIONAL:
 615		return prot | IOMMU_READ | IOMMU_WRITE;
 616	case DMA_TO_DEVICE:
 617		return prot | IOMMU_READ;
 618	case DMA_FROM_DEVICE:
 619		return prot | IOMMU_WRITE;
 620	default:
 621		return 0;
 622	}
 623}
 624
 625static dma_addr_t iommu_dma_alloc_iova(struct iommu_domain *domain,
 626		size_t size, u64 dma_limit, struct device *dev)
 627{
 628	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 629	struct iova_domain *iovad = &cookie->iovad;
 630	unsigned long shift, iova_len, iova = 0;
 631
 632	if (cookie->type == IOMMU_DMA_MSI_COOKIE) {
 633		cookie->msi_iova += size;
 634		return cookie->msi_iova - size;
 635	}
 636
 637	shift = iova_shift(iovad);
 638	iova_len = size >> shift;
 
 
 
 
 
 
 
 
 639
 640	dma_limit = min_not_zero(dma_limit, dev->bus_dma_limit);
 641
 642	if (domain->geometry.force_aperture)
 643		dma_limit = min(dma_limit, (u64)domain->geometry.aperture_end);
 644
 645	/* Try to get PCI devices a SAC address */
 646	if (dma_limit > DMA_BIT_MASK(32) && !iommu_dma_forcedac && dev_is_pci(dev))
 647		iova = alloc_iova_fast(iovad, iova_len,
 648				       DMA_BIT_MASK(32) >> shift, false);
 649
 650	if (!iova)
 651		iova = alloc_iova_fast(iovad, iova_len, dma_limit >> shift,
 652				       true);
 653
 654	return (dma_addr_t)iova << shift;
 655}
 656
 657static void iommu_dma_free_iova(struct iommu_dma_cookie *cookie,
 658		dma_addr_t iova, size_t size, struct iommu_iotlb_gather *gather)
 659{
 660	struct iova_domain *iovad = &cookie->iovad;
 661
 662	/* The MSI case is only ever cleaning up its most recent allocation */
 663	if (cookie->type == IOMMU_DMA_MSI_COOKIE)
 664		cookie->msi_iova -= size;
 665	else if (gather && gather->queued)
 666		queue_iova(cookie, iova_pfn(iovad, iova),
 667				size >> iova_shift(iovad),
 668				&gather->freelist);
 669	else
 670		free_iova_fast(iovad, iova_pfn(iovad, iova),
 671				size >> iova_shift(iovad));
 672}
 673
 674static void __iommu_dma_unmap(struct device *dev, dma_addr_t dma_addr,
 675		size_t size)
 676{
 677	struct iommu_domain *domain = iommu_get_dma_domain(dev);
 678	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 679	struct iova_domain *iovad = &cookie->iovad;
 680	size_t iova_off = iova_offset(iovad, dma_addr);
 681	struct iommu_iotlb_gather iotlb_gather;
 682	size_t unmapped;
 683
 684	dma_addr -= iova_off;
 685	size = iova_align(iovad, size + iova_off);
 686	iommu_iotlb_gather_init(&iotlb_gather);
 687	iotlb_gather.queued = READ_ONCE(cookie->fq_domain);
 688
 689	unmapped = iommu_unmap_fast(domain, dma_addr, size, &iotlb_gather);
 690	WARN_ON(unmapped != size);
 691
 692	if (!iotlb_gather.queued)
 693		iommu_iotlb_sync(domain, &iotlb_gather);
 694	iommu_dma_free_iova(cookie, dma_addr, size, &iotlb_gather);
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 695}
 696
 697static dma_addr_t __iommu_dma_map(struct device *dev, phys_addr_t phys,
 698		size_t size, int prot, u64 dma_mask)
 699{
 700	struct iommu_domain *domain = iommu_get_dma_domain(dev);
 701	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 702	struct iova_domain *iovad = &cookie->iovad;
 703	size_t iova_off = iova_offset(iovad, phys);
 704	dma_addr_t iova;
 705
 706	if (static_branch_unlikely(&iommu_deferred_attach_enabled) &&
 707	    iommu_deferred_attach(dev, domain))
 708		return DMA_MAPPING_ERROR;
 709
 710	size = iova_align(iovad, size + iova_off);
 711
 712	iova = iommu_dma_alloc_iova(domain, size, dma_mask, dev);
 713	if (!iova)
 714		return DMA_MAPPING_ERROR;
 715
 716	if (iommu_map_atomic(domain, iova, phys - iova_off, size, prot)) {
 717		iommu_dma_free_iova(cookie, iova, size, NULL);
 718		return DMA_MAPPING_ERROR;
 719	}
 720	return iova + iova_off;
 721}
 722
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 723static void __iommu_dma_free_pages(struct page **pages, int count)
 724{
 725	while (count--)
 726		__free_page(pages[count]);
 727	kvfree(pages);
 728}
 729
 730static struct page **__iommu_dma_alloc_pages(struct device *dev,
 731		unsigned int count, unsigned long order_mask, gfp_t gfp)
 732{
 733	struct page **pages;
 734	unsigned int i = 0, nid = dev_to_node(dev);
 735
 736	order_mask &= (2U << MAX_ORDER) - 1;
 737	if (!order_mask)
 738		return NULL;
 739
 740	pages = kvcalloc(count, sizeof(*pages), GFP_KERNEL);
 741	if (!pages)
 742		return NULL;
 743
 744	/* IOMMU can map any pages, so himem can also be used here */
 745	gfp |= __GFP_NOWARN | __GFP_HIGHMEM;
 746
 
 
 
 747	while (count) {
 748		struct page *page = NULL;
 749		unsigned int order_size;
 750
 751		/*
 752		 * Higher-order allocations are a convenience rather
 753		 * than a necessity, hence using __GFP_NORETRY until
 754		 * falling back to minimum-order allocations.
 755		 */
 756		for (order_mask &= (2U << __fls(count)) - 1;
 757		     order_mask; order_mask &= ~order_size) {
 758			unsigned int order = __fls(order_mask);
 759			gfp_t alloc_flags = gfp;
 760
 761			order_size = 1U << order;
 762			if (order_mask > order_size)
 763				alloc_flags |= __GFP_NORETRY;
 764			page = alloc_pages_node(nid, alloc_flags, order);
 765			if (!page)
 766				continue;
 767			if (order)
 768				split_page(page, order);
 769			break;
 770		}
 771		if (!page) {
 772			__iommu_dma_free_pages(pages, i);
 773			return NULL;
 774		}
 775		count -= order_size;
 776		while (order_size--)
 777			pages[i++] = page++;
 778	}
 779	return pages;
 780}
 781
 782/*
 783 * If size is less than PAGE_SIZE, then a full CPU page will be allocated,
 784 * but an IOMMU which supports smaller pages might not map the whole thing.
 785 */
 786static struct page **__iommu_dma_alloc_noncontiguous(struct device *dev,
 787		size_t size, struct sg_table *sgt, gfp_t gfp, pgprot_t prot,
 788		unsigned long attrs)
 789{
 790	struct iommu_domain *domain = iommu_get_dma_domain(dev);
 791	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 792	struct iova_domain *iovad = &cookie->iovad;
 793	bool coherent = dev_is_dma_coherent(dev);
 794	int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
 795	unsigned int count, min_size, alloc_sizes = domain->pgsize_bitmap;
 796	struct page **pages;
 797	dma_addr_t iova;
 798	ssize_t ret;
 799
 800	if (static_branch_unlikely(&iommu_deferred_attach_enabled) &&
 801	    iommu_deferred_attach(dev, domain))
 802		return NULL;
 803
 804	min_size = alloc_sizes & -alloc_sizes;
 805	if (min_size < PAGE_SIZE) {
 806		min_size = PAGE_SIZE;
 807		alloc_sizes |= PAGE_SIZE;
 808	} else {
 809		size = ALIGN(size, min_size);
 810	}
 811	if (attrs & DMA_ATTR_ALLOC_SINGLE_PAGES)
 812		alloc_sizes = min_size;
 813
 814	count = PAGE_ALIGN(size) >> PAGE_SHIFT;
 815	pages = __iommu_dma_alloc_pages(dev, count, alloc_sizes >> PAGE_SHIFT,
 816					gfp);
 817	if (!pages)
 818		return NULL;
 819
 820	size = iova_align(iovad, size);
 821	iova = iommu_dma_alloc_iova(domain, size, dev->coherent_dma_mask, dev);
 822	if (!iova)
 823		goto out_free_pages;
 824
 825	if (sg_alloc_table_from_pages(sgt, pages, count, 0, size, GFP_KERNEL))
 826		goto out_free_iova;
 827
 828	if (!(ioprot & IOMMU_CACHE)) {
 829		struct scatterlist *sg;
 830		int i;
 831
 832		for_each_sg(sgt->sgl, sg, sgt->orig_nents, i)
 833			arch_dma_prep_coherent(sg_page(sg), sg->length);
 834	}
 835
 836	ret = iommu_map_sg_atomic(domain, iova, sgt->sgl, sgt->orig_nents, ioprot);
 837	if (ret < 0 || ret < size)
 838		goto out_free_sg;
 839
 840	sgt->sgl->dma_address = iova;
 841	sgt->sgl->dma_length = size;
 842	return pages;
 843
 844out_free_sg:
 845	sg_free_table(sgt);
 846out_free_iova:
 847	iommu_dma_free_iova(cookie, iova, size, NULL);
 848out_free_pages:
 849	__iommu_dma_free_pages(pages, count);
 850	return NULL;
 851}
 852
 853static void *iommu_dma_alloc_remap(struct device *dev, size_t size,
 854		dma_addr_t *dma_handle, gfp_t gfp, pgprot_t prot,
 855		unsigned long attrs)
 856{
 857	struct page **pages;
 858	struct sg_table sgt;
 859	void *vaddr;
 860
 861	pages = __iommu_dma_alloc_noncontiguous(dev, size, &sgt, gfp, prot,
 862						attrs);
 863	if (!pages)
 864		return NULL;
 865	*dma_handle = sgt.sgl->dma_address;
 866	sg_free_table(&sgt);
 867	vaddr = dma_common_pages_remap(pages, size, prot,
 868			__builtin_return_address(0));
 869	if (!vaddr)
 870		goto out_unmap;
 871	return vaddr;
 872
 873out_unmap:
 874	__iommu_dma_unmap(dev, *dma_handle, size);
 875	__iommu_dma_free_pages(pages, PAGE_ALIGN(size) >> PAGE_SHIFT);
 876	return NULL;
 877}
 878
 
 879static struct sg_table *iommu_dma_alloc_noncontiguous(struct device *dev,
 880		size_t size, enum dma_data_direction dir, gfp_t gfp,
 881		unsigned long attrs)
 882{
 883	struct dma_sgt_handle *sh;
 884
 885	sh = kmalloc(sizeof(*sh), gfp);
 886	if (!sh)
 887		return NULL;
 888
 889	sh->pages = __iommu_dma_alloc_noncontiguous(dev, size, &sh->sgt, gfp,
 890						    PAGE_KERNEL, attrs);
 891	if (!sh->pages) {
 892		kfree(sh);
 893		return NULL;
 894	}
 895	return &sh->sgt;
 896}
 897
 898static void iommu_dma_free_noncontiguous(struct device *dev, size_t size,
 899		struct sg_table *sgt, enum dma_data_direction dir)
 900{
 901	struct dma_sgt_handle *sh = sgt_handle(sgt);
 902
 903	__iommu_dma_unmap(dev, sgt->sgl->dma_address, size);
 904	__iommu_dma_free_pages(sh->pages, PAGE_ALIGN(size) >> PAGE_SHIFT);
 905	sg_free_table(&sh->sgt);
 906	kfree(sh);
 907}
 
 908
 909static void iommu_dma_sync_single_for_cpu(struct device *dev,
 910		dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
 911{
 912	phys_addr_t phys;
 913
 914	if (dev_is_dma_coherent(dev) && !dev_use_swiotlb(dev))
 915		return;
 916
 917	phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
 918	if (!dev_is_dma_coherent(dev))
 919		arch_sync_dma_for_cpu(phys, size, dir);
 920
 921	if (is_swiotlb_buffer(dev, phys))
 922		swiotlb_sync_single_for_cpu(dev, phys, size, dir);
 923}
 924
 925static void iommu_dma_sync_single_for_device(struct device *dev,
 926		dma_addr_t dma_handle, size_t size, enum dma_data_direction dir)
 927{
 928	phys_addr_t phys;
 929
 930	if (dev_is_dma_coherent(dev) && !dev_use_swiotlb(dev))
 931		return;
 932
 933	phys = iommu_iova_to_phys(iommu_get_dma_domain(dev), dma_handle);
 934	if (is_swiotlb_buffer(dev, phys))
 935		swiotlb_sync_single_for_device(dev, phys, size, dir);
 936
 937	if (!dev_is_dma_coherent(dev))
 938		arch_sync_dma_for_device(phys, size, dir);
 939}
 940
 941static void iommu_dma_sync_sg_for_cpu(struct device *dev,
 942		struct scatterlist *sgl, int nelems,
 943		enum dma_data_direction dir)
 944{
 945	struct scatterlist *sg;
 946	int i;
 947
 948	if (dev_use_swiotlb(dev))
 949		for_each_sg(sgl, sg, nelems, i)
 950			iommu_dma_sync_single_for_cpu(dev, sg_dma_address(sg),
 951						      sg->length, dir);
 952	else if (!dev_is_dma_coherent(dev))
 953		for_each_sg(sgl, sg, nelems, i)
 954			arch_sync_dma_for_cpu(sg_phys(sg), sg->length, dir);
 
 
 
 
 
 955}
 956
 957static void iommu_dma_sync_sg_for_device(struct device *dev,
 958		struct scatterlist *sgl, int nelems,
 959		enum dma_data_direction dir)
 960{
 961	struct scatterlist *sg;
 962	int i;
 963
 964	if (dev_use_swiotlb(dev))
 965		for_each_sg(sgl, sg, nelems, i)
 966			iommu_dma_sync_single_for_device(dev,
 967							 sg_dma_address(sg),
 968							 sg->length, dir);
 969	else if (!dev_is_dma_coherent(dev))
 970		for_each_sg(sgl, sg, nelems, i)
 
 
 971			arch_sync_dma_for_device(sg_phys(sg), sg->length, dir);
 
 972}
 973
 974static dma_addr_t iommu_dma_map_page(struct device *dev, struct page *page,
 975		unsigned long offset, size_t size, enum dma_data_direction dir,
 976		unsigned long attrs)
 977{
 978	phys_addr_t phys = page_to_phys(page) + offset;
 979	bool coherent = dev_is_dma_coherent(dev);
 980	int prot = dma_info_to_prot(dir, coherent, attrs);
 981	struct iommu_domain *domain = iommu_get_dma_domain(dev);
 982	struct iommu_dma_cookie *cookie = domain->iova_cookie;
 983	struct iova_domain *iovad = &cookie->iovad;
 984	dma_addr_t iova, dma_mask = dma_get_mask(dev);
 985
 986	/*
 987	 * If both the physical buffer start address and size are
 988	 * page aligned, we don't need to use a bounce page.
 989	 */
 990	if (dev_use_swiotlb(dev) && iova_offset(iovad, phys | size)) {
 991		void *padding_start;
 992		size_t padding_size, aligned_size;
 993
 994		if (!is_swiotlb_active(dev)) {
 995			dev_warn_once(dev, "DMA bounce buffers are inactive, unable to map unaligned transaction.\n");
 996			return DMA_MAPPING_ERROR;
 997		}
 998
 999		aligned_size = iova_align(iovad, size);
1000		phys = swiotlb_tbl_map_single(dev, phys, size, aligned_size,
1001					      iova_mask(iovad), dir, attrs);
1002
1003		if (phys == DMA_MAPPING_ERROR)
1004			return DMA_MAPPING_ERROR;
1005
1006		/* Cleanup the padding area. */
1007		padding_start = phys_to_virt(phys);
1008		padding_size = aligned_size;
1009
1010		if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) &&
1011		    (dir == DMA_TO_DEVICE || dir == DMA_BIDIRECTIONAL)) {
1012			padding_start += size;
1013			padding_size -= size;
1014		}
1015
1016		memset(padding_start, 0, padding_size);
1017	}
1018
1019	if (!coherent && !(attrs & DMA_ATTR_SKIP_CPU_SYNC))
 
 
 
1020		arch_sync_dma_for_device(phys, size, dir);
1021
1022	iova = __iommu_dma_map(dev, phys, size, prot, dma_mask);
1023	if (iova == DMA_MAPPING_ERROR && is_swiotlb_buffer(dev, phys))
1024		swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
1025	return iova;
1026}
1027
1028static void iommu_dma_unmap_page(struct device *dev, dma_addr_t dma_handle,
1029		size_t size, enum dma_data_direction dir, unsigned long attrs)
1030{
1031	struct iommu_domain *domain = iommu_get_dma_domain(dev);
1032	phys_addr_t phys;
1033
1034	phys = iommu_iova_to_phys(domain, dma_handle);
1035	if (WARN_ON(!phys))
1036		return;
1037
1038	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC) && !dev_is_dma_coherent(dev))
1039		arch_sync_dma_for_cpu(phys, size, dir);
1040
1041	__iommu_dma_unmap(dev, dma_handle, size);
1042
1043	if (unlikely(is_swiotlb_buffer(dev, phys)))
1044		swiotlb_tbl_unmap_single(dev, phys, size, dir, attrs);
1045}
1046
1047/*
1048 * Prepare a successfully-mapped scatterlist to give back to the caller.
1049 *
1050 * At this point the segments are already laid out by iommu_dma_map_sg() to
1051 * avoid individually crossing any boundaries, so we merely need to check a
1052 * segment's start address to avoid concatenating across one.
1053 */
1054static int __finalise_sg(struct device *dev, struct scatterlist *sg, int nents,
1055		dma_addr_t dma_addr)
1056{
1057	struct scatterlist *s, *cur = sg;
1058	unsigned long seg_mask = dma_get_seg_boundary(dev);
1059	unsigned int cur_len = 0, max_len = dma_get_max_seg_size(dev);
1060	int i, count = 0;
1061
1062	for_each_sg(sg, s, nents, i) {
1063		/* Restore this segment's original unaligned fields first */
1064		dma_addr_t s_dma_addr = sg_dma_address(s);
1065		unsigned int s_iova_off = sg_dma_address(s);
1066		unsigned int s_length = sg_dma_len(s);
1067		unsigned int s_iova_len = s->length;
1068
1069		sg_dma_address(s) = DMA_MAPPING_ERROR;
1070		sg_dma_len(s) = 0;
1071
1072		if (sg_is_dma_bus_address(s)) {
1073			if (i > 0)
1074				cur = sg_next(cur);
1075
1076			sg_dma_unmark_bus_address(s);
1077			sg_dma_address(cur) = s_dma_addr;
1078			sg_dma_len(cur) = s_length;
1079			sg_dma_mark_bus_address(cur);
1080			count++;
1081			cur_len = 0;
1082			continue;
1083		}
1084
1085		s->offset += s_iova_off;
1086		s->length = s_length;
 
 
1087
1088		/*
1089		 * Now fill in the real DMA data. If...
1090		 * - there is a valid output segment to append to
1091		 * - and this segment starts on an IOVA page boundary
1092		 * - but doesn't fall at a segment boundary
1093		 * - and wouldn't make the resulting output segment too long
1094		 */
1095		if (cur_len && !s_iova_off && (dma_addr & seg_mask) &&
1096		    (max_len - cur_len >= s_length)) {
1097			/* ...then concatenate it with the previous one */
1098			cur_len += s_length;
1099		} else {
1100			/* Otherwise start the next output segment */
1101			if (i > 0)
1102				cur = sg_next(cur);
1103			cur_len = s_length;
1104			count++;
1105
1106			sg_dma_address(cur) = dma_addr + s_iova_off;
1107		}
1108
1109		sg_dma_len(cur) = cur_len;
1110		dma_addr += s_iova_len;
1111
1112		if (s_length + s_iova_off < s_iova_len)
1113			cur_len = 0;
1114	}
1115	return count;
1116}
1117
1118/*
1119 * If mapping failed, then just restore the original list,
1120 * but making sure the DMA fields are invalidated.
1121 */
1122static void __invalidate_sg(struct scatterlist *sg, int nents)
1123{
1124	struct scatterlist *s;
1125	int i;
1126
1127	for_each_sg(sg, s, nents, i) {
1128		if (sg_is_dma_bus_address(s)) {
1129			sg_dma_unmark_bus_address(s);
1130		} else {
1131			if (sg_dma_address(s) != DMA_MAPPING_ERROR)
1132				s->offset += sg_dma_address(s);
1133			if (sg_dma_len(s))
1134				s->length = sg_dma_len(s);
1135		}
1136		sg_dma_address(s) = DMA_MAPPING_ERROR;
1137		sg_dma_len(s) = 0;
1138	}
1139}
1140
1141static void iommu_dma_unmap_sg_swiotlb(struct device *dev, struct scatterlist *sg,
1142		int nents, enum dma_data_direction dir, unsigned long attrs)
1143{
1144	struct scatterlist *s;
1145	int i;
1146
1147	for_each_sg(sg, s, nents, i)
1148		iommu_dma_unmap_page(dev, sg_dma_address(s),
1149				sg_dma_len(s), dir, attrs);
1150}
1151
1152static int iommu_dma_map_sg_swiotlb(struct device *dev, struct scatterlist *sg,
1153		int nents, enum dma_data_direction dir, unsigned long attrs)
1154{
1155	struct scatterlist *s;
1156	int i;
1157
1158	for_each_sg(sg, s, nents, i) {
1159		sg_dma_address(s) = iommu_dma_map_page(dev, sg_page(s),
1160				s->offset, s->length, dir, attrs);
 
1161		if (sg_dma_address(s) == DMA_MAPPING_ERROR)
1162			goto out_unmap;
1163		sg_dma_len(s) = s->length;
1164	}
1165
1166	return nents;
1167
1168out_unmap:
1169	iommu_dma_unmap_sg_swiotlb(dev, sg, i, dir, attrs | DMA_ATTR_SKIP_CPU_SYNC);
1170	return -EIO;
1171}
1172
1173/*
1174 * The DMA API client is passing in a scatterlist which could describe
1175 * any old buffer layout, but the IOMMU API requires everything to be
1176 * aligned to IOMMU pages. Hence the need for this complicated bit of
1177 * impedance-matching, to be able to hand off a suitably-aligned list,
1178 * but still preserve the original offsets and sizes for the caller.
1179 */
1180static int iommu_dma_map_sg(struct device *dev, struct scatterlist *sg,
1181		int nents, enum dma_data_direction dir, unsigned long attrs)
1182{
1183	struct iommu_domain *domain = iommu_get_dma_domain(dev);
1184	struct iommu_dma_cookie *cookie = domain->iova_cookie;
1185	struct iova_domain *iovad = &cookie->iovad;
1186	struct scatterlist *s, *prev = NULL;
1187	int prot = dma_info_to_prot(dir, dev_is_dma_coherent(dev), attrs);
1188	struct pci_p2pdma_map_state p2pdma_state = {};
1189	enum pci_p2pdma_map_type map;
1190	dma_addr_t iova;
1191	size_t iova_len = 0;
1192	unsigned long mask = dma_get_seg_boundary(dev);
1193	ssize_t ret;
1194	int i;
1195
1196	if (static_branch_unlikely(&iommu_deferred_attach_enabled)) {
1197		ret = iommu_deferred_attach(dev, domain);
1198		if (ret)
1199			goto out;
1200	}
1201
1202	if (dev_use_swiotlb(dev))
1203		return iommu_dma_map_sg_swiotlb(dev, sg, nents, dir, attrs);
1204
1205	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1206		iommu_dma_sync_sg_for_device(dev, sg, nents, dir);
1207
 
 
 
1208	/*
1209	 * Work out how much IOVA space we need, and align the segments to
1210	 * IOVA granules for the IOMMU driver to handle. With some clever
1211	 * trickery we can modify the list in-place, but reversibly, by
1212	 * stashing the unaligned parts in the as-yet-unused DMA fields.
1213	 */
1214	for_each_sg(sg, s, nents, i) {
1215		size_t s_iova_off = iova_offset(iovad, s->offset);
1216		size_t s_length = s->length;
1217		size_t pad_len = (mask - iova_len + 1) & mask;
1218
1219		if (is_pci_p2pdma_page(sg_page(s))) {
1220			map = pci_p2pdma_map_segment(&p2pdma_state, dev, s);
1221			switch (map) {
1222			case PCI_P2PDMA_MAP_BUS_ADDR:
1223				/*
1224				 * iommu_map_sg() will skip this segment as
1225				 * it is marked as a bus address,
1226				 * __finalise_sg() will copy the dma address
1227				 * into the output segment.
1228				 */
1229				continue;
1230			case PCI_P2PDMA_MAP_THRU_HOST_BRIDGE:
1231				/*
1232				 * Mapping through host bridge should be
1233				 * mapped with regular IOVAs, thus we
1234				 * do nothing here and continue below.
1235				 */
1236				break;
1237			default:
1238				ret = -EREMOTEIO;
1239				goto out_restore_sg;
1240			}
1241		}
1242
1243		sg_dma_address(s) = s_iova_off;
1244		sg_dma_len(s) = s_length;
1245		s->offset -= s_iova_off;
1246		s_length = iova_align(iovad, s_length + s_iova_off);
1247		s->length = s_length;
1248
1249		/*
1250		 * Due to the alignment of our single IOVA allocation, we can
1251		 * depend on these assumptions about the segment boundary mask:
1252		 * - If mask size >= IOVA size, then the IOVA range cannot
1253		 *   possibly fall across a boundary, so we don't care.
1254		 * - If mask size < IOVA size, then the IOVA range must start
1255		 *   exactly on a boundary, therefore we can lay things out
1256		 *   based purely on segment lengths without needing to know
1257		 *   the actual addresses beforehand.
1258		 * - The mask must be a power of 2, so pad_len == 0 if
1259		 *   iova_len == 0, thus we cannot dereference prev the first
1260		 *   time through here (i.e. before it has a meaningful value).
1261		 */
1262		if (pad_len && pad_len < s_length - 1) {
1263			prev->length += pad_len;
1264			iova_len += pad_len;
1265		}
1266
1267		iova_len += s_length;
1268		prev = s;
1269	}
1270
1271	if (!iova_len)
1272		return __finalise_sg(dev, sg, nents, 0);
1273
1274	iova = iommu_dma_alloc_iova(domain, iova_len, dma_get_mask(dev), dev);
1275	if (!iova) {
1276		ret = -ENOMEM;
1277		goto out_restore_sg;
1278	}
1279
1280	/*
1281	 * We'll leave any physical concatenation to the IOMMU driver's
1282	 * implementation - it knows better than we do.
1283	 */
1284	ret = iommu_map_sg_atomic(domain, iova, sg, nents, prot);
1285	if (ret < 0 || ret < iova_len)
1286		goto out_free_iova;
1287
1288	return __finalise_sg(dev, sg, nents, iova);
1289
1290out_free_iova:
1291	iommu_dma_free_iova(cookie, iova, iova_len, NULL);
1292out_restore_sg:
1293	__invalidate_sg(sg, nents);
1294out:
1295	if (ret != -ENOMEM && ret != -EREMOTEIO)
1296		return -EINVAL;
1297	return ret;
1298}
1299
1300static void iommu_dma_unmap_sg(struct device *dev, struct scatterlist *sg,
1301		int nents, enum dma_data_direction dir, unsigned long attrs)
1302{
1303	dma_addr_t end = 0, start;
1304	struct scatterlist *tmp;
1305	int i;
1306
1307	if (dev_use_swiotlb(dev)) {
 
 
 
1308		iommu_dma_unmap_sg_swiotlb(dev, sg, nents, dir, attrs);
1309		return;
1310	}
1311
1312	if (!(attrs & DMA_ATTR_SKIP_CPU_SYNC))
1313		iommu_dma_sync_sg_for_cpu(dev, sg, nents, dir);
1314
1315	/*
1316	 * The scatterlist segments are mapped into a single
1317	 * contiguous IOVA allocation, the start and end points
1318	 * just have to be determined.
1319	 */
1320	for_each_sg(sg, tmp, nents, i) {
1321		if (sg_is_dma_bus_address(tmp)) {
1322			sg_dma_unmark_bus_address(tmp);
1323			continue;
1324		}
1325
1326		if (sg_dma_len(tmp) == 0)
1327			break;
1328
1329		start = sg_dma_address(tmp);
1330		break;
1331	}
1332
1333	nents -= i;
1334	for_each_sg(tmp, tmp, nents, i) {
1335		if (sg_is_dma_bus_address(tmp)) {
1336			sg_dma_unmark_bus_address(tmp);
1337			continue;
1338		}
1339
1340		if (sg_dma_len(tmp) == 0)
1341			break;
1342
1343		end = sg_dma_address(tmp) + sg_dma_len(tmp);
1344	}
1345
1346	if (end)
1347		__iommu_dma_unmap(dev, start, end - start);
1348}
1349
1350static dma_addr_t iommu_dma_map_resource(struct device *dev, phys_addr_t phys,
1351		size_t size, enum dma_data_direction dir, unsigned long attrs)
1352{
1353	return __iommu_dma_map(dev, phys, size,
1354			dma_info_to_prot(dir, false, attrs) | IOMMU_MMIO,
1355			dma_get_mask(dev));
1356}
1357
1358static void iommu_dma_unmap_resource(struct device *dev, dma_addr_t handle,
1359		size_t size, enum dma_data_direction dir, unsigned long attrs)
1360{
1361	__iommu_dma_unmap(dev, handle, size);
1362}
1363
1364static void __iommu_dma_free(struct device *dev, size_t size, void *cpu_addr)
1365{
1366	size_t alloc_size = PAGE_ALIGN(size);
1367	int count = alloc_size >> PAGE_SHIFT;
1368	struct page *page = NULL, **pages = NULL;
1369
1370	/* Non-coherent atomic allocation? Easy */
1371	if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
1372	    dma_free_from_pool(dev, cpu_addr, alloc_size))
1373		return;
1374
1375	if (is_vmalloc_addr(cpu_addr)) {
1376		/*
1377		 * If it the address is remapped, then it's either non-coherent
1378		 * or highmem CMA, or an iommu_dma_alloc_remap() construction.
1379		 */
1380		pages = dma_common_find_pages(cpu_addr);
1381		if (!pages)
1382			page = vmalloc_to_page(cpu_addr);
1383		dma_common_free_remap(cpu_addr, alloc_size);
1384	} else {
1385		/* Lowmem means a coherent atomic or CMA allocation */
1386		page = virt_to_page(cpu_addr);
1387	}
1388
1389	if (pages)
1390		__iommu_dma_free_pages(pages, count);
1391	if (page)
1392		dma_free_contiguous(dev, page, alloc_size);
1393}
1394
1395static void iommu_dma_free(struct device *dev, size_t size, void *cpu_addr,
1396		dma_addr_t handle, unsigned long attrs)
1397{
1398	__iommu_dma_unmap(dev, handle, size);
1399	__iommu_dma_free(dev, size, cpu_addr);
1400}
1401
1402static void *iommu_dma_alloc_pages(struct device *dev, size_t size,
1403		struct page **pagep, gfp_t gfp, unsigned long attrs)
1404{
1405	bool coherent = dev_is_dma_coherent(dev);
1406	size_t alloc_size = PAGE_ALIGN(size);
1407	int node = dev_to_node(dev);
1408	struct page *page = NULL;
1409	void *cpu_addr;
1410
1411	page = dma_alloc_contiguous(dev, alloc_size, gfp);
1412	if (!page)
1413		page = alloc_pages_node(node, gfp, get_order(alloc_size));
1414	if (!page)
1415		return NULL;
1416
1417	if (!coherent || PageHighMem(page)) {
1418		pgprot_t prot = dma_pgprot(dev, PAGE_KERNEL, attrs);
1419
1420		cpu_addr = dma_common_contiguous_remap(page, alloc_size,
1421				prot, __builtin_return_address(0));
1422		if (!cpu_addr)
1423			goto out_free_pages;
1424
1425		if (!coherent)
1426			arch_dma_prep_coherent(page, size);
1427	} else {
1428		cpu_addr = page_address(page);
1429	}
1430
1431	*pagep = page;
1432	memset(cpu_addr, 0, alloc_size);
1433	return cpu_addr;
1434out_free_pages:
1435	dma_free_contiguous(dev, page, alloc_size);
1436	return NULL;
1437}
1438
1439static void *iommu_dma_alloc(struct device *dev, size_t size,
1440		dma_addr_t *handle, gfp_t gfp, unsigned long attrs)
1441{
1442	bool coherent = dev_is_dma_coherent(dev);
1443	int ioprot = dma_info_to_prot(DMA_BIDIRECTIONAL, coherent, attrs);
1444	struct page *page = NULL;
1445	void *cpu_addr;
1446
1447	gfp |= __GFP_ZERO;
1448
1449	if (gfpflags_allow_blocking(gfp) &&
1450	    !(attrs & DMA_ATTR_FORCE_CONTIGUOUS)) {
1451		return iommu_dma_alloc_remap(dev, size, handle, gfp,
1452				dma_pgprot(dev, PAGE_KERNEL, attrs), attrs);
1453	}
1454
1455	if (IS_ENABLED(CONFIG_DMA_DIRECT_REMAP) &&
1456	    !gfpflags_allow_blocking(gfp) && !coherent)
1457		page = dma_alloc_from_pool(dev, PAGE_ALIGN(size), &cpu_addr,
1458					       gfp, NULL);
1459	else
1460		cpu_addr = iommu_dma_alloc_pages(dev, size, &page, gfp, attrs);
1461	if (!cpu_addr)
1462		return NULL;
1463
1464	*handle = __iommu_dma_map(dev, page_to_phys(page), size, ioprot,
1465			dev->coherent_dma_mask);
1466	if (*handle == DMA_MAPPING_ERROR) {
1467		__iommu_dma_free(dev, size, cpu_addr);
1468		return NULL;
1469	}
1470
1471	return cpu_addr;
1472}
1473
1474static int iommu_dma_mmap(struct device *dev, struct vm_area_struct *vma,
1475		void *cpu_addr, dma_addr_t dma_addr, size_t size,
1476		unsigned long attrs)
1477{
1478	unsigned long nr_pages = PAGE_ALIGN(size) >> PAGE_SHIFT;
1479	unsigned long pfn, off = vma->vm_pgoff;
1480	int ret;
1481
1482	vma->vm_page_prot = dma_pgprot(dev, vma->vm_page_prot, attrs);
1483
1484	if (dma_mmap_from_dev_coherent(dev, vma, cpu_addr, size, &ret))
1485		return ret;
1486
1487	if (off >= nr_pages || vma_pages(vma) > nr_pages - off)
1488		return -ENXIO;
1489
1490	if (is_vmalloc_addr(cpu_addr)) {
1491		struct page **pages = dma_common_find_pages(cpu_addr);
1492
1493		if (pages)
1494			return vm_map_pages(vma, pages, nr_pages);
1495		pfn = vmalloc_to_pfn(cpu_addr);
1496	} else {
1497		pfn = page_to_pfn(virt_to_page(cpu_addr));
1498	}
1499
1500	return remap_pfn_range(vma, vma->vm_start, pfn + off,
1501			       vma->vm_end - vma->vm_start,
1502			       vma->vm_page_prot);
1503}
1504
1505static int iommu_dma_get_sgtable(struct device *dev, struct sg_table *sgt,
1506		void *cpu_addr, dma_addr_t dma_addr, size_t size,
1507		unsigned long attrs)
1508{
1509	struct page *page;
1510	int ret;
1511
1512	if (is_vmalloc_addr(cpu_addr)) {
1513		struct page **pages = dma_common_find_pages(cpu_addr);
1514
1515		if (pages) {
1516			return sg_alloc_table_from_pages(sgt, pages,
1517					PAGE_ALIGN(size) >> PAGE_SHIFT,
1518					0, size, GFP_KERNEL);
1519		}
1520
1521		page = vmalloc_to_page(cpu_addr);
1522	} else {
1523		page = virt_to_page(cpu_addr);
1524	}
1525
1526	ret = sg_alloc_table(sgt, 1, GFP_KERNEL);
1527	if (!ret)
1528		sg_set_page(sgt->sgl, page, PAGE_ALIGN(size), 0);
1529	return ret;
1530}
1531
1532static unsigned long iommu_dma_get_merge_boundary(struct device *dev)
1533{
1534	struct iommu_domain *domain = iommu_get_dma_domain(dev);
1535
1536	return (1UL << __ffs(domain->pgsize_bitmap)) - 1;
1537}
1538
1539static size_t iommu_dma_opt_mapping_size(void)
1540{
1541	return iova_rcache_range();
1542}
1543
1544static const struct dma_map_ops iommu_dma_ops = {
1545	.flags			= DMA_F_PCI_P2PDMA_SUPPORTED,
1546	.alloc			= iommu_dma_alloc,
1547	.free			= iommu_dma_free,
1548	.alloc_pages		= dma_common_alloc_pages,
1549	.free_pages		= dma_common_free_pages,
 
1550	.alloc_noncontiguous	= iommu_dma_alloc_noncontiguous,
1551	.free_noncontiguous	= iommu_dma_free_noncontiguous,
 
1552	.mmap			= iommu_dma_mmap,
1553	.get_sgtable		= iommu_dma_get_sgtable,
1554	.map_page		= iommu_dma_map_page,
1555	.unmap_page		= iommu_dma_unmap_page,
1556	.map_sg			= iommu_dma_map_sg,
1557	.unmap_sg		= iommu_dma_unmap_sg,
1558	.sync_single_for_cpu	= iommu_dma_sync_single_for_cpu,
1559	.sync_single_for_device	= iommu_dma_sync_single_for_device,
1560	.sync_sg_for_cpu	= iommu_dma_sync_sg_for_cpu,
1561	.sync_sg_for_device	= iommu_dma_sync_sg_for_device,
1562	.map_resource		= iommu_dma_map_resource,
1563	.unmap_resource		= iommu_dma_unmap_resource,
1564	.get_merge_boundary	= iommu_dma_get_merge_boundary,
1565	.opt_mapping_size	= iommu_dma_opt_mapping_size,
1566};
1567
1568/*
1569 * The IOMMU core code allocates the default DMA domain, which the underlying
1570 * IOMMU driver needs to support via the dma-iommu layer.
1571 */
1572void iommu_setup_dma_ops(struct device *dev, u64 dma_base, u64 dma_limit)
1573{
1574	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1575
1576	if (!domain)
1577		goto out_err;
1578
1579	/*
1580	 * The IOMMU core code allocates the default DMA domain, which the
1581	 * underlying IOMMU driver needs to support via the dma-iommu layer.
1582	 */
1583	if (iommu_is_dma_domain(domain)) {
1584		if (iommu_dma_init_domain(domain, dma_base, dma_limit, dev))
1585			goto out_err;
1586		dev->dma_ops = &iommu_dma_ops;
1587	}
1588
1589	return;
1590out_err:
1591	 pr_warn("Failed to set up IOMMU for device %s; retaining platform DMA ops\n",
1592		 dev_name(dev));
1593}
1594EXPORT_SYMBOL_GPL(iommu_setup_dma_ops);
1595
1596static struct iommu_dma_msi_page *iommu_dma_get_msi_page(struct device *dev,
1597		phys_addr_t msi_addr, struct iommu_domain *domain)
1598{
1599	struct iommu_dma_cookie *cookie = domain->iova_cookie;
1600	struct iommu_dma_msi_page *msi_page;
1601	dma_addr_t iova;
1602	int prot = IOMMU_WRITE | IOMMU_NOEXEC | IOMMU_MMIO;
1603	size_t size = cookie_msi_granule(cookie);
1604
1605	msi_addr &= ~(phys_addr_t)(size - 1);
1606	list_for_each_entry(msi_page, &cookie->msi_page_list, list)
1607		if (msi_page->phys == msi_addr)
1608			return msi_page;
1609
1610	msi_page = kzalloc(sizeof(*msi_page), GFP_KERNEL);
1611	if (!msi_page)
1612		return NULL;
1613
1614	iova = iommu_dma_alloc_iova(domain, size, dma_get_mask(dev), dev);
1615	if (!iova)
1616		goto out_free_page;
1617
1618	if (iommu_map(domain, iova, msi_addr, size, prot))
1619		goto out_free_iova;
1620
1621	INIT_LIST_HEAD(&msi_page->list);
1622	msi_page->phys = msi_addr;
1623	msi_page->iova = iova;
1624	list_add(&msi_page->list, &cookie->msi_page_list);
1625	return msi_page;
1626
1627out_free_iova:
1628	iommu_dma_free_iova(cookie, iova, size, NULL);
1629out_free_page:
1630	kfree(msi_page);
1631	return NULL;
1632}
1633
1634/**
1635 * iommu_dma_prepare_msi() - Map the MSI page in the IOMMU domain
1636 * @desc: MSI descriptor, will store the MSI page
1637 * @msi_addr: MSI target address to be mapped
1638 *
1639 * Return: 0 on success or negative error code if the mapping failed.
1640 */
1641int iommu_dma_prepare_msi(struct msi_desc *desc, phys_addr_t msi_addr)
1642{
1643	struct device *dev = msi_desc_to_dev(desc);
1644	struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1645	struct iommu_dma_msi_page *msi_page;
1646	static DEFINE_MUTEX(msi_prepare_lock); /* see below */
1647
1648	if (!domain || !domain->iova_cookie) {
1649		desc->iommu_cookie = NULL;
1650		return 0;
1651	}
1652
1653	/*
1654	 * In fact the whole prepare operation should already be serialised by
1655	 * irq_domain_mutex further up the callchain, but that's pretty subtle
1656	 * on its own, so consider this locking as failsafe documentation...
1657	 */
1658	mutex_lock(&msi_prepare_lock);
1659	msi_page = iommu_dma_get_msi_page(dev, msi_addr, domain);
1660	mutex_unlock(&msi_prepare_lock);
1661
1662	msi_desc_set_iommu_cookie(desc, msi_page);
1663
1664	if (!msi_page)
1665		return -ENOMEM;
1666	return 0;
1667}
1668
1669/**
1670 * iommu_dma_compose_msi_msg() - Apply translation to an MSI message
1671 * @desc: MSI descriptor prepared by iommu_dma_prepare_msi()
1672 * @msg: MSI message containing target physical address
1673 */
1674void iommu_dma_compose_msi_msg(struct msi_desc *desc, struct msi_msg *msg)
1675{
1676	struct device *dev = msi_desc_to_dev(desc);
1677	const struct iommu_domain *domain = iommu_get_domain_for_dev(dev);
1678	const struct iommu_dma_msi_page *msi_page;
1679
1680	msi_page = msi_desc_get_iommu_cookie(desc);
1681
1682	if (!domain || !domain->iova_cookie || WARN_ON(!msi_page))
1683		return;
1684
1685	msg->address_hi = upper_32_bits(msi_page->iova);
1686	msg->address_lo &= cookie_msi_granule(domain->iova_cookie) - 1;
1687	msg->address_lo += lower_32_bits(msi_page->iova);
1688}
1689
1690static int iommu_dma_init(void)
1691{
1692	if (is_kdump_kernel())
1693		static_branch_enable(&iommu_deferred_attach_enabled);
1694
1695	return iova_cache_get();
1696}
1697arch_initcall(iommu_dma_init);